🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it } from "vitest";
import {
buildEventsFullTableSplitQuery,
EventsAggregationQueryBuilder,
EventsQueryBuilder,
EventsSessionAggregationQueryBuilder,
ExperimentsAggregationQueryBuilder,
} from "./event-query-builder";
describe("EventsQueryBuilder public API v2 field groups", () => {
it.each([
["basic", true],
["core", false],
] as const)(
"projects semantic root status for %s: %s",
(fieldSet, selected) => {
const query = new EventsQueryBuilder({ projectId: "test-project" })
.selectFieldSet(fieldSet)
.buildWithParams().query;
expect(query.includes('"is_root_observation"')).toBe(selected);
},
);
it.each([
{
name: "usage vs metadata",
orderA: ["core", "basic", "usage", "metadata"] as const,
orderB: ["core", "basic", "metadata", "usage"] as const,
},
{
name: "usage vs metrics",
orderA: ["core", "basic", "usage", "metrics"] as const,
orderB: ["core", "basic", "metrics", "usage"] as const,
},
{
name: "usage vs trace_context",
orderA: ["core", "basic", "usage", "trace_context"] as const,
orderB: ["core", "basic", "trace_context", "usage"] as const,
},
])(
"emits identical SQL regardless of field-group order ($name)",
({ orderA, orderB }) => {
const queryFor = (sets: typeof orderA | typeof orderB) =>
new EventsQueryBuilder({ projectId: "test-project" })
.selectFieldSet(...sets)
.buildWithParams().query;
expect(queryFor(orderA)).toBe(queryFor(orderB));
},
);
});
describe("EventsAggregationQueryBuilder", () => {
it("counts distinct non-synthetic observations per trace", () => {
const { query } = new EventsAggregationQueryBuilder({
projectId: "test-project",
})
.selectFieldSet("all")
.buildWithParams();
expect(query).toContain(
"length(groupUniqArrayIf(e.span_id, (e.span_id <> '' and e.span_id <> concat('t-', e.trace_id)))) as observation_count",
);
expect(query).toContain(
"(e.parent_span_id = '' or e.is_app_root = true) and e.name <> ''",
);
});
it("promotes evaluator execution fields into the trace aggregation", () => {
const { query } = new EventsAggregationQueryBuilder({
projectId: "test-project",
})
.selectFieldSet("all")
.buildWithParams();
expect(query).toContain(
"argMaxIf(e.evaluator_id, e.event_ts, e.evaluator_id != '') as evaluator_id",
);
expect(query).toContain(
"argMaxIf(e.evaluation_rule_id, e.event_ts, e.evaluation_rule_id != '') as evaluation_rule_id",
);
});
});
describe("EventsSessionAggregationQueryBuilder", () => {
it("selects metadata arrays from the same deterministic latest observation", () => {
const { query } = new EventsSessionAggregationQueryBuilder({
projectId: "test-project",
})
.selectFieldSet("metadata")
.buildWithParams();
expect(query).toContain(
"argMax(e.metadata_names, tuple(e.start_time, e.event_ts, e.span_id)) as metadata_names",
);
expect(query).toContain(
"argMax(e.metadata_values, tuple(e.start_time, e.event_ts, e.span_id)) as metadata_values",
);
expect(query).toContain("e.project_id = {projectId: String}");
});
it("omits metadata aggregation from the base field set", () => {
const { query } = new EventsSessionAggregationQueryBuilder({
projectId: "test-project",
})
.selectFieldSet("base")
.buildWithParams();
expect(query).not.toContain("metadata_names");
expect(query).not.toContain("metadata_values");
expect(query).not.toContain("tool_definitions");
expect(query).not.toContain("tool_calls");
});
it("selects only the session fields the caller lists", () => {
const { query } = new EventsSessionAggregationQueryBuilder({
projectId: "test-project",
})
.selectSessionFields(["session_id", "trace_count"])
.buildWithParams();
expect(query).toContain("uniq(e.trace_id) as trace_count");
expect(query).not.toContain("trace_ids");
expect(query).not.toContain("sumMap");
});
it("selects tool aggregates only when requested", () => {
const { query } = new EventsSessionAggregationQueryBuilder({
projectId: "test-project",
})
.selectFieldSet("base", "tools")
.buildWithParams();
expect(query).toContain(
"groupUniqArrayArray(mapKeys(e.tool_definitions)) as tool_names",
);
expect(query).toContain(
"groupUniqArrayArray(e.tool_call_names) as called_tool_names",
);
expect(query).toContain("sum(length(e.tool_calls)) as tool_calls_count");
});
});
describe("EventsQueryBuilder.selectIOWithSizeCap", () => {
const build = () =>
new EventsQueryBuilder({ projectId: "test-project" })
.selectFieldSet("base", "calculated", "metadata")
.selectIOWithSizeCap(300_000, 4_000)
.whereRaw("e.trace_id = {traceId: String}", { traceId: "trace-1" })
.buildWithParams();
it("returns full fields under the cap and a preview head above it", () => {
const { query } = build();
// lengthUTF8() is computed in the query on purpose: the materialized
// input_length/output_length columns cannot be assumed present on every
// deployment's events_full, and the full column is read here anyway.
expect(query).toContain(
"if(lengthUTF8(e.input) <= 300000, e.input, leftUTF8(e.input, 4000)) as input",
);
expect(query).toContain(
"if(lengthUTF8(e.output) <= 300000, e.output, leftUTF8(e.output, 4000)) as output",
);
});
it("exposes the true lengths so callers can detect previews", () => {
const { query } = build();
expect(query).toContain("lengthUTF8(e.input) as input_length");
expect(query).toContain("lengthUTF8(e.output) as output_length");
});
it("caps metadata values with the same policy and flags truncation", () => {
const { query } = build();
expect(query).toContain(
"arrayMap(v -> if(lengthUTF8(v) <= 300000, v, leftUTF8(v, 4000)), arrayReverse(e.metadata_values))",
);
// The flag only fires for a key's winning value (a shadowed duplicate
// must not raise it), and the shipped weight counts every capped value.
expect(query).toContain(
"arrayExists((v, i) -> lengthUTF8(v) > 300000 AND arrayFirstIndex(n -> n = e.metadata_names[i], e.metadata_names) = i, e.metadata_values, arrayEnumerate(e.metadata_values)) as metadata_truncated",
);
expect(query).toContain(
"arraySum(arrayMap(v -> if(lengthUTF8(v) <= 300000, lengthUTF8(v), 4000), e.metadata_values)) as metadata_length",
);
// The default full-value metadata expression must not also be present.
expect(query).not.toContain(
"mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata",
);
});
it("reads events_full (true lengths + full under-cap values)", () => {
const { query } = build();
expect(query).toContain("FROM events_full");
});
});
describe("buildEventsFullTableSplitQuery", () => {
const buildBase = () =>
new EventsQueryBuilder({ projectId: "test-project" })
.selectFieldSet("core")
.whereRaw("e.trace_id = {traceId: String}", { traceId: "trace-1" })
.orderByColumns([{ column: "e.start_time", direction: "DESC" }])
.limit(51, undefined);
it("joins the io CTE with LEFT ANY JOIN (no row fan-out)", () => {
const { query } = buildEventsFullTableSplitQuery({
projectId: "test-project",
baseBuilder: buildBase(),
includeIO: true,
includeMetadata: true,
}).buildWithParams();
// ANY takes exactly one matching io row per base row, preventing the
// N_base x N_io amplification a plain LEFT JOIN produces on un-merged
// ReplacingMergeTree duplicates.
expect(query).toContain(
'LEFT ANY JOIN io i ON b."start_time" = i."_io_start_time"',
);
expect(query).not.toMatch(/\bLEFT JOIN io\b/);
// io columns come from the joined events_full CTE.
expect(query).toContain("i.input as input");
expect(query).toContain("i.output as output");
expect(query).toContain("i.metadata as metadata");
});
it("bounds the io lane to base's start_time range and keeps the semi-join", () => {
const { query } = buildEventsFullTableSplitQuery({
projectId: "test-project",
baseBuilder: buildBase(),
includeIO: true,
includeMetadata: false,
}).buildWithParams();
// The bound is derived from base (no re-serialized params) so events_full
// can prune partitions/primary key; the semi-join stays for join exactness.
// Both bounds read one byte-identical (min, max) scalar subquery so
// ClickHouse's scalar cache evaluates base's bounds pass once, not twice.
expect(query).not.toContain("io_bounds");
expect(query).toContain(
"AND e.start_time >= (SELECT (min(start_time), max(start_time)) FROM base).1",
);
expect(query).toContain(
"AND e.start_time <= (SELECT (min(start_time), max(start_time)) FROM base).2",
);
expect(query).toContain(
'AND (e.start_time, e.trace_id, e.span_id) IN (SELECT "start_time", "trace_id", id FROM base)',
);
});
});
describe("ExperimentsAggregationQueryBuilder", () => {
it("reads non propagated experiment-level attributes from experiment item root spans", () => {
const { query } = new ExperimentsAggregationQueryBuilder({
projectId: "test-project",
})
.selectFieldSet("base")
.buildWithParams();
expect(query).toContain(
"anyIf(e.experiment_description, e.span_id = e.experiment_item_root_span_id) as experiment_description",
);
});
});