🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import type { FilterCondition } from "../../../types";
import type { OrderByState } from "../../../interfaces/orderBy";
import { InvalidRequestError } from "../../../errors/InvalidRequestError";
import { describe, expect, it } from "vitest";
import { convertDateToClickhouseDateTime } from "../../clickhouse/client";
import {
buildEventsObservationRowSelection,
groupEventsObservationFilters,
} from "./events-observation-row-selection";
import {
buildEventsBlobExportStreamQuery,
buildEventsStreamQuery,
} from "./events-stream-query";
const projectId = "project-characterization";
const cutoffCreatedAt = new Date("2025-01-02T03:04:05.678Z");
const nativeFilter: FilterCondition = {
column: "type",
operator: "any of",
value: ["GENERATION"],
type: "stringOptions",
};
const scoreFilter: FilterCondition = {
type: "numberObject",
column: "scores_avg",
operator: ">",
key: "quality",
value: 0.5,
};
const normalizeSql = (query: string) => query.replace(/\s+/g, " ").trim();
const buildSelection = ({
filter,
orderBy,
}: {
filter: FilterCondition[];
orderBy?: OrderByState;
}) => {
const selection = buildEventsObservationRowSelection({
projectId,
filter,
orderBy,
});
const built = selection.queryBuilder
.selectRaw("e.span_id AS id")
.buildWithParams();
return { ...selection, ...built };
};
describe("buildEventsStreamQuery", () => {
it("builds the common event-stream selection", () => {
const { queryBuilder } = buildEventsStreamQuery({
projectId,
cutoffCreatedAt,
filter: [nativeFilter],
searchQuery: "needle",
searchType: ["id", "content"],
rowLimit: 17,
});
const { query: rawQuery, params } = queryBuilder
.selectFieldSet("eval")
.buildWithParams();
const query = normalizeSql(rawQuery);
expect(query).toContain("e.project_id = {projectId: String}");
expect(query).toContain("e.is_deleted = 0");
expect(query).toContain("e.is_app_root as is_app_root");
expect(query).toContain("e.experiment_name as experiment_name");
const orderIndex = query.lastIndexOf("ORDER BY ");
const deduplicationIndex = query.lastIndexOf("LIMIT 1 BY ");
const rowLimitIndex = query.lastIndexOf("LIMIT {");
expect(orderIndex).toBeGreaterThan(-1);
expect(orderIndex).toBeLessThan(deduplicationIndex);
expect(deduplicationIndex).toBeLessThan(rowLimitIndex);
expect(params).toMatchObject({
projectId,
searchString: "%needle%",
limit: 17,
});
expect(Object.values(params)).toContainEqual(["GENERATION"]);
expect(Object.values(params)).toContain(
convertDateToClickhouseDateTime(cutoffCreatedAt),
);
});
it("omits the optional cutoff when it is absent", () => {
const { queryBuilder } = buildEventsStreamQuery({
projectId,
filter: null,
rowLimit: 7,
});
const { query, params } = queryBuilder
.selectFieldSet("eval")
.buildWithParams();
expect(normalizeSql(query)).not.toMatch(/e\."start_time" < \{/);
expect(params).toEqual({ projectId, limit: 7 });
});
it("reads events_full when a metadata filter is present", () => {
const { queryBuilder } = buildEventsStreamQuery({
projectId,
filter: [
{
column: "metadata",
key: "payload",
operator: "contains",
value: "needle",
type: "stringObject",
},
],
rowLimit: 7,
});
const { query } = queryBuilder.selectFieldSet("eval").buildWithParams();
expect(normalizeSql(query)).toContain("FROM events_full e");
});
it("reads events_core when neither search nor filters need full I/O", () => {
const { queryBuilder } = buildEventsStreamQuery({
projectId,
filter: [nativeFilter],
rowLimit: 7,
});
const { query } = queryBuilder.selectFieldSet("eval").buildWithParams();
expect(normalizeSql(query)).toContain("FROM events_core e");
});
it("shares one bounded score dependency between filtering and projection", () => {
const { queryBuilder, startTimeFrom } = buildEventsBlobExportStreamQuery({
projectId,
filter: [
{
type: "datetime",
column: "startTime",
operator: ">=",
value: new Date("2025-01-02T03:04:05.678Z"),
},
scoreFilter,
],
rowLimit: 7,
});
const { query, params } = queryBuilder.buildWithParams();
expect(startTimeFrom).toBe(params.startTimeFrom);
expect(query.match(/\bscores_agg AS \(/g)).toHaveLength(1);
expect(query).toContain("s.scores_avg as scores_avg");
expect(query).toContain("s.score_categories as score_categories");
expect(query).toContain(
"s.score_categories_tuples as score_categories_tuples",
);
expect(query).toContain(
"ON s.trace_id = e.trace_id AND s.observation_id = e.span_id",
);
expect(query).toContain(
"AND timestamp >= {startTimeFrom: DateTime64(3)} - INTERVAL 1 HOUR",
);
expect(query).not.toContain("name IN ({");
});
it("exports trace-level scores alongside observation-level ones (LFE-10596)", () => {
// No score filter at all: the export must still join and select the
// trace-score aggregate so trace-level scores land in the file exactly
// like the UI's unified Scores column.
const { queryBuilder } = buildEventsBlobExportStreamQuery({
projectId,
filter: [],
rowLimit: 7,
});
const { query } = queryBuilder.buildWithParams();
expect(query.match(/\btrace_scores_agg AS \(/g)).toHaveLength(1);
expect(query).toContain("ts.scores_avg as trace_scores_avg");
expect(query).toContain(
"ts.score_categories_tuples as trace_score_categories_tuples",
);
expect(query).toContain(
"ON ts.trace_id = e.trace_id AND ts.project_id = e.project_id",
);
});
it("applies score filters without exposing them as native event filters", () => {
const { queryBuilder } = buildEventsStreamQuery({
projectId,
filter: [
nativeFilter,
{
column: "Scores",
key: "quality",
operator: ">",
value: 0.5,
type: "numberObject",
},
],
rowLimit: 7,
});
const { params } = queryBuilder.selectFieldSet("eval").buildWithParams();
expect(Object.values(params)).toContain("quality");
});
});
describe("buildEventsObservationRowSelection", () => {
it("routes score filters and bounds both score scans from Events time", () => {
const startTime = new Date("2025-01-02T03:04:05.678Z");
const { query, params, filterGroups } = buildSelection({
filter: [
{
type: "datetime",
column: "startTime",
operator: ">=",
value: startTime,
},
{
type: "numberObject",
column: "scores_avg",
operator: ">",
key: "observation-quality",
value: 0.5,
},
{
type: "numberObject",
column: "trace_scores_avg",
operator: ">",
key: "trace-quality",
value: 0.5,
},
],
});
expect(filterGroups.events).toHaveLength(1);
expect(filterGroups.observationScores).toHaveLength(1);
expect(filterGroups.traceScores).toHaveLength(1);
expect(Object.values(params)).toContain("observation-quality");
expect(Object.values(params)).toContain("trace-quality");
expect(Object.values(params)).toContainEqual([
"observation-quality",
"trace-quality",
]);
expect(query.match(/\bname IN \(\{/g)).toHaveLength(2);
expect(query).toContain(
"AND timestamp >= {startTimeFrom: DateTime64(3)} - INTERVAL 1 HOUR",
);
expect(query).toContain(
"AND timestamp >= {startTimeFrom: DateTime64(3)} - INTERVAL 2 DAY - INTERVAL 1 HOUR",
);
});
it("does not invent score time bounds without an Events lower bound", () => {
const { query, startTimeFrom } = buildSelection({
filter: [
{
type: "numberObject",
column: "scores_avg",
operator: ">",
key: "observation-quality",
value: 0.5,
},
{
type: "numberObject",
column: "trace_scores_avg",
operator: ">",
key: "trace-quality",
value: 0.5,
},
],
});
expect(startTimeFrom).toBeNull();
expect(query).not.toContain("AND timestamp >=");
});
it("keeps complete score data when ordering by a score", () => {
const { query } = buildSelection({
filter: [scoreFilter],
orderBy: { column: "scores_avg", order: "DESC" },
});
expect(query).not.toContain("name IN ({");
for (const [column, expectedJoin] of [
["scores_avg", "LEFT JOIN scores_agg AS s"],
["trace_scores_avg", "LEFT JOIN trace_scores_agg AS ts"],
] as const) {
const scoreOrderQuery = buildSelection({
filter: [],
orderBy: { column, order: "DESC" },
}).query;
expect(scoreOrderQuery).toContain(expectedJoin);
}
});
it.each([
["scores_avg", "observationScores"],
["trace_scores_avg", "traceScores"],
["Scores", "observationScores"],
["SCORES", "observationScores"],
["Scores (numeric)", "observationScores"],
["Trace Scores (numeric)", "traceScores"],
] as const)(
"classifies the score column %s as %s",
(column, expectedGroup) => {
const filter: FilterCondition = {
type: "numberObject",
column,
operator: "=",
key: "score-key",
value: 1,
};
const filterGroups = groupEventsObservationFilters([filter]);
expect(filterGroups[expectedGroup]).toHaveLength(1);
},
);
it("rejects unresolved comment filters", () => {
expect(() =>
buildSelection({
filter: [
{
column: "commentContent",
operator: "contains",
value: "comment-needle",
type: "string",
},
],
}),
).toThrow(InvalidRequestError);
});
});