🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import type { z } from "zod";
import type { PrismaClient } from "../../db";
import { eventsTableTraceNameSelectSql } from "../../eventsTable";
import type { topicTraceSelectionCriteriaSchema } from "../../topics";
import { applyCommentFilters } from "../services/commentFilterService";
import { CTEQueryBuilder } from "../queries/clickhouse-sql/event-query-builder";
import { buildEventsObservationRowSelection } from "../queries/clickhouse-sql/events-observation-row-selection";
import { queryClickhouse } from "../repositories/clickhouse";
import {
INTERNAL_EVAL_ENVIRONMENT_PREFIX,
PUBLIC_LLM_JUDGE_ENVIRONMENT,
} from "../llm/isInternalEvalEnvironment";
type SelectionRows = {
ids: { id: string };
backfill: {
id: string;
timestampMs: string | number;
latestMs: string | number;
matchedTraceCount: string | number;
};
preview: {
id: string;
timestampMs: string | number;
name: string;
environment: string;
matchedTraceCount: string | number;
};
};
/** Select trace identities from matching root observations. */
export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(
input: z.infer<typeof topicTraceSelectionCriteriaSchema> & {
projectId: string;
},
prisma: PrismaClient,
mode: Mode,
): Promise<SelectionRows[Mode][]> {
const { filterState, hasNoMatches } = await applyCommentFilters({
projectId: input.projectId,
prisma,
objectType: "OBSERVATION",
filterState: input.filter.flatMap((filter) => {
if (filter.column === "isRootObservation") return [];
if (filter.column === "tags")
return [{ ...filter, column: "traceTags" as const }];
return [filter];
}),
});
if (hasNoMatches) return [];
const { queryBuilder } = buildEventsObservationRowSelection({
projectId: input.projectId,
filter: filterState.concat([
{
column: "isRootObservation",
type: "boolean",
operator: "=",
value: true,
},
{
column: "startTime",
type: "datetime",
operator: ">=",
value: input.from,
},
{ column: "startTime", type: "datetime", operator: "<", value: input.to },
]),
});
const preview = mode === "preview";
const matching = queryBuilder
.selectRaw(
"e.trace_id AS id",
"e.start_time AS start_time",
"e.span_id AS span_id",
"e.event_ts AS event_ts",
"e.environment AS environment",
...(preview ? [`${eventsTableTraceNameSelectSql} AS trace_name`] : []),
)
.whereRaw("e.trace_id != ''")
.buildWithParams();
const traces = new CTEQueryBuilder()
.withCTE("matching_observations", {
...matching,
schema: [
"id",
"start_time",
"span_id",
"event_ts",
"environment",
...(preview ? (["trace_name"] as const) : []),
] as const,
})
.from("matching_observations", "m")
.select(
"m.id AS id",
"min(m.start_time) AS timestamp",
"max(m.start_time) AS latest_match",
"argMax(m.environment, tuple(m.event_ts, m.span_id, m.environment)) AS environment",
...(preview
? [
"argMaxIf(m.trace_name, tuple(m.event_ts, m.span_id, m.trace_name), m.trace_name != '') AS name",
]
: []),
)
.groupBy("m.id")
.havingRaw(
"NOT startsWith(environment, {internalEnvironmentPrefix:String}) AND environment != {publicLlmJudgeEnvironment:String}",
{
internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
},
)
.buildWithParams();
const selection = new CTEQueryBuilder()
.withCTE("matching_traces", {
...traces,
schema: [
"id",
"timestamp",
"latest_match",
"environment",
...(preview ? (["name"] as const) : []),
] as const,
})
.from("matching_traces", "t")
.select(
"t.id AS id",
...(mode === "ids"
? []
: [
"toUnixTimestamp64Milli(t.timestamp) AS timestampMs",
"count() OVER () AS matchedTraceCount",
...(preview
? ["t.name AS name", "t.environment AS environment"]
: ["toUnixTimestamp64Milli(t.latest_match) AS latestMs"]),
]),
)
.orderByColumns([
input.sampling === "random"
? {
column: "cityHash64(t.id, {samplingSeed: String})",
direction: "ASC",
}
: { column: "t.latest_match", direction: "DESC" },
{ column: "t.id", direction: "ASC" },
]);
const limit = preview ? Math.min(input.limit ?? Infinity, 100) : input.limit;
if (limit !== null) selection.limit(limit);
const selected = selection.buildWithParams();
return queryClickhouse<SelectionRows[Mode]>({
...selected,
params: { ...selected.params, samplingSeed: input.seed },
preferredClickhouseService: "EventsReadOnly",
tags: { projectId: input.projectId, route: "topics-trace-selection" },
clickhouseSettings: {
max_threads: 2,
max_execution_time: 30,
timeout_overflow_mode: "throw",
},
});
}