🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import type { ColumnDefinition } from "./tableDefinitions/types";
export const eventsTableHasParentObservationSql = "e.parent_span_id != ''";
export const eventsTableIsRootObservationSqlForAlias = (alias: string) => {
const prefix = alias ? `${alias}.` : "";
return `(${prefix}parent_span_id = '' OR ${prefix}is_app_root = true)`;
};
export const eventsTableIsRootObservationSql =
eventsTableIsRootObservationSqlForAlias("e");
export const eventsTableTraceNameSqlForAlias = (alias: string) =>
`COALESCE(nullIf(${alias}.trace_name, ''), if(${eventsTableIsRootObservationSqlForAlias(alias)}, nullIf(${alias}.name, ''), NULL))`;
export const eventsTableTraceNameSql = eventsTableTraceNameSqlForAlias("e");
// Row-projection variant. The fallback above is Nullable(String), but
// events_core.trace_name is a non-null String. Projecting the nullable
// expression under the column's own name while a filter reads the physical
// column puts two `trace_name` headers of different types into one pipeline,
// and ClickHouse 25.x rejects that with AMBIGUOUS_COLUMN_NAME (code 352) as
// soon as the query also sorts (LFE-14924). Matching the stored String type
// keeps the alias — an export/stream wire name — stable. "no trace name" is
// therefore '' on the wire; JS consumers map it back to null.
export const eventsTableTraceNameSelectSqlForAlias = (alias: string) =>
`ifNull(${eventsTableTraceNameSqlForAlias(alias)}, '')`;
export const eventsTableTraceNameSelectSql =
eventsTableTraceNameSelectSqlForAlias("e");
/**
* Maps the wire form of `eventsTableTraceNameSelectSql` ('' == no trace name)
* back to the null every JS-facing surface uses - tRPC/UI, the public API, the
* eval stream, analytics integrations, and batch export.
*
* Blob storage export deliberately does NOT use this: its published contract
* types `trace_name` as a plain string
* (https://langfuse.com/docs/api-and-data-platform/features/blob-storage-export-fields),
* and its raw JSONL / Parquet paths never pass through JS, so normalizing only
* the enriched path would make the three export formats disagree.
*/
export const normalizeEventsTraceName = (
traceName: string | null | undefined,
): string | null => traceName || null;
export const eventsTableTraceNameAggregationSqlForAlias = (alias: string) =>
`COALESCE(nullIf(argMaxIf(${alias}.trace_name, ${alias}.event_ts, ${alias}.trace_name <> ''), ''), nullIf(argMaxIf(${alias}.name, ${alias}.event_ts, (${eventsTableIsRootObservationSqlForAlias(alias)} AND ${alias}.name <> '')), ''))`;
export const eventsTableTraceNameAggregationSql =
eventsTableTraceNameAggregationSqlForAlias("e");
export const isRootObservation = ({
parentObservationId,
isAppRoot,
}: {
parentObservationId: string | null | undefined;
isAppRoot: boolean | null | undefined;
}): boolean => !parentObservationId || isAppRoot === true;
// True when the observation carries input / output. NULL/'' both count as
// absent (NULL != '' is NULL, i.e. not true), so only real payloads match.
export const eventsTableHasInputSql = "e.input != ''";
export const eventsTableHasOutputSql = "e.output != ''";
export const eventsTableLoadedSkillNamesSql =
"e.skills_resource_loaded.skillName";
export const eventsTableLoadedSkillVersionsSql =
"arrayMap(skill -> concat(skill.skillName, '@', toString(skill.skillVersion)), e.skills_resource_loaded)";
export const eventsTableAvailableSkillNamesSql = "e.skills_available.skillName";
export const eventsTableAvailableSkillVersionsSql =
"arrayMap(skill -> concat(skill.skillName, '@', toString(skill.skillVersion)), e.skills_available)";
const isCachedInputMetric = (key: string): boolean => {
const normalizedKey = key.toLowerCase();
return (
normalizedKey.includes("cached") || normalizedKey.includes("cache_read")
);
};
const findCachedInputMetric = (
details?: Record<string, number> | null,
): number | undefined => {
const values = Object.entries(details ?? {})
.filter(([key]) => isCachedInputMetric(key))
.map(([, value]) => Number(value));
return values.length > 0
? values.reduce((sum, value) => sum + value, 0)
: undefined;
};
export const getCachedInputMetric = (
details?: Record<string, number> | null,
): number | undefined => findCachedInputMetric(details);
export const getCachedInputCost = (
details?: Record<string, number> | null,
): number | undefined => findCachedInputMetric(details);
const cachedInputMetricSql = (detailsColumn: string): string => {
const keyPredicate = (key: string) =>
`(positionCaseInsensitive(${key}, 'cached') > 0 OR positionCaseInsensitive(${key}, 'cache_read') > 0)`;
const filteredDetails = `mapFilter(x -> ${keyPredicate("x.1")}, ${detailsColumn})`;
const sum = `arraySum(mapValues(${filteredDetails}))`;
return `if(mapExists((k, v) -> ${keyPredicate("k")}, ${detailsColumn}), ${sum}, NULL)`;
};
export const eventsTableCachedInputTokensSql =
cachedInputMetricSql("e.usage_details");
export const eventsTableCachedInputCostSql =
cachedInputMetricSql("e.cost_details");
type MutableDeep<T> = T extends readonly (infer U)[]
? MutableDeep<U>[]
: T extends object
? { -readonly [K in keyof T]: MutableDeep<T[K]> }
: T;
// Column definitions for the ClickHouse events table
// Used for filtering, sorting, and mapping UI columns to ClickHouse columns
const eventsTableColsDefinition = [
{
name: "ID",
id: "id",
type: "stringOptions",
internal: "e.span_id",
options: [], // to be added at runtime
},
{
name: "Trace ID",
id: "traceId",
type: "string",
internal: "e.trace_id",
},
{
name: "Start Time",
id: "startTime",
type: "datetime",
internal: "e.start_time",
},
{
name: "End Time",
id: "endTime",
type: "datetime",
internal: "e.end_time",
nullable: true,
},
{
name: "Name",
id: "name",
type: "stringOptions",
internal: "e.name",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Type",
id: "type",
type: "stringOptions",
internal: "e.type",
options: [],
},
{
name: "Environment",
id: "environment",
type: "stringOptions",
internal: "e.environment",
options: [], // to be added at runtime
nullable: true,
},
{
name: "API Key",
id: "ingestionApiKey",
type: "stringOptions",
internal: "e.ingestion_api_key",
options: [], // to be added at runtime
},
{
name: "SDK Name",
id: "ingestionSdkName",
type: "stringOptions",
internal: "e.ingestion_sdk_name",
options: [], // to be added at runtime
},
{
name: "SDK Version",
id: "ingestionSdkVersion",
type: "stringOptions",
internal: "e.ingestion_sdk_version",
options: [], // to be added at runtime
},
{
name: "Ingestion Source",
id: "ingestionSource",
type: "stringOptions",
internal: "e.source",
options: [], // to be added at runtime
},
{
name: "Version",
id: "version",
type: "string",
internal: "e.version",
nullable: true,
},
{
name: "Release",
id: "release",
type: "string",
internal: "e.release",
nullable: true,
},
{
name: "User ID",
id: "userId",
type: "string",
internal: "e.user_id",
nullable: true,
},
{
name: "Session ID",
id: "sessionId",
type: "string",
internal: "e.session_id",
nullable: true,
},
{
name: "Trace Name",
id: "traceName",
type: "stringOptions",
internal: eventsTableTraceNameSql,
options: [], // to be added at runtime
nullable: true,
},
{
name: "Status",
id: "level",
type: "stringOptions",
internal: "e.level",
options: [],
aliases: ["Level"],
},
{
name: "Status Message",
id: "statusMessage",
type: "string",
internal: "e.status_message",
nullable: true,
},
{
name: "Prompt Name",
id: "promptName",
type: "stringOptions",
internal: "e.prompt_name",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Prompt Version",
id: "promptVersion",
type: "number",
step: 1,
internal: "e.prompt_version",
nullable: true,
},
{
name: "Model ID",
id: "modelId",
type: "stringOptions",
internal: "e.model_id",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Provided Model Name",
id: "providedModelName",
type: "stringOptions",
internal: "e.provided_model_name",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Cost ($)",
id: "totalCost",
type: "number",
internal:
"if(mapExists((k, v) -> (k = 'total'), cost_details), cost_details['total'], NULL)",
nullable: true,
},
{
name: "Input Tokens",
id: "inputTokens",
type: "number",
step: 1,
internal:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",
nullable: true,
},
{
name: "Cached Input Tokens",
id: "cachedInputTokens",
type: "number",
step: 1,
internal: eventsTableCachedInputTokensSql,
nullable: true,
},
{
name: "Output Tokens",
id: "outputTokens",
type: "number",
step: 1,
internal:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))",
nullable: true,
},
{
name: "Total Tokens",
id: "totalTokens",
type: "number",
step: 1,
internal:
"if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
nullable: true,
},
{
name: "Input Cost ($)",
id: "inputCost",
type: "number",
internal:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, cost_details)))",
nullable: true,
},
{
name: "Cached Input Cost ($)",
id: "cachedInputCost",
type: "number",
internal: eventsTableCachedInputCostSql,
nullable: true,
},
{
name: "Output Cost ($)",
id: "outputCost",
type: "number",
internal:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, cost_details)))",
nullable: true,
},
{
name: "Latency (s)",
id: "latency",
type: "number",
internal: "date_diff('millisecond', e.start_time, e.end_time) / 1000.0",
nullable: true,
},
{
name: "Time To First Token (s)",
id: "timeToFirstToken",
type: "number",
internal:
"date_diff('millisecond', e.start_time, e.completion_start_time) / 1000.0",
nullable: true,
},
{
name: "Tokens per second",
id: "tokensPerSecond",
type: "number",
internal:
"(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('millisecond', start_time, end_time) / 1000))",
nullable: true,
},
{
name: "Input",
id: "input",
type: "string",
internal: "e.input",
nullable: true,
},
{
name: "Output",
id: "output",
type: "string",
internal: "e.output",
nullable: true,
},
{
name: "Metadata",
id: "metadata",
type: "stringObject",
internal: "e.metadata",
},
{
name: "Evaluator ID",
id: "evaluatorId",
type: "stringOptions",
internal: "e.evaluator_id",
options: [],
},
{
name: "Rule ID",
id: "ruleId",
type: "stringOptions",
internal: "e.evaluation_rule_id",
options: [],
},
{
name: "Trace Tags",
id: "traceTags",
type: "arrayOptions",
internal: "e.tags",
options: [], // to be added at runtime
},
{
name: "Scores (numeric)",
id: "scores_avg",
type: "numberObject",
internal: "scores_avg",
},
{
name: "Scores (categorical)",
id: "score_categories",
type: "categoryOptions",
internal: "score_categories",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Scores (boolean)",
id: "score_booleans",
type: "booleanObject",
internal: "score_booleans",
nullable: true,
},
{
name: "Trace Scores (numeric)",
id: "trace_scores_avg",
type: "numberObject",
internal: "trace_scores_avg",
},
{
name: "Trace Scores (categorical)",
id: "trace_score_categories",
type: "categoryOptions",
internal: "trace_score_categories",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Trace Scores (boolean)",
id: "trace_score_booleans",
type: "booleanObject",
internal: "trace_score_booleans",
nullable: true,
},
{
name: "Comment Count",
id: "commentCount",
type: "number",
step: 1,
internal: "", // handled by comment filter helpers
},
{
name: "Comment Content",
id: "commentContent",
type: "string",
internal: "", // handled by comment filter helpers
},
{
name: "Has Parent Observation",
id: "hasParentObservation",
type: "boolean",
internal: eventsTableHasParentObservationSql,
},
{
name: "Is Root Observation",
id: "isRootObservation",
type: "boolean",
internal: eventsTableIsRootObservationSql,
},
{
name: "Has Input",
id: "hasInput",
type: "boolean",
internal: eventsTableHasInputSql,
},
{
name: "Has Output",
id: "hasOutput",
type: "boolean",
internal: eventsTableHasOutputSql,
},
{
name: "Experiment Dataset ID",
id: "experimentDatasetId",
type: "stringOptions",
internal: "e.experiment_dataset_id",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Experiment ID",
id: "experimentId",
type: "stringOptions",
internal: "e.experiment_id",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Experiment Name",
id: "experimentName",
type: "stringOptions",
internal: "e.experiment_name",
options: [], // to be added at runtime
nullable: true,
},
{
name: "Loaded Skill Names",
id: "skillsNamesLoaded",
type: "arrayOptions",
internal: eventsTableLoadedSkillNamesSql,
options: [],
},
{
name: "Loaded Skill Versions",
id: "skillsVersionsLoaded",
type: "arrayOptions",
internal: eventsTableLoadedSkillVersionsSql,
options: [],
},
{
name: "Available Skill Names",
id: "skillNamesAvailable",
type: "arrayOptions",
internal: eventsTableAvailableSkillNamesSql,
options: [],
},
{
name: "Available Skill Versions",
id: "skillVersionsAvailable",
type: "arrayOptions",
internal: eventsTableAvailableSkillVersionsSql,
options: [],
},
{
name: "Available Tool Names",
id: "toolNames",
type: "arrayOptions",
internal: "mapKeys(e.tool_definitions)",
options: [],
nullable: true,
},
{
name: "Called Tool Names",
id: "calledToolNames",
type: "arrayOptions",
internal: "e.tool_call_names",
options: [],
nullable: true,
},
{
name: "Available Tools",
id: "toolDefinitions",
type: "number",
step: 1,
internal: "length(mapKeys(e.tool_definitions))",
nullable: true,
},
{
name: "Tool Calls",
id: "toolCalls",
type: "number",
step: 1,
internal: "length(e.tool_calls)",
nullable: true,
},
{
name: "Is Experiment Item Root Span",
id: "isExperimentItemRootSpan",
type: "boolean",
internal: "e.experiment_item_root_span_id = e.span_id",
},
] as const satisfies readonly ColumnDefinition[];
// TODO: Remove MutableDeep once consumers accept readonly column definitions.
export const eventsTableCols =
eventsTableColsDefinition as unknown as MutableDeep<
typeof eventsTableColsDefinition
> &
ColumnDefinition[];
type EventsTableColumnId = (typeof eventsTableColsDefinition)[number]["id"];
export type NumericEventsTableColumnId = Extract<
(typeof eventsTableColsDefinition)[number],
{ type: "number" }
>["id"];
export const isNumericEventsTableColumnId = (
column: EventsTableColumnId,
): column is NumericEventsTableColumnId =>
eventsTableColsDefinition.some(
(col) => col.id === column && col.type === "number",
);
// Subset of columns that are allowed to be used as filters in the MCP observations API
// listObservations has explicit fromStartTime and toStartTime arguments to bound database scans.
// A startTime condition in its filter array would replace both bounds, bypassing the 14-day limit.
const OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMN_IDS = [
"id",
"traceId",
"endTime",
"name",
"type",
"environment",
"version",
"userId",
"sessionId",
"traceName",
"level",
"statusMessage",
"promptName",
"promptVersion",
"modelId",
"providedModelName",
"totalCost",
"inputTokens",
"outputTokens",
"totalTokens",
"inputCost",
"outputCost",
"latency",
"timeToFirstToken",
"input",
"output",
"metadata",
"traceTags",
"isRootObservation",
"hasParentObservation",
] as const satisfies readonly EventsTableColumnId[];
export type ObservationMcpAllowedEventsTableFilterColumn =
(typeof OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMN_IDS)[number];
export const OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMNS: ReadonlySet<EventsTableColumnId> =
new Set(OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMN_IDS);