🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
// This structure is maintained to relate the frontend table definitions with the clickhouse table definitions.
// The frontend only sends the column names to the backend. This needs to be changed in the future to send column IDs.
import { UiColumnMappings } from "../../tableDefinitions/types";
import {
eventsTableHasParentObservationSql,
eventsTableIsRootObservationSql,
eventsTableHasInputSql,
eventsTableHasOutputSql,
eventsTableCachedInputCostSql,
eventsTableCachedInputTokensSql,
eventsTableTraceNameSql,
eventsTableLoadedSkillNamesSql,
eventsTableLoadedSkillVersionsSql,
eventsTableAvailableSkillNamesSql,
eventsTableAvailableSkillVersionsSql,
} from "../../eventsTable";
import { tokenCountFilterClickhouseType } from "./tokenCountFilterType";
export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [
{
uiTableName: "Environment",
uiTableId: "environment",
clickhouseTableName: "events_proto",
clickhouseSelect: "environment",
queryPrefix: "e",
},
{
uiTableName: "API Key",
uiTableId: "ingestionApiKey",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."ingestion_api_key"',
},
{
uiTableName: "SDK Name",
uiTableId: "ingestionSdkName",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."ingestion_sdk_name"',
},
{
uiTableName: "SDK Version",
uiTableId: "ingestionSdkVersion",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."ingestion_sdk_version"',
},
{
uiTableName: "Ingestion Source",
uiTableId: "ingestionSource",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."source"',
},
{
uiTableName: "Type",
uiTableId: "type",
clickhouseTableName: "events_proto",
clickhouseSelect: "type",
queryPrefix: "e",
},
{
uiTableName: "ID",
uiTableId: "id",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."span_id"',
},
{
uiTableName: "Name",
uiTableId: "name",
clickhouseTableName: "events_proto",
clickhouseSelect: "name",
queryPrefix: "e",
},
{
uiTableName: "Trace ID",
uiTableId: "traceId",
clickhouseTableName: "events_proto",
clickhouseSelect: "trace_id",
queryPrefix: "e",
},
{
uiTableName: "Start Time",
uiTableId: "startTime",
clickhouseTableName: "events_proto",
clickhouseSelect: "start_time",
queryPrefix: "e",
},
{
uiTableName: "End Time",
uiTableId: "endTime",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."end_time"',
},
{
uiTableName: "Time To First Token (s)",
uiTableId: "timeToFirstToken",
clickhouseTableName: "events_proto",
clickhouseSelect:
"if(isNull(e.completion_start_time), NULL, date_diff('millisecond', e.start_time, e.completion_start_time) / 1000)",
// If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
clickhouseTypeOverwrite: "Decimal64(3)",
},
{
uiTableName: "Latency (s)",
uiTableId: "latency",
clickhouseTableName: "events_proto",
clickhouseSelect:
"if(isNull(e.end_time), NULL, date_diff('millisecond', e.start_time, e.end_time) / 1000)",
// If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
clickhouseTypeOverwrite: "Decimal64(3)",
},
{
uiTableName: "Tokens per second",
uiTableId: "tokensPerSecond",
clickhouseTableName: "events_proto",
clickhouseSelect:
"(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('millisecond', start_time, end_time) / 1000))",
},
{
uiTableName: "Input Cost ($)",
uiTableId: "inputCost",
clickhouseTableName: "events_proto",
clickhouseSelect:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, cost_details)))",
},
{
uiTableName: "Cached Input Cost ($)",
uiTableId: "cachedInputCost",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableCachedInputCostSql,
},
{
uiTableName: "Output Cost ($)",
uiTableId: "outputCost",
clickhouseTableName: "events_proto",
clickhouseSelect:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, cost_details)))",
},
{
uiTableName: "Total Cost ($)",
uiTableId: "totalCost",
clickhouseTableName: "events_proto",
clickhouseSelect:
"if(mapExists((k, v) -> (k = 'total'), cost_details), cost_details['total'], NULL)",
},
{
uiTableName: "Status",
uiTableId: "level",
aliases: ["Level"],
clickhouseTableName: "events_proto",
clickhouseSelect: "level",
queryPrefix: "e",
},
{
uiTableName: "Status Message",
uiTableId: "statusMessage",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."status_message"',
},
{
uiTableName: "Model",
uiTableId: "model",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."provided_model_name"',
},
{
uiTableName: "Provided Model Name",
uiTableId: "providedModelName",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."provided_model_name"',
},
{
uiTableName: "Model ID",
uiTableId: "modelId",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."model_id"',
},
{
uiTableName: "Input Tokens",
uiTableId: "inputTokens",
clickhouseTableName: "events_proto",
clickhouseSelect:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",
clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
},
{
uiTableName: "Cached Input Tokens",
uiTableId: "cachedInputTokens",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableCachedInputTokensSql,
clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
},
{
uiTableName: "Output Tokens",
uiTableId: "outputTokens",
clickhouseTableName: "events_proto",
clickhouseSelect:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))",
clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
},
{
uiTableName: "Total Tokens",
uiTableId: "totalTokens",
clickhouseTableName: "events_proto",
clickhouseSelect:
"if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
},
{
uiTableName: "Tokens",
uiTableId: "tokens",
clickhouseTableName: "events_proto",
clickhouseSelect:
"if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
},
{
uiTableName: "Metadata",
uiTableId: "metadata",
clickhouseTableName: "events_proto",
clickhouseSelect: "metadata",
queryPrefix: "e",
},
{
uiTableName: "Evaluator ID",
uiTableId: "evaluatorId",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.evaluator_id",
},
{
uiTableName: "Rule ID",
uiTableId: "ruleId",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.evaluation_rule_id",
},
{
uiTableName: "Version",
uiTableId: "version",
clickhouseTableName: "events_proto",
clickhouseSelect: "version",
queryPrefix: "e",
},
{
uiTableName: "Release",
uiTableId: "release",
clickhouseTableName: "events_proto",
clickhouseSelect: "release",
queryPrefix: "e",
emptyEqualsNull: true,
},
{
uiTableName: "Prompt Name",
uiTableId: "promptName",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.prompt_name",
},
{
uiTableName: "Prompt Version",
uiTableId: "promptVersion",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.prompt_version",
},
{
uiTableName: "Input",
uiTableId: "input",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.input",
},
{
uiTableName: "Output",
uiTableId: "output",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.output",
},
{
uiTableName: "Session ID",
uiTableId: "sessionId",
clickhouseTableName: "events_proto",
clickhouseSelect: "session_id",
queryPrefix: "e",
},
{
uiTableName: "Trace Name",
uiTableId: "traceName",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableTraceNameSql,
},
{
uiTableName: "User ID",
uiTableId: "userId",
clickhouseTableName: "events_proto",
clickhouseSelect: "user_id",
queryPrefix: "e",
},
{
uiTableName: "Trace Tags",
uiTableId: "traceTags",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."tags"',
},
{
uiTableName: "Tags",
uiTableId: "tags",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."tags"',
},
{
uiTableName: "Trace Environment",
uiTableId: "traceEnvironment",
clickhouseTableName: "events_proto",
clickhouseSelect: "environment",
queryPrefix: "e",
},
{
uiTableName: "Has Parent Observation",
uiTableId: "hasParentObservation",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableHasParentObservationSql,
},
{
uiTableName: "Is Root Observation",
uiTableId: "isRootObservation",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableIsRootObservationSql,
},
{
uiTableName: "Has Input",
uiTableId: "hasInput",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableHasInputSql,
},
{
uiTableName: "Has Output",
uiTableId: "hasOutput",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableHasOutputSql,
},
{
uiTableName: "Parent Observation ID",
uiTableId: "parentObservationId",
clickhouseTableName: "events_proto",
clickhouseSelect: "parent_span_id",
queryPrefix: "e",
emptyEqualsNull: true,
},
{
uiTableName: "Experiment Dataset ID",
uiTableId: "experimentDatasetId",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."experiment_dataset_id"',
},
{
uiTableName: "Experiment ID",
uiTableId: "experimentId",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."experiment_id"',
emptyEqualsNull: true,
},
{
uiTableName: "Experiment Name",
uiTableId: "experimentName",
clickhouseTableName: "events_proto",
clickhouseSelect: 'e."experiment_name"',
},
{
uiTableName: "Is Experiment Item Root Span",
uiTableId: "isExperimentItemRootSpan",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.experiment_item_root_span_id = e.span_id",
},
{
uiTableName: "Available Tools",
uiTableId: "toolDefinitions",
clickhouseTableName: "events_proto",
clickhouseSelect: "length(mapKeys(e.tool_definitions))",
},
{
uiTableName: "Tool Calls",
uiTableId: "toolCalls",
clickhouseTableName: "events_proto",
clickhouseSelect: "length(e.tool_calls)",
},
{
uiTableName: "Loaded Skill Names",
uiTableId: "skillsNamesLoaded",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableLoadedSkillNamesSql,
},
{
uiTableName: "Loaded Skill Versions",
uiTableId: "skillsVersionsLoaded",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableLoadedSkillVersionsSql,
},
{
uiTableName: "Loaded Skill Files",
uiTableId: "skillFilesLoaded",
clickhouseTableName: "events_proto",
// Pair each path with its immutable skill-version ID to keep matches within one loaded resource.
clickhouseSelect:
"arrayMap(skill -> concat(skill.langfuseSkillId, '/', skill.filePath), e.skills_resource_loaded)",
},
{
uiTableName: "Available Skill Names",
uiTableId: "skillNamesAvailable",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableAvailableSkillNamesSql,
},
{
uiTableName: "Available Skill Versions",
uiTableId: "skillVersionsAvailable",
clickhouseTableName: "events_proto",
clickhouseSelect: eventsTableAvailableSkillVersionsSql,
},
{
uiTableName: "Skill Names",
uiTableId: "skillNames",
clickhouseTableName: "events_proto",
clickhouseSelect: `arrayConcat(${eventsTableLoadedSkillNamesSql}, ${eventsTableAvailableSkillNamesSql})`,
},
{
uiTableName: "Skill Versions",
uiTableId: "skillVersions",
clickhouseTableName: "events_proto",
clickhouseSelect: `arrayConcat(${eventsTableLoadedSkillVersionsSql}, ${eventsTableAvailableSkillVersionsSql})`,
},
{
uiTableName: "Tool Names",
uiTableId: "toolNames",
clickhouseTableName: "events_proto",
clickhouseSelect: "mapKeys(e.tool_definitions)",
},
{
uiTableName: "Called Tool Names",
uiTableId: "calledToolNames",
clickhouseTableName: "events_proto",
clickhouseSelect: "e.tool_call_names",
},
];
export const eventsTableUiColumnDefinitions: UiColumnMappings = [
...eventsTableNativeUiColumnDefinitions,
// Scores column duplicated to allow renaming column name. Will be removed once session storage cache is outdated
// Column names are cached in user sessions - changing them breaks existing filters
{
uiTableName: "Scores",
uiTableId: "scores",
clickhouseTableName: "scores",
clickhouseSelect: "s.scores_avg",
},
{
uiTableName: "Scores (numeric)",
uiTableId: "scores_avg",
clickhouseTableName: "scores",
clickhouseSelect: "s.scores_avg",
},
{
uiTableName: "Scores (categorical)",
uiTableId: "score_categories",
clickhouseTableName: "scores",
clickhouseSelect: "s.score_categories",
},
{
uiTableName: "Scores (boolean)",
uiTableId: "score_booleans",
clickhouseTableName: "scores",
clickhouseSelect: "s.score_booleans",
},
{
uiTableName: "Trace Scores (numeric)",
uiTableId: "trace_scores_avg",
clickhouseTableName: "scores",
clickhouseSelect: "ts.scores_avg",
},
{
uiTableName: "Trace Scores (categorical)",
uiTableId: "trace_score_categories",
clickhouseTableName: "scores",
clickhouseSelect: "ts.score_categories",
},
{
uiTableName: "Trace Scores (boolean)",
uiTableId: "trace_score_booleans",
clickhouseTableName: "scores",
clickhouseSelect: "ts.score_booleans",
},
{
uiTableName: "Comment Count",
uiTableId: "commentCount",
clickhouseTableName: "comments",
clickhouseSelect: "", // handled by comment filter helpers
},
{
uiTableName: "Comment Content",
uiTableId: "commentContent",
clickhouseTableName: "comments",
clickhouseSelect: "", // handled by comment filter helpers
},
];