Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

14.4 KB
Raw Blame
// This structure is maintained to relate the frontend table definitions with the clickhouse table definitions.
// The frontend only sends the column names to the backend. This needs to be changed in the future to send column IDs.

import { UiColumnMappings } from "../../tableDefinitions/types";
import {
  eventsTableHasParentObservationSql,
  eventsTableIsRootObservationSql,
  eventsTableHasInputSql,
  eventsTableHasOutputSql,
  eventsTableCachedInputCostSql,
  eventsTableCachedInputTokensSql,
  eventsTableTraceNameSql,
  eventsTableLoadedSkillNamesSql,
  eventsTableLoadedSkillVersionsSql,
  eventsTableAvailableSkillNamesSql,
  eventsTableAvailableSkillVersionsSql,
} from "../../eventsTable";
import { tokenCountFilterClickhouseType } from "./tokenCountFilterType";

export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [
  {
    uiTableName: "Environment",
    uiTableId: "environment",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "environment",
    queryPrefix: "e",
  },
  {
    uiTableName: "API Key",
    uiTableId: "ingestionApiKey",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."ingestion_api_key"',
  },
  {
    uiTableName: "SDK Name",
    uiTableId: "ingestionSdkName",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."ingestion_sdk_name"',
  },
  {
    uiTableName: "SDK Version",
    uiTableId: "ingestionSdkVersion",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."ingestion_sdk_version"',
  },
  {
    uiTableName: "Ingestion Source",
    uiTableId: "ingestionSource",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."source"',
  },
  {
    uiTableName: "Type",
    uiTableId: "type",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "type",
    queryPrefix: "e",
  },
  {
    uiTableName: "ID",
    uiTableId: "id",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."span_id"',
  },
  {
    uiTableName: "Name",
    uiTableId: "name",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "name",
    queryPrefix: "e",
  },
  {
    uiTableName: "Trace ID",
    uiTableId: "traceId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "trace_id",
    queryPrefix: "e",
  },

  {
    uiTableName: "Start Time",
    uiTableId: "startTime",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "start_time",
    queryPrefix: "e",
  },
  {
    uiTableName: "End Time",
    uiTableId: "endTime",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."end_time"',
  },
  {
    uiTableName: "Time To First Token (s)",
    uiTableId: "timeToFirstToken",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "if(isNull(e.completion_start_time), NULL,  date_diff('millisecond', e.start_time, e.completion_start_time) / 1000)",
    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
    clickhouseTypeOverwrite: "Decimal64(3)",
  },
  {
    uiTableName: "Latency (s)",
    uiTableId: "latency",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "if(isNull(e.end_time), NULL, date_diff('millisecond', e.start_time, e.end_time) / 1000)",
    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
    clickhouseTypeOverwrite: "Decimal64(3)",
  },
  {
    uiTableName: "Tokens per second",
    uiTableId: "tokensPerSecond",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('millisecond', start_time, end_time) / 1000))",
  },
  {
    uiTableName: "Input Cost ($)",
    uiTableId: "inputCost",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, cost_details)))",
  },
  {
    uiTableName: "Cached Input Cost ($)",
    uiTableId: "cachedInputCost",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableCachedInputCostSql,
  },
  {
    uiTableName: "Output Cost ($)",
    uiTableId: "outputCost",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, cost_details)))",
  },
  {
    uiTableName: "Total Cost ($)",
    uiTableId: "totalCost",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "if(mapExists((k, v) -> (k = 'total'), cost_details), cost_details['total'], NULL)",
  },
  {
    uiTableName: "Status",
    uiTableId: "level",
    aliases: ["Level"],
    clickhouseTableName: "events_proto",
    clickhouseSelect: "level",
    queryPrefix: "e",
  },
  {
    uiTableName: "Status Message",
    uiTableId: "statusMessage",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."status_message"',
  },
  {
    uiTableName: "Model",
    uiTableId: "model",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."provided_model_name"',
  },
  {
    uiTableName: "Provided Model Name",
    uiTableId: "providedModelName",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."provided_model_name"',
  },
  {
    uiTableName: "Model ID",
    uiTableId: "modelId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."model_id"',
  },
  {
    uiTableName: "Input Tokens",
    uiTableId: "inputTokens",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",
    clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
  },
  {
    uiTableName: "Cached Input Tokens",
    uiTableId: "cachedInputTokens",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableCachedInputTokensSql,
    clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
  },
  {
    uiTableName: "Output Tokens",
    uiTableId: "outputTokens",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))",
    clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
  },
  {
    uiTableName: "Total Tokens",
    uiTableId: "totalTokens",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
    clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
  },
  {
    uiTableName: "Tokens",
    uiTableId: "tokens",
    clickhouseTableName: "events_proto",
    clickhouseSelect:
      "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
    clickhouseTypeOverwrite: tokenCountFilterClickhouseType,
  },
  {
    uiTableName: "Metadata",
    uiTableId: "metadata",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "metadata",
    queryPrefix: "e",
  },
  {
    uiTableName: "Evaluator ID",
    uiTableId: "evaluatorId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.evaluator_id",
  },
  {
    uiTableName: "Rule ID",
    uiTableId: "ruleId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.evaluation_rule_id",
  },
  {
    uiTableName: "Version",
    uiTableId: "version",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "version",
    queryPrefix: "e",
  },
  {
    uiTableName: "Release",
    uiTableId: "release",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "release",
    queryPrefix: "e",
    emptyEqualsNull: true,
  },
  {
    uiTableName: "Prompt Name",
    uiTableId: "promptName",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.prompt_name",
  },
  {
    uiTableName: "Prompt Version",
    uiTableId: "promptVersion",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.prompt_version",
  },
  {
    uiTableName: "Input",
    uiTableId: "input",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.input",
  },
  {
    uiTableName: "Output",
    uiTableId: "output",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.output",
  },
  {
    uiTableName: "Session ID",
    uiTableId: "sessionId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "session_id",
    queryPrefix: "e",
  },
  {
    uiTableName: "Trace Name",
    uiTableId: "traceName",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableTraceNameSql,
  },
  {
    uiTableName: "User ID",
    uiTableId: "userId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "user_id",
    queryPrefix: "e",
  },
  {
    uiTableName: "Trace Tags",
    uiTableId: "traceTags",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."tags"',
  },
  {
    uiTableName: "Tags",
    uiTableId: "tags",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."tags"',
  },
  {
    uiTableName: "Trace Environment",
    uiTableId: "traceEnvironment",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "environment",
    queryPrefix: "e",
  },
  {
    uiTableName: "Has Parent Observation",
    uiTableId: "hasParentObservation",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableHasParentObservationSql,
  },
  {
    uiTableName: "Is Root Observation",
    uiTableId: "isRootObservation",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableIsRootObservationSql,
  },
  {
    uiTableName: "Has Input",
    uiTableId: "hasInput",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableHasInputSql,
  },
  {
    uiTableName: "Has Output",
    uiTableId: "hasOutput",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableHasOutputSql,
  },
  {
    uiTableName: "Parent Observation ID",
    uiTableId: "parentObservationId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "parent_span_id",
    queryPrefix: "e",
    emptyEqualsNull: true,
  },
  {
    uiTableName: "Experiment Dataset ID",
    uiTableId: "experimentDatasetId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."experiment_dataset_id"',
  },
  {
    uiTableName: "Experiment ID",
    uiTableId: "experimentId",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."experiment_id"',
    emptyEqualsNull: true,
  },
  {
    uiTableName: "Experiment Name",
    uiTableId: "experimentName",
    clickhouseTableName: "events_proto",
    clickhouseSelect: 'e."experiment_name"',
  },
  {
    uiTableName: "Is Experiment Item Root Span",
    uiTableId: "isExperimentItemRootSpan",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.experiment_item_root_span_id = e.span_id",
  },
  {
    uiTableName: "Available Tools",
    uiTableId: "toolDefinitions",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "length(mapKeys(e.tool_definitions))",
  },
  {
    uiTableName: "Tool Calls",
    uiTableId: "toolCalls",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "length(e.tool_calls)",
  },
  {
    uiTableName: "Loaded Skill Names",
    uiTableId: "skillsNamesLoaded",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableLoadedSkillNamesSql,
  },
  {
    uiTableName: "Loaded Skill Versions",
    uiTableId: "skillsVersionsLoaded",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableLoadedSkillVersionsSql,
  },
  {
    uiTableName: "Loaded Skill Files",
    uiTableId: "skillFilesLoaded",
    clickhouseTableName: "events_proto",
    // Pair each path with its immutable skill-version ID to keep matches within one loaded resource.
    clickhouseSelect:
      "arrayMap(skill -> concat(skill.langfuseSkillId, '/', skill.filePath), e.skills_resource_loaded)",
  },
  {
    uiTableName: "Available Skill Names",
    uiTableId: "skillNamesAvailable",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableAvailableSkillNamesSql,
  },
  {
    uiTableName: "Available Skill Versions",
    uiTableId: "skillVersionsAvailable",
    clickhouseTableName: "events_proto",
    clickhouseSelect: eventsTableAvailableSkillVersionsSql,
  },
  {
    uiTableName: "Skill Names",
    uiTableId: "skillNames",
    clickhouseTableName: "events_proto",
    clickhouseSelect: `arrayConcat(${eventsTableLoadedSkillNamesSql}, ${eventsTableAvailableSkillNamesSql})`,
  },
  {
    uiTableName: "Skill Versions",
    uiTableId: "skillVersions",
    clickhouseTableName: "events_proto",
    clickhouseSelect: `arrayConcat(${eventsTableLoadedSkillVersionsSql}, ${eventsTableAvailableSkillVersionsSql})`,
  },
  {
    uiTableName: "Tool Names",
    uiTableId: "toolNames",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "mapKeys(e.tool_definitions)",
  },
  {
    uiTableName: "Called Tool Names",
    uiTableId: "calledToolNames",
    clickhouseTableName: "events_proto",
    clickhouseSelect: "e.tool_call_names",
  },
];

export const eventsTableUiColumnDefinitions: UiColumnMappings = [
  ...eventsTableNativeUiColumnDefinitions,
  // Scores column duplicated to allow renaming column name. Will be removed once session storage cache is outdated
  // Column names are cached in user sessions - changing them breaks existing filters
  {
    uiTableName: "Scores",
    uiTableId: "scores",
    clickhouseTableName: "scores",
    clickhouseSelect: "s.scores_avg",
  },
  {
    uiTableName: "Scores (numeric)",
    uiTableId: "scores_avg",
    clickhouseTableName: "scores",
    clickhouseSelect: "s.scores_avg",
  },
  {
    uiTableName: "Scores (categorical)",
    uiTableId: "score_categories",
    clickhouseTableName: "scores",
    clickhouseSelect: "s.score_categories",
  },
  {
    uiTableName: "Scores (boolean)",
    uiTableId: "score_booleans",
    clickhouseTableName: "scores",
    clickhouseSelect: "s.score_booleans",
  },
  {
    uiTableName: "Trace Scores (numeric)",
    uiTableId: "trace_scores_avg",
    clickhouseTableName: "scores",
    clickhouseSelect: "ts.scores_avg",
  },
  {
    uiTableName: "Trace Scores (categorical)",
    uiTableId: "trace_score_categories",
    clickhouseTableName: "scores",
    clickhouseSelect: "ts.score_categories",
  },
  {
    uiTableName: "Trace Scores (boolean)",
    uiTableId: "trace_score_booleans",
    clickhouseTableName: "scores",
    clickhouseSelect: "ts.score_booleans",
  },
  {
    uiTableName: "Comment Count",
    uiTableId: "commentCount",
    clickhouseTableName: "comments",
    clickhouseSelect: "", // handled by comment filter helpers
  },
  {
    uiTableName: "Comment Content",
    uiTableId: "commentContent",
    clickhouseTableName: "comments",
    clickhouseSelect: "", // handled by comment filter helpers
  },
];