Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

16.9 KB
Raw Blame
import type { ColumnDefinition } from "./tableDefinitions/types";

export const eventsTableHasParentObservationSql = "e.parent_span_id != ''";
export const eventsTableIsRootObservationSqlForAlias = (alias: string) => {
  const prefix = alias ? `${alias}.` : "";
  return `(${prefix}parent_span_id = '' OR ${prefix}is_app_root = true)`;
};
export const eventsTableIsRootObservationSql =
  eventsTableIsRootObservationSqlForAlias("e");
export const eventsTableTraceNameSqlForAlias = (alias: string) =>
  `COALESCE(nullIf(${alias}.trace_name, ''), if(${eventsTableIsRootObservationSqlForAlias(alias)}, nullIf(${alias}.name, ''), NULL))`;
export const eventsTableTraceNameSql = eventsTableTraceNameSqlForAlias("e");
// Row-projection variant. The fallback above is Nullable(String), but
// events_core.trace_name is a non-null String. Projecting the nullable
// expression under the column's own name while a filter reads the physical
// column puts two `trace_name` headers of different types into one pipeline,
// and ClickHouse 25.x rejects that with AMBIGUOUS_COLUMN_NAME (code 352) as
// soon as the query also sorts (LFE-14924). Matching the stored String type
// keeps the alias — an export/stream wire name — stable. "no trace name" is
// therefore '' on the wire; JS consumers map it back to null.
export const eventsTableTraceNameSelectSqlForAlias = (alias: string) =>
  `ifNull(${eventsTableTraceNameSqlForAlias(alias)}, '')`;
export const eventsTableTraceNameSelectSql =
  eventsTableTraceNameSelectSqlForAlias("e");
/**
 * Maps the wire form of `eventsTableTraceNameSelectSql` ('' == no trace name)
 * back to the null every JS-facing surface uses - tRPC/UI, the public API, the
 * eval stream, analytics integrations, and batch export.
 *
 * Blob storage export deliberately does NOT use this: its published contract
 * types `trace_name` as a plain string
 * (https://langfuse.com/docs/api-and-data-platform/features/blob-storage-export-fields),
 * and its raw JSONL / Parquet paths never pass through JS, so normalizing only
 * the enriched path would make the three export formats disagree.
 */
export const normalizeEventsTraceName = (
  traceName: string | null | undefined,
): string | null => traceName || null;
export const eventsTableTraceNameAggregationSqlForAlias = (alias: string) =>
  `COALESCE(nullIf(argMaxIf(${alias}.trace_name, ${alias}.event_ts, ${alias}.trace_name <> ''), ''), nullIf(argMaxIf(${alias}.name, ${alias}.event_ts, (${eventsTableIsRootObservationSqlForAlias(alias)} AND ${alias}.name <> '')), ''))`;
export const eventsTableTraceNameAggregationSql =
  eventsTableTraceNameAggregationSqlForAlias("e");

export const isRootObservation = ({
  parentObservationId,
  isAppRoot,
}: {
  parentObservationId: string | null | undefined;
  isAppRoot: boolean | null | undefined;
}): boolean => !parentObservationId || isAppRoot === true;
// True when the observation carries input / output. NULL/'' both count as
// absent (NULL != '' is NULL, i.e. not true), so only real payloads match.
export const eventsTableHasInputSql = "e.input != ''";
export const eventsTableHasOutputSql = "e.output != ''";
export const eventsTableLoadedSkillNamesSql =
  "e.skills_resource_loaded.skillName";
export const eventsTableLoadedSkillVersionsSql =
  "arrayMap(skill -> concat(skill.skillName, '@', toString(skill.skillVersion)), e.skills_resource_loaded)";
export const eventsTableAvailableSkillNamesSql = "e.skills_available.skillName";
export const eventsTableAvailableSkillVersionsSql =
  "arrayMap(skill -> concat(skill.skillName, '@', toString(skill.skillVersion)), e.skills_available)";

const isCachedInputMetric = (key: string): boolean => {
  const normalizedKey = key.toLowerCase();
  return (
    normalizedKey.includes("cached") || normalizedKey.includes("cache_read")
  );
};

const findCachedInputMetric = (
  details?: Record<string, number> | null,
): number | undefined => {
  const values = Object.entries(details ?? {})
    .filter(([key]) => isCachedInputMetric(key))
    .map(([, value]) => Number(value));

  return values.length > 0
    ? values.reduce((sum, value) => sum + value, 0)
    : undefined;
};

export const getCachedInputMetric = (
  details?: Record<string, number> | null,
): number | undefined => findCachedInputMetric(details);

export const getCachedInputCost = (
  details?: Record<string, number> | null,
): number | undefined => findCachedInputMetric(details);

const cachedInputMetricSql = (detailsColumn: string): string => {
  const keyPredicate = (key: string) =>
    `(positionCaseInsensitive(${key}, 'cached') > 0 OR positionCaseInsensitive(${key}, 'cache_read') > 0)`;
  const filteredDetails = `mapFilter(x -> ${keyPredicate("x.1")}, ${detailsColumn})`;
  const sum = `arraySum(mapValues(${filteredDetails}))`;

  return `if(mapExists((k, v) -> ${keyPredicate("k")}, ${detailsColumn}), ${sum}, NULL)`;
};

export const eventsTableCachedInputTokensSql =
  cachedInputMetricSql("e.usage_details");
export const eventsTableCachedInputCostSql =
  cachedInputMetricSql("e.cost_details");

type MutableDeep<T> = T extends readonly (infer U)[]
  ? MutableDeep<U>[]
  : T extends object
    ? { -readonly [K in keyof T]: MutableDeep<T[K]> }
    : T;

// Column definitions for the ClickHouse events table
// Used for filtering, sorting, and mapping UI columns to ClickHouse columns
const eventsTableColsDefinition = [
  {
    name: "ID",
    id: "id",
    type: "stringOptions",
    internal: "e.span_id",
    options: [], // to be added at runtime
  },
  {
    name: "Trace ID",
    id: "traceId",
    type: "string",
    internal: "e.trace_id",
  },
  {
    name: "Start Time",
    id: "startTime",
    type: "datetime",
    internal: "e.start_time",
  },
  {
    name: "End Time",
    id: "endTime",
    type: "datetime",
    internal: "e.end_time",
    nullable: true,
  },
  {
    name: "Name",
    id: "name",
    type: "stringOptions",
    internal: "e.name",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Type",
    id: "type",
    type: "stringOptions",
    internal: "e.type",
    options: [],
  },
  {
    name: "Environment",
    id: "environment",
    type: "stringOptions",
    internal: "e.environment",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "API Key",
    id: "ingestionApiKey",
    type: "stringOptions",
    internal: "e.ingestion_api_key",
    options: [], // to be added at runtime
  },
  {
    name: "SDK Name",
    id: "ingestionSdkName",
    type: "stringOptions",
    internal: "e.ingestion_sdk_name",
    options: [], // to be added at runtime
  },
  {
    name: "SDK Version",
    id: "ingestionSdkVersion",
    type: "stringOptions",
    internal: "e.ingestion_sdk_version",
    options: [], // to be added at runtime
  },
  {
    name: "Ingestion Source",
    id: "ingestionSource",
    type: "stringOptions",
    internal: "e.source",
    options: [], // to be added at runtime
  },
  {
    name: "Version",
    id: "version",
    type: "string",
    internal: "e.version",
    nullable: true,
  },
  {
    name: "Release",
    id: "release",
    type: "string",
    internal: "e.release",
    nullable: true,
  },
  {
    name: "User ID",
    id: "userId",
    type: "string",
    internal: "e.user_id",
    nullable: true,
  },
  {
    name: "Session ID",
    id: "sessionId",
    type: "string",
    internal: "e.session_id",
    nullable: true,
  },
  {
    name: "Trace Name",
    id: "traceName",
    type: "stringOptions",
    internal: eventsTableTraceNameSql,
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Status",
    id: "level",
    type: "stringOptions",
    internal: "e.level",
    options: [],
    aliases: ["Level"],
  },
  {
    name: "Status Message",
    id: "statusMessage",
    type: "string",
    internal: "e.status_message",
    nullable: true,
  },
  {
    name: "Prompt Name",
    id: "promptName",
    type: "stringOptions",
    internal: "e.prompt_name",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Prompt Version",
    id: "promptVersion",
    type: "number",
    step: 1,
    internal: "e.prompt_version",
    nullable: true,
  },
  {
    name: "Model ID",
    id: "modelId",
    type: "stringOptions",
    internal: "e.model_id",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Provided Model Name",
    id: "providedModelName",
    type: "stringOptions",
    internal: "e.provided_model_name",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Cost ($)",
    id: "totalCost",
    type: "number",
    internal:
      "if(mapExists((k, v) -> (k = 'total'), cost_details), cost_details['total'], NULL)",
    nullable: true,
  },
  {
    name: "Input Tokens",
    id: "inputTokens",
    type: "number",
    step: 1,
    internal:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",
    nullable: true,
  },
  {
    name: "Cached Input Tokens",
    id: "cachedInputTokens",
    type: "number",
    step: 1,
    internal: eventsTableCachedInputTokensSql,
    nullable: true,
  },
  {
    name: "Output Tokens",
    id: "outputTokens",
    type: "number",
    step: 1,
    internal:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))",
    nullable: true,
  },
  {
    name: "Total Tokens",
    id: "totalTokens",
    type: "number",
    step: 1,
    internal:
      "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
    nullable: true,
  },
  {
    name: "Input Cost ($)",
    id: "inputCost",
    type: "number",
    internal:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, cost_details)))",
    nullable: true,
  },
  {
    name: "Cached Input Cost ($)",
    id: "cachedInputCost",
    type: "number",
    internal: eventsTableCachedInputCostSql,
    nullable: true,
  },
  {
    name: "Output Cost ($)",
    id: "outputCost",
    type: "number",
    internal:
      "arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, cost_details)))",
    nullable: true,
  },
  {
    name: "Latency (s)",
    id: "latency",
    type: "number",
    internal: "date_diff('millisecond', e.start_time, e.end_time) / 1000.0",
    nullable: true,
  },
  {
    name: "Time To First Token (s)",
    id: "timeToFirstToken",
    type: "number",
    internal:
      "date_diff('millisecond', e.start_time, e.completion_start_time) / 1000.0",
    nullable: true,
  },
  {
    name: "Tokens per second",
    id: "tokensPerSecond",
    type: "number",
    internal:
      "(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('millisecond', start_time, end_time) / 1000))",
    nullable: true,
  },
  {
    name: "Input",
    id: "input",
    type: "string",
    internal: "e.input",
    nullable: true,
  },
  {
    name: "Output",
    id: "output",
    type: "string",
    internal: "e.output",
    nullable: true,
  },
  {
    name: "Metadata",
    id: "metadata",
    type: "stringObject",
    internal: "e.metadata",
  },
  {
    name: "Evaluator ID",
    id: "evaluatorId",
    type: "stringOptions",
    internal: "e.evaluator_id",
    options: [],
  },
  {
    name: "Rule ID",
    id: "ruleId",
    type: "stringOptions",
    internal: "e.evaluation_rule_id",
    options: [],
  },
  {
    name: "Trace Tags",
    id: "traceTags",
    type: "arrayOptions",
    internal: "e.tags",
    options: [], // to be added at runtime
  },
  {
    name: "Scores (numeric)",
    id: "scores_avg",
    type: "numberObject",
    internal: "scores_avg",
  },
  {
    name: "Scores (categorical)",
    id: "score_categories",
    type: "categoryOptions",
    internal: "score_categories",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Scores (boolean)",
    id: "score_booleans",
    type: "booleanObject",
    internal: "score_booleans",
    nullable: true,
  },
  {
    name: "Trace Scores (numeric)",
    id: "trace_scores_avg",
    type: "numberObject",
    internal: "trace_scores_avg",
  },
  {
    name: "Trace Scores (categorical)",
    id: "trace_score_categories",
    type: "categoryOptions",
    internal: "trace_score_categories",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Trace Scores (boolean)",
    id: "trace_score_booleans",
    type: "booleanObject",
    internal: "trace_score_booleans",
    nullable: true,
  },
  {
    name: "Comment Count",
    id: "commentCount",
    type: "number",
    step: 1,
    internal: "", // handled by comment filter helpers
  },
  {
    name: "Comment Content",
    id: "commentContent",
    type: "string",
    internal: "", // handled by comment filter helpers
  },
  {
    name: "Has Parent Observation",
    id: "hasParentObservation",
    type: "boolean",
    internal: eventsTableHasParentObservationSql,
  },
  {
    name: "Is Root Observation",
    id: "isRootObservation",
    type: "boolean",
    internal: eventsTableIsRootObservationSql,
  },
  {
    name: "Has Input",
    id: "hasInput",
    type: "boolean",
    internal: eventsTableHasInputSql,
  },
  {
    name: "Has Output",
    id: "hasOutput",
    type: "boolean",
    internal: eventsTableHasOutputSql,
  },
  {
    name: "Experiment Dataset ID",
    id: "experimentDatasetId",
    type: "stringOptions",
    internal: "e.experiment_dataset_id",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Experiment ID",
    id: "experimentId",
    type: "stringOptions",
    internal: "e.experiment_id",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Experiment Name",
    id: "experimentName",
    type: "stringOptions",
    internal: "e.experiment_name",
    options: [], // to be added at runtime
    nullable: true,
  },
  {
    name: "Loaded Skill Names",
    id: "skillsNamesLoaded",
    type: "arrayOptions",
    internal: eventsTableLoadedSkillNamesSql,
    options: [],
  },
  {
    name: "Loaded Skill Versions",
    id: "skillsVersionsLoaded",
    type: "arrayOptions",
    internal: eventsTableLoadedSkillVersionsSql,
    options: [],
  },
  {
    name: "Available Skill Names",
    id: "skillNamesAvailable",
    type: "arrayOptions",
    internal: eventsTableAvailableSkillNamesSql,
    options: [],
  },
  {
    name: "Available Skill Versions",
    id: "skillVersionsAvailable",
    type: "arrayOptions",
    internal: eventsTableAvailableSkillVersionsSql,
    options: [],
  },
  {
    name: "Available Tool Names",
    id: "toolNames",
    type: "arrayOptions",
    internal: "mapKeys(e.tool_definitions)",
    options: [],
    nullable: true,
  },
  {
    name: "Called Tool Names",
    id: "calledToolNames",
    type: "arrayOptions",
    internal: "e.tool_call_names",
    options: [],
    nullable: true,
  },
  {
    name: "Available Tools",
    id: "toolDefinitions",
    type: "number",
    step: 1,
    internal: "length(mapKeys(e.tool_definitions))",
    nullable: true,
  },
  {
    name: "Tool Calls",
    id: "toolCalls",
    type: "number",
    step: 1,
    internal: "length(e.tool_calls)",
    nullable: true,
  },
  {
    name: "Is Experiment Item Root Span",
    id: "isExperimentItemRootSpan",
    type: "boolean",
    internal: "e.experiment_item_root_span_id = e.span_id",
  },
] as const satisfies readonly ColumnDefinition[];

// TODO: Remove MutableDeep once consumers accept readonly column definitions.
export const eventsTableCols =
  eventsTableColsDefinition as unknown as MutableDeep<
    typeof eventsTableColsDefinition
  > &
    ColumnDefinition[];

type EventsTableColumnId = (typeof eventsTableColsDefinition)[number]["id"];

export type NumericEventsTableColumnId = Extract<
  (typeof eventsTableColsDefinition)[number],
  { type: "number" }
>["id"];

export const isNumericEventsTableColumnId = (
  column: EventsTableColumnId,
): column is NumericEventsTableColumnId =>
  eventsTableColsDefinition.some(
    (col) => col.id === column && col.type === "number",
  );

// Subset of columns that are allowed to be used as filters in the MCP observations API
// listObservations has explicit fromStartTime and toStartTime arguments to bound database scans.
// A startTime condition in its filter array would replace both bounds, bypassing the 14-day limit.
const OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMN_IDS = [
  "id",
  "traceId",
  "endTime",
  "name",
  "type",
  "environment",
  "version",
  "userId",
  "sessionId",
  "traceName",
  "level",
  "statusMessage",
  "promptName",
  "promptVersion",
  "modelId",
  "providedModelName",
  "totalCost",
  "inputTokens",
  "outputTokens",
  "totalTokens",
  "inputCost",
  "outputCost",
  "latency",
  "timeToFirstToken",
  "input",
  "output",
  "metadata",
  "traceTags",
  "isRootObservation",
  "hasParentObservation",
] as const satisfies readonly EventsTableColumnId[];

export type ObservationMcpAllowedEventsTableFilterColumn =
  (typeof OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMN_IDS)[number];

export const OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMNS: ReadonlySet<EventsTableColumnId> =
  new Set(OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMN_IDS);