🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(skills): trace skill availability and resource loads (#18493)
tobi_kle1cb5300ac14
diff --git a/.env.prod.example b/.env.prod.exampleindex 46f1d9783b..763652fe4f 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -266,6 +266,8 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# API_KEY_ORG_ROLES_ENABLE=false# LANGFUSE_CACHE_PROMPT_ENABLED=# LANGFUSE_CACHE_PROMPT_TTL_SECONDS=+# LANGFUSE_CACHE_SKILL_ENABLED=+# LANGFUSE_CACHE_SKILL_TTL_SECONDS=# Clickhouse configuration# CLICKHOUSE_URL=diff --git a/packages/native/src/native_codec.rs b/packages/native/src/native_codec.rsindex c7a214604c..d65e560383 100644--- a/packages/native/src/native_codec.rs+++ b/packages/native/src/native_codec.rs@@ -54,6 +54,52 @@ impl Encode for Decimal64 {}}+#[derive(Debug)]+pub(crate) struct SkillReference {+ pub(crate) langfuse_skill_id: Option<String>,+ pub(crate) skill_version: Option<u32>,+ pub(crate) skill_name: String,+}++impl Encode for SkillReference {+ fn produces() -> DataTypeNode {+ <(Option<String>, Option<u32>, String)>::produces()+ }++ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {+ (+ &self.langfuse_skill_id,+ &self.skill_version,+ &self.skill_name,+ )+ .encode(writer)+ }+}++#[derive(Debug)]+pub(crate) struct SkillResourceLoaded {+ pub(crate) langfuse_skill_id: Option<String>,+ pub(crate) skill_version: Option<u32>,+ pub(crate) skill_name: String,+ pub(crate) file_path: String,+}++impl Encode for SkillResourceLoaded {+ fn produces() -> DataTypeNode {+ <(Option<String>, Option<u32>, String, String)>::produces()+ }++ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {+ (+ &self.langfuse_skill_id,+ &self.skill_version,+ &self.skill_name,+ &self.file_path,+ )+ .encode(writer)+ }+}+/// An owned ClickHouse Native-format block and its row count.////// Ownership is part of the transport contract: the later JS transport can retry this completediff --git a/packages/native/src/native_js.rs b/packages/native/src/native_js.rsindex 7ef453ae96..68cc495e7d 100644--- a/packages/native/src/native_js.rs+++ b/packages/native/src/native_js.rs@@ -1,7 +1,10 @@//! Direct reads of finalized JavaScript rows into Rust-owned schema fields.//! JS handles stay on the calling thread; only owned fields reach the async encoder.-use crate::native_codec::{parse_datetime_text, parse_decimal, DateTime64Micros, Decimal64};+use crate::native_codec::{+ parse_datetime_text, parse_decimal, DateTime64Micros, Decimal64, SkillReference,+ SkillResourceLoaded,+};use crate::native_schema::DefaultPolicy;use napi::bindgen_prelude::{Array, JsObjectValue, Object, Unknown};use napi::{JsNumber, JsValue};@@ -213,7 +216,7 @@ macro_rules! unsigned_js_value {)+};}-unsigned_js_value!(u8, u16, u64);+unsigned_js_value!(u8, u16, u32, u64);impl FromJsValue for DateTime64Micros {fn from_js(value: Unknown<'_>) -> Result<Self, String> {@@ -253,6 +256,54 @@ impl FromJsValue for Decimal64 {}}+fn skill_object(value: Unknown<'_>) -> Result<Object<'_>, String> {+ if value.get_type().map_err(|error| error.to_string())? != napi::ValueType::Object {+ return Err("expected a skill object".to_owned());+ }+ let object: Object<'_> = unsafe { value.cast() }.map_err(|error| error.to_string())?;+ if object.is_array().map_err(|error| error.to_string())? {+ return Err("expected a skill object, got an array".to_owned());+ }+ Ok(object)+}++impl FromJsValue for SkillReference {+ fn from_js(value: Unknown<'_>) -> Result<Self, String> {+ let object = skill_object(value)?;+ Ok(Self {+ langfuse_skill_id: Option::<String>::from_js(get_js_property(+ &object,+ "langfuseSkillId",+ )?)?,+ skill_version: Option::<u32>::from_js(get_js_property(&object, "skillVersion")?)?,+ skill_name: String::from_js(get_js_property(&object, "skillName")?)?,+ })+ }++ fn missing(_policy: DefaultPolicy, _metadata: &MetadataValues) -> Result<Self, String> {+ Err("skill object is required".to_owned())+ }+}++impl FromJsValue for SkillResourceLoaded {+ fn from_js(value: Unknown<'_>) -> Result<Self, String> {+ let object = skill_object(value)?;+ Ok(Self {+ langfuse_skill_id: Option::<String>::from_js(get_js_property(+ &object,+ "langfuseSkillId",+ )?)?,+ skill_version: Option::<u32>::from_js(get_js_property(&object, "skillVersion")?)?,+ skill_name: String::from_js(get_js_property(&object, "skillName")?)?,+ file_path: String::from_js(get_js_property(&object, "filePath")?)?,+ })+ }++ fn missing(_policy: DefaultPolicy, _metadata: &MetadataValues) -> Result<Self, String> {+ Err("skill resource object is required".to_owned())+ }+}+impl<T: FromJsValue> FromJsValue for Option<T> {fn from_js(value: Unknown<'_>) -> Result<Self, String> {if is_missing(&value)? {diff --git a/packages/native/src/native_schema.rs b/packages/native/src/native_schema.rsindex 7cc21b0cbc..39a8aa3f07 100644--- a/packages/native/src/native_schema.rs+++ b/packages/native/src/native_schema.rs@@ -7,7 +7,7 @@ use clickhouse::native::builder::BlockBuilder;use clickhouse::native::encode::Encode;use napi::bindgen_prelude::{Object, Unknown};-use crate::native_codec::{DateTime64Micros, Decimal64};+use crate::native_codec::{DateTime64Micros, Decimal64, SkillReference, SkillResourceLoaded};#[derive(Clone, Copy, Debug, Eq, PartialEq)]pub(crate) enum DefaultPolicy {@@ -251,6 +251,8 @@ clickhouse_table_schema! {tags: Vec<String> => DefaultPolicy::None,tool_calls: Vec<String> => DefaultPolicy::None,tool_call_names: Vec<String> => DefaultPolicy::None,+ skills_available: Vec<SkillReference> => DefaultPolicy::None,+ skills_resource_loaded: Vec<SkillResourceLoaded> => DefaultPolicy::None,metadata_names: Vec<String> => @metadata_names DefaultPolicy::None,metadata_values: Vec<String> => @metadata_values DefaultPolicy::None,experiment_metadata_names: Vec<String> => DefaultPolicy::None,diff --git a/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.down.sql b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.down.sqlnew file mode 100644index 0000000000..c2f787f7d0--- /dev/null+++ b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.down.sql@@ -0,0 +1,84 @@+ALTER TABLE events_core_mv {CLICKHOUSE_CLUSTER_CLAUSE} MODIFY QUERY+SELECT+ project_id,+ trace_id,+ span_id,+ parent_span_id,+ start_time,+ end_time,+ name,+ type,+ environment,+ version,+ release,+ trace_name,+ user_id,+ session_id,+ tags,+ level,+ status_message,+ completion_start_time,+ is_app_root,+ bookmarked,+ public,+ prompt_id,+ prompt_name,+ prompt_version,+ model_id,+ provided_model_name,+ model_parameters,+ provided_usage_details,+ usage_details,+ provided_cost_details,+ cost_details,+ usage_pricing_tier_id,+ usage_pricing_tier_name,+ tool_definitions,+ tool_calls,+ tool_call_names,+ leftUTF8(input, 200) as input,+ leftUTF8(output, 200) as output,+ metadata_names,+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,+ evaluator_id,+ evaluation_rule_id,+ evaluator_execution_is_test,+ experiment_id,+ experiment_name,+ experiment_metadata_names,+ experiment_metadata_values,+ experiment_description,+ experiment_dataset_id,+ experiment_item_id,+ experiment_item_version,+ experiment_item_expected_output,+ experiment_item_metadata_names,+ experiment_item_metadata_values,+ experiment_item_root_span_id,+ source,+ service_name,+ service_version,+ scope_name,+ scope_version,+ telemetry_sdk_language,+ telemetry_sdk_name,+ telemetry_sdk_version,+ blob_storage_file_path,+ event_bytes,+ created_at,+ updated_at,+ event_ts,+ is_deleted,+ ingestion_api_key,+ ingestion_sdk_name,+ ingestion_sdk_version+FROM events_full+SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core {CLICKHOUSE_CLUSTER_CLAUSE}+ DROP COLUMN IF EXISTS skills_resource_loaded,+ DROP COLUMN IF EXISTS skills_available+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};+ALTER TABLE events_full {CLICKHOUSE_CLUSTER_CLAUSE}+ DROP COLUMN IF EXISTS skills_resource_loaded,+ DROP COLUMN IF EXISTS skills_available+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};diff --git a/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sql b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sqlnew file mode 100644index 0000000000..f20dfd2f3d--- /dev/null+++ b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sql@@ -0,0 +1,104 @@+ALTER TABLE events_full {CLICKHOUSE_CLUSTER_CLAUSE}+ ADD COLUMN IF NOT EXISTS skills_available Array(Tuple(+ langfuseSkillId Nullable(String),+ skillVersion Nullable(UInt32),+ skillName String+ )) DEFAULT [],+ ADD COLUMN IF NOT EXISTS skills_resource_loaded Array(Tuple(+ langfuseSkillId Nullable(String),+ skillVersion Nullable(UInt32),+ skillName String,+ filePath String+ )) DEFAULT []+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};+ALTER TABLE events_core {CLICKHOUSE_CLUSTER_CLAUSE}+ ADD COLUMN IF NOT EXISTS skills_available Array(Tuple(+ langfuseSkillId Nullable(String),+ skillVersion Nullable(UInt32),+ skillName String+ )) DEFAULT [],+ ADD COLUMN IF NOT EXISTS skills_resource_loaded Array(Tuple(+ langfuseSkillId Nullable(String),+ skillVersion Nullable(UInt32),+ skillName String,+ filePath String+ )) DEFAULT []+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};+ALTER TABLE events_core_mv {CLICKHOUSE_CLUSTER_CLAUSE} MODIFY QUERY+SELECT+ project_id,+ trace_id,+ span_id,+ parent_span_id,+ start_time,+ end_time,+ name,+ type,+ environment,+ version,+ release,+ trace_name,+ user_id,+ session_id,+ tags,+ level,+ status_message,+ completion_start_time,+ is_app_root,+ bookmarked,+ public,+ prompt_id,+ prompt_name,+ prompt_version,+ model_id,+ provided_model_name,+ model_parameters,+ provided_usage_details,+ usage_details,+ provided_cost_details,+ cost_details,+ usage_pricing_tier_id,+ usage_pricing_tier_name,+ tool_definitions,+ tool_calls,+ tool_call_names,+ skills_available,+ skills_resource_loaded,+ leftUTF8(input, 200) as input,+ leftUTF8(output, 200) as output,+ metadata_names,+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,+ evaluator_id,+ evaluation_rule_id,+ evaluator_execution_is_test,+ experiment_id,+ experiment_name,+ experiment_metadata_names,+ experiment_metadata_values,+ experiment_description,+ experiment_dataset_id,+ experiment_item_id,+ experiment_item_version,+ experiment_item_expected_output,+ experiment_item_metadata_names,+ experiment_item_metadata_values,+ experiment_item_root_span_id,+ source,+ service_name,+ service_version,+ scope_name,+ scope_version,+ telemetry_sdk_language,+ telemetry_sdk_name,+ telemetry_sdk_version,+ blob_storage_file_path,+ event_bytes,+ created_at,+ updated_at,+ event_ts,+ is_deleted,+ ingestion_api_key,+ ingestion_sdk_name,+ ingestion_sdk_version+FROM events_full+SETTINGS enable_full_text_index = 1;diff --git a/packages/shared/src/domain/observations.ts b/packages/shared/src/domain/observations.tsindex 64536ebb8e..c120763fdd 100644--- a/packages/shared/src/domain/observations.ts+++ b/packages/shared/src/domain/observations.ts@@ -1,6 +1,10 @@import z from "zod";import { jsonSchema } from "../utils/zod";import { MetadataDomain } from "./traces";+import {+ SkillsAvailableSchema,+ SkillsResourceLoadedSchema,+} from "../features/skills/trace";export const ObservationType = {SPAN: "SPAN",@@ -98,6 +102,8 @@ export const ObservationSchema = z.object({toolDefinitions: z.record(z.string(), z.string()).nullable(),toolCalls: z.array(z.string()).nullable(),toolCallNames: z.array(z.string()).nullable(),+ skillsAvailable: SkillsAvailableSchema.optional(),+ skillsResourceLoaded: SkillsResourceLoadedSchema.optional(),});export type Observation = z.infer<typeof ObservationSchema>;diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex b2d7da2efb..62195e3aa1 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -160,6 +160,12 @@ const EnvSchema = z.object({.default(20_000),LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(3600), // 1h+ LANGFUSE_CACHE_SKILL_ENABLED: z.enum(["true", "false"]).default("true"),+ LANGFUSE_CACHE_SKILL_TTL_SECONDS: z.coerce+ .number()+ .int()+ .positive()+ .default(3600), // 1hCLICKHOUSE_URL: z.url(),CLICKHOUSE_READ_ONLY_URL: z.url().optional(),CLICKHOUSE_EVENTS_READ_ONLY_URL: z.url().optional(),diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.tsindex 1d3828f4ac..2945e0fc96 100644--- a/packages/shared/src/features/query/dataModel.ts+++ b/packages/shared/src/features/query/dataModel.ts@@ -1382,6 +1382,20 @@ export const eventsObservationsView: ViewDeclarationType = {type: "string",description: "Month of the observation start_time in YYYY-MM format.",},+ skillNamesAvailable: {+ sql: "arrayDistinct(events_observations.skills_available.skillName)",+ alias: "skillNamesAvailable",+ type: "arrayString",+ explodeArray: true,+ description: "Names of skills made available.",+ },+ skillsNamesLoaded: {+ sql: "arrayDistinct(events_observations.skills_resource_loaded.skillName)",+ alias: "skillsNamesLoaded",+ type: "arrayString",+ explodeArray: true,+ description: "Names of skills made available.",+ },toolNames: {sql: "mapKeys(events_observations.tool_definitions)",alias: "toolNames",diff --git a/packages/shared/src/features/skills/constants.ts b/packages/shared/src/features/skills/constants.tsindex e7ef695f1f..a0a1c95587 100644--- a/packages/shared/src/features/skills/constants.ts+++ b/packages/shared/src/features/skills/constants.ts@@ -3,3 +3,16 @@ export const SKILL_PRODUCTION_LABEL = "production";export const MAX_SKILL_FILES = 200;export const MAX_SKILL_PATH_LENGTH = 512;export const MAX_SKILL_BYTES = 1_000_000;++export const MAX_SKILL_CATALOG_JSON_BYTES = 64 * 1024;+export const MAX_SKILL_TOOL_ARGS_JSON_BYTES = 16 * 1024;+export const MAX_SKILL_ERROR_JSON_BYTES = 4 * 1024;++export const SKILL_RESOURCE_TOOL_NAMES = new Set([+ "skill",+ "Skill",+ "loadSkill",+ "skill_read",+ "readFile",+ "Read",+]);diff --git a/packages/shared/src/features/skills/trace.ts b/packages/shared/src/features/skills/trace.tsnew file mode 100644index 0000000000..4afe748509--- /dev/null+++ b/packages/shared/src/features/skills/trace.ts@@ -0,0 +1,28 @@+import { z } from "zod/v4";+import { SkillFilePathSchema } from "./types";++export const MAX_SKILLS_PER_OBSERVATION = 100;++export const SkillsAvailableSchema = z+ .array(+ z.object({+ langfuseSkillId: z.string().min(1).max(256).nullish(),+ skillName: z.string().min(1).max(128),+ skillVersion: z.number().int().min(1).max(2_147_483_647).nullish(),+ }),+ )+ .max(MAX_SKILLS_PER_OBSERVATION);++export const SkillsResourceLoadedSchema = z+ .array(+ z.object({+ langfuseSkillId: z.string().min(1).max(256).nullish(),+ filePath: SkillFilePathSchema,+ skillName: z.string().min(1).max(128),+ skillVersion: z.number().int().min(1).max(2_147_483_647).nullish(),+ }),+ )+ .max(MAX_SKILLS_PER_OBSERVATION);++export type SkillsAvailable = z.infer<typeof SkillsAvailableSchema>;+export type SkillsResourceLoaded = z.infer<typeof SkillsResourceLoadedSchema>;diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.tsindex 363158452d..c6627d5a93 100644--- a/packages/shared/src/index.ts+++ b/packages/shared/src/index.ts@@ -99,6 +99,7 @@ export * from "./features/prompts/constants";// skillsexport * from "./features/skills/types";+export * from "./features/skills/trace";export * from "./features/skills/constants";export {diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex 78086b12ec..5a24181239 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -22,6 +22,7 @@ export * from "./services/email/commentMention/sendCommentMentionEmail";export * from "./services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail";export * from "./services/email/posthogExportFailed/sendPostHogExportFailedEmail";export * from "./services/PromptService";+export * from "./services/SkillCacheService";export * from "./services/PromptService/types";export * from "./services/traces-ui-table-service";export * from "./services/InMemoryFilterService";diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.tsindex 325142ab59..9fe2f9b797 100644--- a/packages/shared/src/server/llm/internalTraceEvents.ts+++ b/packages/shared/src/server/llm/internalTraceEvents.ts@@ -1,3 +1,7 @@+import type {+ SkillsAvailable,+ SkillsResourceLoaded,+} from "../../features/skills/trace";import {asBoolean,asNumberRecord,@@ -66,6 +70,8 @@ export type InternalTraceEventInput = {promptId?: string;promptName?: string;promptVersion?: string;+ skillsAvailable?: SkillsAvailable;+ skillsResourceLoaded?: SkillsResourceLoaded;modelId?: string;modelName?: string;modelParameters?: string | Record<string, unknown>;diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 1a7d40da62..bad1965664 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -1,5 +1,19 @@/* eslint-disable no-nested-ternary */import { randomUUID } from "crypto";+import {+ MAX_SKILLS_PER_OBSERVATION,+ SkillsAvailableSchema,+ SkillsResourceLoadedSchema,+ type SkillsAvailable,+ type SkillsResourceLoaded,+} from "../../features/skills/trace";+import {+ MAX_SKILL_CATALOG_JSON_BYTES,+ MAX_SKILL_ERROR_JSON_BYTES,+ MAX_SKILL_PATH_LENGTH,+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ SKILL_RESOURCE_TOOL_NAMES,+} from "../../features/skills/constants";import {ForbiddenError,@@ -627,6 +641,14 @@ export class OtelIngestionProcessor {span.status?.message ??null,+ ...this.extractSkills(+ spanAttributes,+ observationType,+ this.extractName(span.name, spanAttributes),+ input,+ output,+ span.status?.code,+ ),promptName: canLinkPrompt? (spanAttributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME@@ -1352,6 +1374,14 @@ export class OtelIngestionProcessor {model: skipModelUsageAndCost? undefined: this.extractModelName(attributes),+ ...this.extractSkills(+ attributes,+ mappedObservationType,+ this.extractName(span.name, attributes),+ input,+ output,+ span.status?.code,+ ),promptName: canLinkPrompt? (attributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME] ??attributes["langfuse.prompt.name"] ??@@ -1895,6 +1925,7 @@ export class OtelIngestionProcessor {LangfuseOtelSpanAttributes.TRACE_OUTPUT,LangfuseOtelSpanAttributes.OBSERVATION_INPUT,LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT,+ LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE,// Vercel AI SDK"ai.prompt.messages","ai.prompt",@@ -3904,6 +3935,203 @@ export class OtelIngestionProcessor {}}+ private extractSkills(+ attributes: Record<string, unknown>,+ observationType: string,+ toolName: string,+ input: unknown,+ output: unknown,+ statusCode: number | undefined,+ ): {+ skillsAvailable?: SkillsAvailable;+ skillsResourceLoaded?: SkillsResourceLoaded;+ } {+ const manifest =+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE];+ // Skill inference is enabled only when an availability attribute is reported.+ if (!manifest) return {};+ if (+ observationType !== ObservationType.GENERATION &&+ (observationType !== ObservationType.TOOL ||+ !SKILL_RESOURCE_TOOL_NAMES.has(toolName) ||+ output == null ||+ statusCode === 2 ||+ parseObservationLevel(+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],+ ) === ObservationLevel.ERROR)+ ) {+ return {};+ }++ let parsedManifest = Array.isArray(manifest)+ ? manifest+ : this.parseJsonPayloadWithMaxSize(+ manifest,+ MAX_SKILL_CATALOG_JSON_BYTES,+ "skillsAvailable",+ );+ if (+ Array.isArray(parsedManifest) &&+ parsedManifest.length > MAX_SKILLS_PER_OBSERVATION+ ) {+ logger.warn("OTEL skill catalog dropped: item limit exceeded", {+ projectId: this.projectId,+ count: parsedManifest.length,+ limit: MAX_SKILLS_PER_OBSERVATION,+ });+ parsedManifest = undefined;+ }+ const available = Array.isArray(parsedManifest)+ ? SkillsAvailableSchema.safeParse(parsedManifest)+ : undefined;+ if (observationType === ObservationType.GENERATION) {+ if (available?.success) return { skillsAvailable: available.data };+ return {};+ }++ const candidate = this.extractSkillResource(+ toolName,+ input,+ output,+ available?.success ? available.data : [],+ );+ if (!candidate) return {};+ if (+ typeof candidate.filePath === "string" &&+ candidate.filePath.length > MAX_SKILL_PATH_LENGTH+ ) {+ logger.warn(+ "OTEL skill resource load dropped: path length limit exceeded",+ {+ projectId: this.projectId,+ toolName,+ length: candidate.filePath.length,+ limit: MAX_SKILL_PATH_LENGTH,+ },+ );+ return {};+ }+ const loaded = SkillsResourceLoadedSchema.element.safeParse(candidate);+ if (!loaded.success) return {};++ return {+ ...(available?.success ? { skillsAvailable: available.data } : {}),+ skillsResourceLoaded: [+ { skillName: loaded.data.skillName, filePath: loaded.data.filePath },+ ],+ };+ }++ private extractSkillResource(+ toolName: string,+ input: unknown,+ output: unknown,+ available: SkillsAvailable,+ ) {+ const args = this.parseJsonPayloadWithMaxSize(+ input,+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ "toolInput",+ );+ if (!OtelIngestionProcessor.isPlainObject(args)) return;+ const oversizedPath = [args.path, args.file_path].find(+ (path) =>+ typeof path === "string" &&+ path.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ );+ if (typeof oversizedPath === "string") {+ logger.warn(+ "OTEL skill resource load dropped: input path length limit exceeded",+ {+ projectId: this.projectId,+ toolName,+ length: oversizedPath.length,+ limit: MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ },+ );+ return;+ }++ switch (toolName) {+ case "skill":+ case "Skill":+ case "loadSkill":+ if (this.isFailedSkillActivation(output)) return;+ return {+ skillName: args.name ?? args.skill,+ filePath: "SKILL.md",+ };+ case "skill_read":+ return {+ skillName: args.skillName,+ filePath: args.path,+ };+ case "readFile":+ case "Read":+ return this.extractSkillResourceFromPath(+ args.path ?? args.file_path,+ available,+ );+ default:+ return;+ }+ }++ private extractSkillResourceFromPath(+ path: unknown,+ available: SkillsAvailable,+ ) {+ if (typeof path !== "string") return;+ const segments = path.split("/");+ const matches = segments.filter((segment) =>+ available.some((skill) => skill.skillName === segment),+ );+ if (matches.length !== 1) return;+ const skillName = matches[0];+ return {+ skillName,+ filePath: segments.slice(segments.indexOf(skillName) + 1).join("/"),+ };+ }++ private isFailedSkillActivation(output: unknown): boolean {+ const result = this.parseJsonPayloadWithMaxSize(+ output,+ MAX_SKILL_ERROR_JSON_BYTES,+ "toolOutput",+ );+ const text = typeof result === "string" ? result : output;+ return (+ (OtelIngestionProcessor.isPlainObject(result) &&+ (result.isError === true || result.error !== undefined)) ||+ (typeof text === "string" &&+ /^(?:Skill|File) ".*" not found/.test(+ text.slice(0, MAX_SKILL_ERROR_JSON_BYTES),+ ))+ );+ }++ private parseJsonPayloadWithMaxSize(+ payload: unknown,+ maxBytes: number,+ field: "skillsAvailable" | "toolInput" | "toolOutput",+ ): unknown {+ if (+ typeof payload === "string" &&+ (payload.length > maxBytes ||+ Buffer.byteLength(payload, "utf8") > maxBytes)+ ) {+ logger.warn("OTEL skill payload parsing skipped: byte limit exceeded", {+ projectId: this.projectId,+ field,+ limitBytes: maxBytes,+ stringLength: payload.length,+ });+ return undefined;+ }+ return this.parseJsonPayload(payload);+ }+/*** Parses a payload (object or JSON string) and returns the object* or `undefined` if invalid.diff --git a/packages/shared/src/server/otel/attributes.ts b/packages/shared/src/server/otel/attributes.tsindex 21358cab9e..022c52b230 100644--- a/packages/shared/src/server/otel/attributes.ts+++ b/packages/shared/src/server/otel/attributes.ts@@ -18,6 +18,7 @@ export enum LangfuseOtelSpanAttributes {OBSERVATION_STATUS_MESSAGE = "langfuse.observation.status_message",OBSERVATION_INPUT = "langfuse.observation.input",OBSERVATION_OUTPUT = "langfuse.observation.output",+ OBSERVATION_SKILLS_AVAILABLE = "langfuse.observation.skills_available",// Langfuse-observation of type Generation attributesOBSERVATION_COMPLETION_START_TIME = "langfuse.observation.completion_start_time",diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts b/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.tsindex cce72aafd2..d0d45dab18 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts@@ -216,6 +216,16 @@ const EVENTS_FILTER_OPTION_DEFINITIONS = {expression: eventsTableHasParentObservationSql,sort: "booleanAsc",},+ skillNamesAvailable: {+ kind: "array",+ expression: "arrayDistinct(e.skills_available.skillName)",+ sort: "countDesc",+ },+ skillsNamesLoaded: {+ kind: "array",+ expression: "arrayDistinct(e.skills_resource_loaded.skillName)",+ sort: "countDesc",+ },toolNames: {kind: "array",expression: "mapKeys(e.tool_definitions)",diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex 3384fbfdab..a84d6dddfb 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -7,6 +7,10 @@ import {eventsTableTraceNameAggregationSql,eventsTableTraceNameSelectSql,} from "../../../eventsTable";+import type {+ SkillsAvailable,+ SkillsResourceLoaded,+} from "../../../features/skills/trace";import { expressionBuilder, sql, type SelectExpression } from "kysely";import { renderClickhouseExpression } from "../../query-ast/compile";@@ -184,6 +188,10 @@ const EVENTS_FIELDS = {promptName: "e.prompt_name as prompt_name",promptVersion: "e.prompt_version as prompt_version",+ // Skill fields+ skillsAvailable: 'e.skills_available as "skills_available"',+ skillsResourceLoaded: 'e.skills_resource_loaded as "skills_resource_loaded"',+// Tool fieldstoolDefinitions: "e.tool_definitions as tool_definitions",toolCalls: "e.tool_calls as tool_calls",@@ -376,6 +384,8 @@ const FIELD_SETS = {"toolDefinitions","toolCalls","toolCallNames",+ "skillsAvailable",+ "skillsResourceLoaded",],byIdModel: ["providedModelName",@@ -554,6 +564,12 @@ type EventsColumnField = {const EVENTS_FIELD_EXPRESSIONS = {traceName: sql.raw<string>(eventsTableTraceNameSelectSql).as("trace_name"),metadata: eventsMetadataMap.as("metadata"),+ skillsAvailable: sql+ .ref<SkillsAvailable>("e.skills_available")+ .as("skills_available"),+ skillsResourceLoaded: sql+ .ref<SkillsResourceLoaded>("e.skills_resource_loaded")+ .as("skills_resource_loaded"),} satisfies Partial<Record<Exclude<EventsFieldKey, EventsColumnField>, EventsSelection>>;diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex 762b55b54a..07bb507d38 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -200,7 +200,7 @@ WHERE (e.project_id = {param1:String}) AND (e.span_id IN ({param2:Array(String)}]`;-exports[`golden: events IO payload reads family > getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls 1`] = `+exports[`golden: events IO payload reads family > getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls+skills 1`] = `[{"fn": "queryClickhouse",@@ -249,6 +249,8 @@ exports[`golden: events IO payload reads family > getObservationsBatchIOFromEvenarrayReverse(e.experiment_item_metadata_names),arrayReverse(arrayMap(v -> leftUTF8(v, 5000), e.experiment_item_metadata_values))) AS experiment_item_metadata,+ e.skills_available AS skills_available,+ e.skills_resource_loaded AS skills_resource_loaded,e.tool_calls AS tool_calls,e.tool_call_names AS tool_call_names,mapFromArrays(@@ -833,6 +835,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromEe.prompt_id AS prompt_id,e.prompt_name AS prompt_name,e.prompt_version AS prompt_version,+ e.skills_available AS skills_available,+ e.skills_resource_loaded AS skills_resource_loaded,e.tool_definitions AS tool_definitions,e.tool_calls AS tool_calls,e.tool_call_names AS tool_call_names,@@ -900,6 +904,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromEe.prompt_id AS prompt_id,e.prompt_name AS prompt_name,e.prompt_version AS prompt_version,+ e.skills_available AS skills_available,+ e.skills_resource_loaded AS skills_resource_loaded,e.tool_definitions AS tool_definitions,e.tool_calls AS tool_calls,e.tool_call_names AS tool_call_names,@@ -969,6 +975,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromEe.prompt_id AS prompt_id,e.prompt_name AS prompt_name,e.prompt_version AS prompt_version,+ e.skills_available AS skills_available,+ e.skills_resource_loaded AS skills_resource_loaded,e.tool_definitions AS tool_definitions,e.tool_calls AS tool_calls,e.tool_call_names AS tool_call_names,@@ -1042,6 +1050,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromEe.prompt_id AS prompt_id,e.prompt_name AS prompt_name,e.prompt_version AS prompt_version,+ e.skills_available AS skills_available,+ e.skills_resource_loaded AS skills_resource_loaded,e.tool_definitions AS tool_definitions,e.tool_calls AS tool_calls,e.tool_call_names AS tool_call_names,diff --git a/packages/shared/src/server/repositories/definitions.ts b/packages/shared/src/server/repositories/definitions.tsindex 5a696023b5..2a46916adf 100644--- a/packages/shared/src/server/repositories/definitions.ts+++ b/packages/shared/src/server/repositories/definitions.ts@@ -1,4 +1,8 @@import z from "zod";+import {+ SkillsAvailableSchema,+ SkillsResourceLoadedSchema,+} from "../../features/skills/trace";import type { IngestionAttribution } from "../ingestion/ingestionAttribution";import { DEFAULT_TRACE_ENVIRONMENT } from "../ingestion/types";import { toClickhouseDateTime } from "../clickhouse/client";@@ -118,6 +122,8 @@ export const eventsObservationRecordBaseSchema =export const eventsObservationRecordReadSchema =observationRecordReadSchema.extend({+ skills_available: SkillsAvailableSchema.optional(),+ skills_resource_loaded: SkillsResourceLoadedSchema.optional(),user_id: z.string().nullish(),session_id: z.string().nullish(),is_root_observation: z.boolean().optional(),@@ -513,6 +519,8 @@ export const eventRecordBaseSchema = z.object({tool_definitions: z.record(z.string(), z.string()).default({}),tool_calls: z.array(z.string()).default([]),tool_call_names: z.array(z.string()).default([]),+ skills_available: SkillsAvailableSchema.optional(),+ skills_resource_loaded: SkillsResourceLoadedSchema.optional(),// I/Oinput: z.string().nullish(),diff --git a/packages/shared/src/server/repositories/events.golden.test.ts b/packages/shared/src/server/repositories/events.golden.test.tsindex faf8b4cbd1..6a9e7e0a51 100644--- a/packages/shared/src/server/repositories/events.golden.test.ts+++ b/packages/shared/src/server/repositories/events.golden.test.ts@@ -399,13 +399,14 @@ describeWithClickhouse("golden: events IO payload reads family", () => {expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();});- it("getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls", async () => {+ it("getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls+skills", async () => {await getObservationsBatchIOFromEventsTable({...batchBase,truncated: false,ioCharLimit: 5000.7,includeExperimentFields: true,includeToolCallFields: true,+ includeSkillFields: true,});expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();});diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 2a7100c71f..e8de269d2e 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -175,6 +175,10 @@ import {} from "../../tableDefinitions";import { tracesTableCols } from "../../tableDefinitions/tracesTable";import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";+import type {+ SkillsAvailable,+ SkillsResourceLoaded,+} from "../../features/skills/trace";export type EventBatchIOStringOutput = {id: string;@@ -184,6 +188,11 @@ export type EventBatchIOStringOutput = {metadata: MetadataDomain;};+export type EventBatchIOSkillFields = {+ skillsAvailable: SkillsAvailable;+ skillsResourceLoaded: SkillsResourceLoaded;+};+/** Raw ClickHouse storage shape: name-less JSON strings, names in the parallel toolCallNames array. */export type EventBatchIOToolCallFields = {toolCalls: string[];@@ -198,10 +207,12 @@ export type EventBatchIOWithExperimentOutput = EventBatchIOStringOutput & {export type EventBatchIOResult<TIncludeExperiment extends boolean,TIncludeToolCalls extends boolean,+ TIncludeSkills extends boolean = false,> = (TIncludeExperiment extends true? EventBatchIOWithExperimentOutput: EventBatchIOStringOutput) &- (TIncludeToolCalls extends true ? EventBatchIOToolCallFields : object);+ (TIncludeToolCalls extends true ? EventBatchIOToolCallFields : object) &+ (TIncludeSkills extends true ? EventBatchIOSkillFields : object);const BATCH_IO_STRING_RENDERING_PROPS: RenderingProps = {// Batch I/O truncation is handled in SQL via leftUTF8 for performance.@@ -2721,6 +2732,7 @@ export const deleteEventsOlderThanDays = async (export const getObservationsBatchIOFromEventsTable = async <TIncludeExperiment extends boolean = false,TIncludeToolCalls extends boolean = false,+ TIncludeSkills extends boolean = false,>(opts: {projectId: string;observations: Array<{@@ -2741,8 +2753,11 @@ export const getObservationsBatchIOFromEventsTable = async <includeExperimentFields?: TIncludeExperiment;/** Opt-in: tool-call arrays can be large; only eval consumers need them. */includeToolCallFields?: TIncludeToolCalls;+ includeSkillFields?: TIncludeSkills;}): Promise<- Array<EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls>>+ Array<+ EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls, TIncludeSkills>+ >> => {if (opts.observations.length === 0) {return [];@@ -2805,6 +2820,14 @@ export const getObservationsBatchIOFromEventsTable = async <).as("experiment_item_metadata"),]),)+ .$if(opts.includeSkillFields === true, (qb) =>+ qb.select([+ sql.ref<SkillsAvailable>("e.skills_available").as("skills_available"),+ sql+ .ref<SkillsResourceLoaded>("e.skills_resource_loaded")+ .as("skills_resource_loaded"),+ ]),+ ).$if(opts.includeToolCallFields === true, (qb) =>qb.select(["e.tool_calls as tool_calls",@@ -2846,6 +2869,8 @@ export const getObservationsBatchIOFromEventsTable = async <input: string | null;output: string | null;metadata: Record<string, string>;+ skills_available?: SkillsAvailable;+ skills_resource_loaded?: SkillsResourceLoaded;tool_calls?: string[];tool_call_names?: string[];experiment_item_expected_output?: string | null;@@ -2864,6 +2889,12 @@ export const getObservationsBatchIOFromEventsTable = async <output: applyBatchIOStringRendering(r.output),metadata:r.metadata !== undefined ? parseMetadataCHRecordToDomain(r.metadata) : {},+ ...(opts.includeSkillFields+ ? {+ skillsAvailable: r.skills_available ?? [],+ skillsResourceLoaded: r.skills_resource_loaded ?? [],+ }+ : {}),...(opts.includeToolCallFields? {toolCalls: r.tool_calls ?? [],@@ -2879,7 +2910,9 @@ export const getObservationsBatchIOFromEventsTable = async <: {},}: {}),- })) as Array<EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls>>;+ })) as Array<+ EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls, TIncludeSkills>+ >;};/**diff --git a/packages/shared/src/server/repositories/observations_converters.ts b/packages/shared/src/server/repositories/observations_converters.tsindex e51b212c9c..8d215a7f73 100644--- a/packages/shared/src/server/repositories/observations_converters.ts+++ b/packages/shared/src/server/repositories/observations_converters.ts@@ -408,6 +408,14 @@ export function convertEventsObservation(renderingProps: RenderingProps = DEFAULT_RENDERING_PROPS,complete: boolean,): EventsObservation | PartialEventsObservation {+ const skills = {+ ...(record.skills_available !== undefined && {+ skillsAvailable: record.skills_available,+ }),+ ...(record.skills_resource_loaded !== undefined && {+ skillsResourceLoaded: record.skills_resource_loaded,+ }),+ };if (complete) {const baseObservation = convertObservationPartial(record as ObservationRecordReadType,@@ -416,6 +424,7 @@ export function convertEventsObservation();return {...baseObservation,+ ...skills,userId: record.user_id ?? null,sessionId: record.session_id ?? null,...(record.is_root_observation !== undefined && {@@ -436,6 +445,7 @@ export function convertEventsObservation();return {...baseObservation,+ ...skills,...(record.user_id !== undefined && { userId: record.user_id }),...(record.session_id !== undefined && { sessionId: record.session_id }),...(record.is_root_observation !== undefined && {diff --git a/packages/shared/src/server/services/SkillCacheService.ts b/packages/shared/src/server/services/SkillCacheService.tsnew file mode 100644index 0000000000..891ad55549--- /dev/null+++ b/packages/shared/src/server/services/SkillCacheService.ts@@ -0,0 +1,104 @@+import type { Redis, Cluster } from "ioredis";+import type { PrismaClient, Skill } from "../../db";+import { env } from "../../env";+import { logger } from "../logger";++type SkillVersion = Pick<Skill, "name" | "version">;+type SkillReference = Pick<Skill, "id" | "name" | "version">;++export class SkillCacheService {+ private readonly cacheEnabled: boolean;++ constructor(+ private readonly prisma: PrismaClient,+ private readonly redis: Redis | Cluster | null,+ ) {+ this.cacheEnabled =+ Boolean(redis) && env.LANGFUSE_CACHE_SKILL_ENABLED === "true";+ }++ async getReferences({+ projectId,+ versions,+ }: {+ projectId: string;+ versions: SkillVersion[];+ }): Promise<SkillReference[]> {+ const unique = new Map<string, SkillVersion>();+ for (const version of versions) {+ unique.set(JSON.stringify([version.name, version.version]), version);+ }+ versions = Array.from(unique.values());+ if (versions.length === 0) return [];++ let cachedIds: (string | null)[] = [];+ if (this.cacheEnabled && this.redis) {+ try {+ const keys = versions.map((version) =>+ this.getCacheKey(projectId, version),+ );+ cachedIds = await this.redis.mget(...keys);+ } catch (error) {+ logger.error("[SkillCacheService] Error reading cache", error);+ }+ }++ const missing = versions.filter((_, index) => !cachedIds[index]);+ const fetched = missing.length+ ? await this.prisma.skill.findMany({+ where: { projectId, OR: missing },+ select: { id: true, name: true, version: true },+ })+ : [];++ if (this.cacheEnabled && this.redis && fetched.length) {+ try {+ const pipeline = this.redis.pipeline();+ for (const skill of fetched) {+ pipeline.set(+ this.getCacheKey(projectId, skill),+ skill.id,+ "EX",+ env.LANGFUSE_CACHE_SKILL_TTL_SECONDS,+ );+ }+ const results = await pipeline.exec();+ const error = results?.find(([error]) => error)?.[0];+ if (error) throw error;+ } catch (error) {+ logger.error("[SkillCacheService] Error writing cache", error);+ }+ }++ const cached = versions.flatMap((version, index) => {+ const id = cachedIds[index];+ return id ? [{ ...version, id }] : [];+ });+ return cached.concat(fetched);+ }++ async invalidateCache({+ projectId,+ versions,+ }: {+ projectId: string;+ versions: SkillVersion[];+ }): Promise<void> {+ if (!this.cacheEnabled || !this.redis || versions.length === 0) return;+ try {+ await this.redis.del(+ ...versions.map((version) => this.getCacheKey(projectId, version)),+ );+ } catch (error) {+ logger.error("[SkillCacheService] Error invalidating cache", error);+ }+ }++ private getCacheKey(+ projectId: string,+ { name, version }: SkillVersion,+ ): string {+ // A project hash tag keeps batch reads and writes in one Redis Cluster slot.+ return `skill_reference:{${encodeURIComponent(projectId)}}:${encodeURIComponent(name)}:version:${version}`;+ }+}diff --git a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.tsindex 110f1e7e97..66a99c4034 100644--- a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts+++ b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts@@ -136,6 +136,16 @@ const viewFilterDefinitions: Record<"level",sourceSpec("Status", { uiTableId: "level", aliases: ["Level"] }),),+ defineField(+ "skillNamesAvailable",+ sourceSpec("Skill Names (Available)", {+ uiTableId: "skillNamesAvailable",+ }),+ ),+ defineField(+ "skillsNamesLoaded",+ sourceSpec("Skill Names (Loaded)", { uiTableId: "skillsNamesLoaded" }),+ ),defineField("toolNames",sourceSpec("Tool Names (Available)", { uiTableId: "toolNames" }),diff --git a/web/src/features/events/server/eventsRouter.ts b/web/src/features/events/server/eventsRouter.tsindex 8d6517fb0a..9bd1c092e1 100644--- a/web/src/features/events/server/eventsRouter.ts+++ b/web/src/features/events/server/eventsRouter.ts@@ -13,6 +13,8 @@ import {} from "@/src/server/api/trpc";import {type OrderByState,+ type SkillsAvailable,+ type SkillsResourceLoaded,normalizeOrderByForTable,paginationZod,singleFilterList,@@ -74,6 +76,8 @@ export type EventBatchIOOutput = {input: string | null;output: string | null;metadata: MetadataDomainClient;+ skillsAvailable?: SkillsAvailable;+ skillsResourceLoaded?: SkillsResourceLoaded;};export type GetAllEventsInput = z.infer<typeof GetAllEventsInput>;@@ -123,6 +127,7 @@ const BatchIOInput = zodSchema.object({// rendering anyway (IO_TABLE_CHAR_LIMIT).ioCharLimit: zodSchema.number().int().positive().max(10_000).optional(),includeToolCalls: zodSchema.boolean().optional(), // Defaults to false; tool-call arrays can be large+ includeSkills: zodSchema.boolean().optional(),// Opts into trace-level auth (public traces) in protectedGetEventsTraceProceduretraceId: zodSchema.string().optional(),});@@ -349,6 +354,7 @@ export const eventsRouter = createTRPCRouter({truncated: input.truncated,ioCharLimit: input.ioCharLimit,includeToolCallFields: input.includeToolCalls,+ includeSkillFields: input.includeSkills,});return batchIO.map(toDomainWithStringifiedMetadata);@@ -374,6 +380,7 @@ export const eventsRouter = createTRPCRouter({truncated: input.truncated,ioCharLimit: input.ioCharLimit,includeToolCallFields: input.includeToolCalls,+ includeSkillFields: input.includeSkills,});return batchIO.map(toDomainWithStringifiedMetadata);@@ -398,6 +405,7 @@ export const eventsRouter = createTRPCRouter({ioCharLimit: input.ioCharLimit,includeExperimentFields: true,includeToolCallFields: input.includeToolCalls,+ includeSkillFields: input.includeSkills,});return batchIO.map(toDomainWithStringifiedMetadata);diff --git a/web/src/features/events/server/eventsService.ts b/web/src/features/events/server/eventsService.tsindex 04bbc9c3b1..37ad1f7573 100644--- a/web/src/features/events/server/eventsService.ts+++ b/web/src/features/events/server/eventsService.ts@@ -161,6 +161,8 @@ const EVENT_FILTER_OPTION_COLUMNS = ["isRootObservation","toolNames","calledToolNames",+ "skillNamesAvailable",+ "skillsNamesLoaded","metadataKeys",] as const satisfies readonly EventFilterOptionColumn[];@@ -472,6 +474,8 @@ const EVENT_FILTER_VALUE_ONLY_COLUMNS = new Set<EventFilterOptionColumn>(["traceTags","toolNames","calledToolNames",+ "skillNamesAvailable",+ "skillsNamesLoaded",]);const EVENT_FILTER_OPTIONS_NON_PARTICIPATING_COLUMNS = new Set([@@ -923,6 +927,7 @@ export async function getEventMetadataValues(interface GetEventBatchIOParams<TIncludeExperiment extends boolean = false,TIncludeToolCalls extends boolean = false,+ TIncludeSkills extends boolean = false,> {projectId: string;observations: Array<{@@ -937,6 +942,7 @@ interface GetEventBatchIOParams<includeExperimentFields?: TIncludeExperiment;/** Opt-in: tool-call arrays can be large; only eval consumers need them. */includeToolCallFields?: TIncludeToolCalls;+ includeSkillFields?: TIncludeSkills;}/**@@ -945,9 +951,18 @@ interface GetEventBatchIOParams<export async function getEventBatchIO<TIncludeExperiment extends boolean = false,TIncludeToolCalls extends boolean = false,+ TIncludeSkills extends boolean = false,>(- params: GetEventBatchIOParams<TIncludeExperiment, TIncludeToolCalls>,-): Promise<Array<EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls>>> {+ params: GetEventBatchIOParams<+ TIncludeExperiment,+ TIncludeToolCalls,+ TIncludeSkills+ >,+): Promise<+ Array<+ EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls, TIncludeSkills>+ >+> {return getObservationsBatchIOFromEventsTable({projectId: params.projectId,observations: params.observations,@@ -958,5 +973,6 @@ export async function getEventBatchIO<sessionId: params.sessionId,includeExperimentFields: params.includeExperimentFields,includeToolCallFields: params.includeToolCallFields,+ includeSkillFields: params.includeSkillFields,});}diff --git a/web/src/features/metrics/components/MetricsFilterBuilder.tsx b/web/src/features/metrics/components/MetricsFilterBuilder.tsxindex 1455d59324..07dfeb0ecf 100644--- a/web/src/features/metrics/components/MetricsFilterBuilder.tsx+++ b/web/src/features/metrics/components/MetricsFilterBuilder.tsx@@ -74,6 +74,8 @@ const eventsFilterOptionsColumns = ["experimentName","isRootObservation","calledToolNames",+ "skillNamesAvailable",+ "skillsNamesLoaded","metadataKeys","scores_avg","score_categories",@@ -388,6 +390,8 @@ const buildV2FilterColumnsParams = ({modelOptions: filterOptions?.providedModelName ?? [],toolNamesOptions: slowFilterOptions?.toolNames ?? [],calledToolNamesOptions: filterOptions?.calledToolNames ?? [],+ skillNamesAvailableOptions: filterOptions?.skillNamesAvailable ?? [],+ skillsNamesLoadedOptions: filterOptions?.skillsNamesLoaded ?? [],observationLevelOptions,experimentNameOptions: filterOptions?.experimentName ?? [],experimentDatasetOptions:diff --git a/web/src/features/metrics/metricsFilterColumns.ts b/web/src/features/metrics/metricsFilterColumns.tsindex b9a6d4ad0e..9b4fb23cc4 100644--- a/web/src/features/metrics/metricsFilterColumns.ts+++ b/web/src/features/metrics/metricsFilterColumns.ts@@ -17,6 +17,8 @@ export type GetMetricsFilterColumnsParams = {modelOptions: SingleValueOption[];toolNamesOptions: SingleValueOption[];calledToolNamesOptions: SingleValueOption[];+ skillNamesAvailableOptions?: SingleValueOption[];+ skillsNamesLoadedOptions?: SingleValueOption[];observationLevelOptions: SingleValueOption[];experimentNameOptions: SingleValueOption[];experimentDatasetOptions: SingleValueOption[];@@ -46,6 +48,8 @@ const getMetricsFilterColumnSpecs = ({modelOptions,toolNamesOptions,calledToolNamesOptions,+ skillNamesAvailableOptions = [],+ skillsNamesLoadedOptions = [],observationLevelOptions,experimentNameOptions,experimentDatasetOptions,@@ -232,9 +236,29 @@ const getMetricsFilterColumnSpecs = ({if (selectedView === "observations") {filterColumns.push(- // v2-only filter columns (experiment data only exists in events table)+ // These filters require the v2 events view....(viewVersion === "v2"? [+ {+ column: {+ name: "Skill Names (Available)",+ id: "skillNamesAvailable",+ type: "arrayOptions",+ options: skillNamesAvailableOptions,+ internal: "internalValue",+ },+ customSelect: true,+ } satisfies MetricsFilterColumnSpec,+ {+ column: {+ name: "Skill Names (Loaded)",+ id: "skillsNamesLoaded",+ type: "arrayOptions",+ options: skillsNamesLoadedOptions,+ internal: "internalValue",+ },+ customSelect: true,+ } satisfies MetricsFilterColumnSpec,{column: {name: "Experiment Name",diff --git a/web/src/features/public-api/types/observations.ts b/web/src/features/public-api/types/observations.tsindex 337bbd65a5..2e1ea6f2a6 100644--- a/web/src/features/public-api/types/observations.ts+++ b/web/src/features/public-api/types/observations.ts@@ -168,6 +168,10 @@ export const transformDbToApiObservation = (toolCallNames,+ // Exclude skill data from the legacy public API.+ skillsAvailable,+ skillsResourceLoaded,+// Exclude publish/bookmark flags from V1 public observations API.// V2 observations already exposes these on the events-based contract.bookmarked,diff --git a/web/src/features/skills/server/skill-service.ts b/web/src/features/skills/server/skill-service.tsindex f5bf1f80a2..164f946a24 100644--- a/web/src/features/skills/server/skill-service.ts+++ b/web/src/features/skills/server/skill-service.ts@@ -19,6 +19,7 @@ import {type SkillSelector,} from "@langfuse/shared";import type { Prisma, PrismaClient, SkillBlob } from "@langfuse/shared/src/db";+import { SkillCacheService, redis } from "@langfuse/shared/src/server";import type { ProjectAuthedContext } from "@/src/server/api/trpc";import type { AuthorizationContext } from "@/src/features/auth/policy/types";import { auditLog } from "@/src/features/audit-logs/server";@@ -123,7 +124,11 @@ function serializeVersion(skill: SkillWithRelations) {}export class SkillService {- constructor(private readonly prisma: PrismaClient) {}+ private readonly skillCacheService: SkillCacheService;++ constructor(private readonly prisma: PrismaClient) {+ this.skillCacheService = new SkillCacheService(prisma, redis);+ }async skillVersions(params: {projectId: string;@@ -590,7 +595,7 @@ export class SkillService {actor: SkillActor;}): Promise<void> {const where = { projectId: params.projectId, name: params.name };- await this.prisma.$transaction(async (tx) => {+ const versions = await this.prisma.$transaction(async (tx) => {await this.lockSkill(tx, params.projectId, params.name);const versions = await tx.skill.findMany({where,@@ -621,6 +626,11 @@ export class SkillService {tx,);await tx.skill.deleteMany({ where });+ return versions.map(({ version }) => ({ name: params.name, version }));+ });+ await this.skillCacheService.invalidateCache({+ projectId: params.projectId,+ versions,});}@@ -691,6 +701,10 @@ export class SkillService {}}});+ await this.skillCacheService.invalidateCache({+ projectId: params.projectId,+ versions: [{ name: params.name, version: params.version }],+ });}private async prepareVersionFiles(params: {diff --git a/web/src/features/traces/components/DetailAttributesTab.clienttest.tsx b/web/src/features/traces/components/DetailAttributesTab.clienttest.tsxindex 285e385eae..0312a945ae 100644--- a/web/src/features/traces/components/DetailAttributesTab.clienttest.tsx+++ b/web/src/features/traces/components/DetailAttributesTab.clienttest.tsx@@ -9,6 +9,9 @@ import { MediaEnabledFields } from "@/src/features/media/validation";vi.mock("@/src/components/ui/PrettyJsonView", () => ({PrettyJsonView: vi.fn(() => null),}));+vi.mock("@/src/features/feature-flags", () => ({+ useInternalFeaturesEnabled: () => false,+}));vi.mock("./ObservationDetailView/AttributeRowActions", () => ({AttributeRowActions: () => null,}));diff --git a/web/src/features/traces/components/DetailAttributesTab.tsx b/web/src/features/traces/components/DetailAttributesTab.tsxindex ab3c4995db..e1f01be6a8 100644--- a/web/src/features/traces/components/DetailAttributesTab.tsx+++ b/web/src/features/traces/components/DetailAttributesTab.tsx@@ -1,9 +1,14 @@/** Trace metadata, or observation attributes, model parameters and metadata. */import { useMemo } from "react";+import {+ type SkillsAvailable,+ type SkillsResourceLoaded,+} from "@langfuse/shared";import { type MediaReturnType } from "@/src/features/media";import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";+import { useInternalFeaturesEnabled } from "@/src/features/feature-flags";import { cn } from "@/src/utils/tailwind";import { IO_SECTIONS_FLUSH_CLASS } from "@/src/features/traces/constants/ioSectionClasses";import { type MetadataFilterActions } from "@/src/components/table/ValueCell";@@ -31,6 +36,8 @@ export function DetailAttributesTab({attributes: Record<string, unknown>;attributesAnchorTime: Date;modelParameters: Record<string, unknown> | null;+ skillsAvailable?: SkillsAvailable;+ skillsResourceLoaded?: SkillsResourceLoaded;};metadata: unknown;/** Avoids re-parsing. */@@ -48,6 +55,12 @@ export function DetailAttributesTab({projectId,filterTarget: observation ? "observations" : "traces",};+ const internalFeaturesEnabled = useInternalFeaturesEnabled();+ const skillsAvailable = observation?.skillsAvailable ?? [];+ const skillsResourceLoaded = observation?.skillsResourceLoaded ?? [];+ const hasSkills =+ internalFeaturesEnabled &&+ (skillsAvailable.length > 0 || skillsResourceLoaded.length > 0);const hasAttributes =observation && Object.keys(observation.attributes).length > 0;const hasMetadata =@@ -120,7 +133,24 @@ export function DetailAttributesTab({)}</div>) : null}- {!hasAttributes && !observation?.modelParameters && !hasMetadata ? (+ {hasSkills ? (+ <>+ <PrettyJsonView+ title="Available skills"+ json={skillsAvailable}+ currentView={currentView}+ />+ <PrettyJsonView+ title="Loaded resources"+ json={skillsResourceLoaded}+ currentView={currentView}+ />+ </>+ ) : null}+ {!hasSkills &&+ !hasAttributes &&+ !observation?.modelParameters &&+ !hasMetadata ? (<p className="text-muted-foreground text-base">{observation? "No attributes, model parameters or metadata on this observation."diff --git a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsxindex 91347f8e59..483b8243b3 100644--- a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx+++ b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx@@ -553,6 +553,8 @@ export function ConnectedObservationDetailView({attributes,attributesAnchorTime: observation.startTime,modelParameters,+ skillsAvailable: observationWithIO?.skillsAvailable,+ skillsResourceLoaded: observationWithIO?.skillsResourceLoaded,}}metadata={observationWithIOCompat.data?.metadata ?? undefined}parsedMetadata={parsedMetadata}diff --git a/web/src/features/traces/hooks/useParsedObservation.ts b/web/src/features/traces/hooks/useParsedObservation.tsindex e7fcf2d74c..6f55a8b2c3 100644--- a/web/src/features/traces/hooks/useParsedObservation.ts+++ b/web/src/features/traces/hooks/useParsedObservation.ts@@ -75,12 +75,13 @@ export function useParsedObservation({minStartTime: startTime ?? new Date(0),maxStartTime: startTime ?? new Date(),truncated: false,+ includeSkills: true,},{...sendAsPostOption,enabled: isV4,staleTime: 5 * 60 * 1000, // 5 minutes- select: (data) => data[0], // Extract single result from batch+ select: (data): EventBatchIOOutput | undefined => data[0],},);@@ -91,6 +92,8 @@ export function useParsedObservation({...baseObservation,input: eventsQuery.data.input,output: eventsQuery.data.output,+ skillsAvailable: eventsQuery.data.skillsAvailable,+ skillsResourceLoaded: eventsQuery.data.skillsResourceLoaded,// Stringify metadata to match ObservationReturnTypeWithMetadata formatmetadata: stringifyMetadata(eventsQuery.data.metadata),} satisfies ObservationWithStringifiedIO;diff --git a/worker/src/__tests__/nativeCodec.integration.test.ts b/worker/src/__tests__/nativeCodec.integration.test.tsindex 98a6f3f396..a37787d8ab 100644--- a/worker/src/__tests__/nativeCodec.integration.test.ts+++ b/worker/src/__tests__/nativeCodec.integration.test.ts@@ -30,10 +30,6 @@ describe.runIf(});columns = await result.json<Column>();expect(columns.length).toBeGreaterThan(0);- const normalizeType = (type: string) =>- type- .replaceAll(/LowCardinality\(([^()]*)\)/g, "$1")- .replaceAll(/\s/g, "");// Names and types come from the same Rust declaration as JS reads and column writes.// Check the full insertable set, including columns that captured traffic omits.const liveColumns = columns.filter(@@ -325,6 +321,31 @@ function preparedRowArbitrary(columns: Column[]): Arbitrary<Row> {});}+// Native tuple values are positional; compare element types without their server-side names.+function normalizeType(type: string): string {+ const tokens = (+ type.match(+ /'(?:\\.|''|[^'\\])*'|`(?:\\.|``|[^`\\])*`|"(?:\\.|""|[^"\\])*"|[A-Za-z_]\w*|\s+|./gs,+ ) ?? []+ ).filter((token) => !/^\s+$/.test(token));+ const parents: string[] = [];+ return tokens+ .filter((token, index) => {+ const previous = tokens[index - 1];+ const tupleElement =+ parents.at(-1) === "Tuple" && (previous === "(" || previous === ",");+ if (token === "(") parents.push(previous);+ else if (token === ")") parents.pop();+ return !(+ tupleElement &&+ /^(?:[A-Za-z_]\w*|`.*`|".*")$/s.test(token) &&+ /^[A-Za-z_]\w*$/.test(tokens[index + 1] ?? "")+ );+ })+ .join("")+ .replaceAll(/LowCardinality\(([^()]*)\)/g, "$1");+}+function columnValueArbitrary(type: string): Arbitrary<unknown> {const normalized = type.replaceAll(/LowCardinality\(([^()]*)\)/g, "$1")@@ -333,6 +354,19 @@ function columnValueArbitrary(type: string): Arbitrary<unknown> {fc.fullUnicodeString({ maxLength: 24 }),fc.constantFrom("", "true"),);+ if (normalized.startsWith("Array(Tuple(")) {+ return fc.array(+ fc.record({+ langfuseSkillId: fc.option(text, { nil: null }),+ skillVersion: fc.option(fc.integer({ min: 1, max: 4_294_967_295 }), {+ nil: null,+ }),+ skillName: text,+ ...(normalized.includes("filePathString") ? { filePath: text } : {}),+ }),+ { maxLength: 4 },+ );+ }const nullable = normalized.match(/^Nullable\((.*)\)$/);if (nullable)return fc.option(columnValueArbitrary(nullable[1]), { nil: null });diff --git a/worker/src/features/database-read-stream/observation-stream.ts b/worker/src/features/database-read-stream/observation-stream.tsindex 41f02247ca..c838e0e8da 100644--- a/worker/src/features/database-read-stream/observation-stream.ts+++ b/worker/src/features/database-read-stream/observation-stream.ts@@ -657,6 +657,8 @@ const getObservationStreamFromEvents = async (release: _release,bookmarked: _bookmarked,public: _public,+ skillsAvailable: _skillsAvailable,+ skillsResourceLoaded: _skillsResourceLoaded,userId,traceName,...observationFieldsdiff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 7fc90692df..0895572483 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -6,11 +6,14 @@ import {InvalidRequestError,LangfuseNotFoundError,Model,+ MAX_SKILLS_PER_OBSERVATION,ObservationLevel,PrismaClient,Prompt,safeJsonParse,type JsonNested,+ type SkillsAvailable,+ type SkillsResourceLoaded,} from "@langfuse/shared";import {ClickhouseClientType,@@ -31,6 +34,7 @@ import {ObservationRecordInsertType,observationRecordReadSchema,PromptService,+ SkillCacheService,QueueJobs,recordIncrement,ScoreEventType,@@ -225,6 +229,7 @@ const immutableEntityKeys: {export class IngestionService {private promptService: PromptService;+ private skillCacheService: SkillCacheService;constructor(private redis: Redis | Cluster,@@ -233,6 +238,7 @@ export class IngestionService {private clickhouseClient: ClickhouseClientType,) {this.promptService = new PromptService(prisma, redis);+ this.skillCacheService = new SkillCacheService(prisma, redis);}public async mergeAndWrite(params: MergeAndWriteParams): Promise<void> {@@ -335,8 +341,8 @@ export class IngestionService {Object.keys(eventData.providedUsageDetails ?? {}).length > 0 ||Object.keys(eventData.providedCostDetails ?? {}).length > 0;- // Perform lookups for prompt and model/usage enrichment- const [prompt, generationUsage] = await Promise.all([+ // Enrich prompt, usage and skill references concurrently.+ const [prompt, generationUsage, skillReferences] = await Promise.all([// Lookup prompt by name and versioneventData.promptName && eventData.promptVersion? this.promptService.getPrompt({@@ -372,6 +378,12 @@ export class IngestionService {},}): null,+ this.resolveSkillReferences({+ projectId: eventData.projectId,+ type: eventData.type ?? "SPAN",+ available: eventData.skillsAvailable,+ loaded: eventData.skillsResourceLoaded,+ }),]);const now = toClickhouseDateTime();@@ -424,6 +436,8 @@ export class IngestionService {prompt_id: prompt?.id || "",prompt_name: eventData.promptName,prompt_version: parseUInt16(eventData.promptVersion),+ skills_available: skillReferences.skills_available,+ skills_resource_loaded: skillReferences.skills_resource_loaded,// Modelmodel_id: generationUsage?.internal_model_id || "",@@ -1235,6 +1249,77 @@ export class IngestionService {return parsedObservationRecord;}+ private async resolveSkillReferences({+ projectId,+ type,+ available = [],+ loaded = [],+ }: {+ projectId: string;+ type: EventRecordInsertType["type"];+ available?: SkillsAvailable;+ loaded?: SkillsResourceLoaded;+ }): Promise<{+ skills_available: SkillsAvailable;+ skills_resource_loaded: SkillsResourceLoaded;+ }> {+ // Bound internal callers too, before building the lookup or its cache key.+ if (+ available.length > MAX_SKILLS_PER_OBSERVATION ||+ loaded.length > MAX_SKILLS_PER_OBSERVATION+ ) {+ logger.warn("Skill references truncated: item limit exceeded", {+ projectId,+ availableCount: available.length,+ loadedCount: loaded.length,+ limit: MAX_SKILLS_PER_OBSERVATION,+ });+ }+ available = available.slice(0, MAX_SKILLS_PER_OBSERVATION);+ loaded = loaded.slice(0, MAX_SKILLS_PER_OBSERVATION);+ const versions = available.flatMap(({ skillName, skillVersion }) =>+ skillVersion == null ? [] : [{ name: skillName, version: skillVersion }],+ );+ const skills = await this.skillCacheService.getReferences({+ projectId,+ versions,+ });++ const availableSkills = available.map(({ skillName, skillVersion }) => {+ const skill = skills.find(+ (skill) => skill.name === skillName && skill.version === skillVersion,+ );+ return {+ skillName,+ skillVersion: skill?.version ?? skillVersion,+ langfuseSkillId: skill?.id,+ };+ });++ const loadedResources = loaded.map(+ ({ skillName, filePath, skillVersion }) => {+ const candidates = availableSkills.filter(+ (skill) => skill.skillName === skillName,+ );+ if (candidates.length !== 1)+ return { skillName, filePath, skillVersion };++ const skill = candidates[0];+ return {+ skillName,+ filePath,+ langfuseSkillId: skill?.langfuseSkillId,+ skillVersion: skill.skillVersion ?? skillVersion,+ };+ },+ );++ return {+ skills_available: type === "GENERATION" ? availableSkills : [],+ skills_resource_loaded: loadedResources,+ };+ }+private mergeRecords<T extends InsertRecord>(records: T[],immutableEntityKeys: string[],diff --git a/worker/src/services/IngestionService/prepareNativeEvent.ts b/worker/src/services/IngestionService/prepareNativeEvent.tsindex c4c9ef9a20..54b018cc79 100644--- a/worker/src/services/IngestionService/prepareNativeEvent.ts+++ b/worker/src/services/IngestionService/prepareNativeEvent.ts@@ -3,7 +3,8 @@ import type { EventRecordInsertType } from "@langfuse/shared/src/server";/*** Snapshot a row after overflow handling and byte accounting into Rust. Serialize model parameters- * for their ClickHouse String column; keep the caller's structured row available for evaluations.+ * for their ClickHouse String column.+ * Keep the caller's structured row available for evaluations.*/export function prepareNativeEvent(record: Omit<EventRecordInsertType, "model_parameters"> & {