Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(skills): trace skill availability and resource loads (#18493)

tobi_kle1cb5300ac14
diff --git a/.env.prod.example b/.env.prod.example
index 46f1d9783b..763652fe4f 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -266,6 +266,8 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# API_KEY_ORG_ROLES_ENABLE=false
# LANGFUSE_CACHE_PROMPT_ENABLED=
# LANGFUSE_CACHE_PROMPT_TTL_SECONDS=
+# LANGFUSE_CACHE_SKILL_ENABLED=
+# LANGFUSE_CACHE_SKILL_TTL_SECONDS=
# Clickhouse configuration
# CLICKHOUSE_URL=
diff --git a/packages/native/src/native_codec.rs b/packages/native/src/native_codec.rs
index c7a214604c..d65e560383 100644
--- a/packages/native/src/native_codec.rs
+++ b/packages/native/src/native_codec.rs
@@ -54,6 +54,52 @@ impl Encode for Decimal64 {
}
}
+#[derive(Debug)]
+pub(crate) struct SkillReference {
+ pub(crate) langfuse_skill_id: Option<String>,
+ pub(crate) skill_version: Option<u32>,
+ pub(crate) skill_name: String,
+}
+
+impl Encode for SkillReference {
+ fn produces() -> DataTypeNode {
+ <(Option<String>, Option<u32>, String)>::produces()
+ }
+
+ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
+ (
+ &self.langfuse_skill_id,
+ &self.skill_version,
+ &self.skill_name,
+ )
+ .encode(writer)
+ }
+}
+
+#[derive(Debug)]
+pub(crate) struct SkillResourceLoaded {
+ pub(crate) langfuse_skill_id: Option<String>,
+ pub(crate) skill_version: Option<u32>,
+ pub(crate) skill_name: String,
+ pub(crate) file_path: String,
+}
+
+impl Encode for SkillResourceLoaded {
+ fn produces() -> DataTypeNode {
+ <(Option<String>, Option<u32>, String, String)>::produces()
+ }
+
+ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
+ (
+ &self.langfuse_skill_id,
+ &self.skill_version,
+ &self.skill_name,
+ &self.file_path,
+ )
+ .encode(writer)
+ }
+}
+
/// An owned ClickHouse Native-format block and its row count.
///
/// Ownership is part of the transport contract: the later JS transport can retry this complete
diff --git a/packages/native/src/native_js.rs b/packages/native/src/native_js.rs
index 7ef453ae96..68cc495e7d 100644
--- a/packages/native/src/native_js.rs
+++ b/packages/native/src/native_js.rs
@@ -1,7 +1,10 @@
//! Direct reads of finalized JavaScript rows into Rust-owned schema fields.
//! JS handles stay on the calling thread; only owned fields reach the async encoder.
-use crate::native_codec::{parse_datetime_text, parse_decimal, DateTime64Micros, Decimal64};
+use crate::native_codec::{
+ parse_datetime_text, parse_decimal, DateTime64Micros, Decimal64, SkillReference,
+ SkillResourceLoaded,
+};
use crate::native_schema::DefaultPolicy;
use napi::bindgen_prelude::{Array, JsObjectValue, Object, Unknown};
use napi::{JsNumber, JsValue};
@@ -213,7 +216,7 @@ macro_rules! unsigned_js_value {
)+
};
}
-unsigned_js_value!(u8, u16, u64);
+unsigned_js_value!(u8, u16, u32, u64);
impl FromJsValue for DateTime64Micros {
fn from_js(value: Unknown<'_>) -> Result<Self, String> {
@@ -253,6 +256,54 @@ impl FromJsValue for Decimal64 {
}
}
+fn skill_object(value: Unknown<'_>) -> Result<Object<'_>, String> {
+ if value.get_type().map_err(|error| error.to_string())? != napi::ValueType::Object {
+ return Err("expected a skill object".to_owned());
+ }
+ let object: Object<'_> = unsafe { value.cast() }.map_err(|error| error.to_string())?;
+ if object.is_array().map_err(|error| error.to_string())? {
+ return Err("expected a skill object, got an array".to_owned());
+ }
+ Ok(object)
+}
+
+impl FromJsValue for SkillReference {
+ fn from_js(value: Unknown<'_>) -> Result<Self, String> {
+ let object = skill_object(value)?;
+ Ok(Self {
+ langfuse_skill_id: Option::<String>::from_js(get_js_property(
+ &object,
+ "langfuseSkillId",
+ )?)?,
+ skill_version: Option::<u32>::from_js(get_js_property(&object, "skillVersion")?)?,
+ skill_name: String::from_js(get_js_property(&object, "skillName")?)?,
+ })
+ }
+
+ fn missing(_policy: DefaultPolicy, _metadata: &MetadataValues) -> Result<Self, String> {
+ Err("skill object is required".to_owned())
+ }
+}
+
+impl FromJsValue for SkillResourceLoaded {
+ fn from_js(value: Unknown<'_>) -> Result<Self, String> {
+ let object = skill_object(value)?;
+ Ok(Self {
+ langfuse_skill_id: Option::<String>::from_js(get_js_property(
+ &object,
+ "langfuseSkillId",
+ )?)?,
+ skill_version: Option::<u32>::from_js(get_js_property(&object, "skillVersion")?)?,
+ skill_name: String::from_js(get_js_property(&object, "skillName")?)?,
+ file_path: String::from_js(get_js_property(&object, "filePath")?)?,
+ })
+ }
+
+ fn missing(_policy: DefaultPolicy, _metadata: &MetadataValues) -> Result<Self, String> {
+ Err("skill resource object is required".to_owned())
+ }
+}
+
impl<T: FromJsValue> FromJsValue for Option<T> {
fn from_js(value: Unknown<'_>) -> Result<Self, String> {
if is_missing(&value)? {
diff --git a/packages/native/src/native_schema.rs b/packages/native/src/native_schema.rs
index 7cc21b0cbc..39a8aa3f07 100644
--- a/packages/native/src/native_schema.rs
+++ b/packages/native/src/native_schema.rs
@@ -7,7 +7,7 @@ use clickhouse::native::builder::BlockBuilder;
use clickhouse::native::encode::Encode;
use napi::bindgen_prelude::{Object, Unknown};
-use crate::native_codec::{DateTime64Micros, Decimal64};
+use crate::native_codec::{DateTime64Micros, Decimal64, SkillReference, SkillResourceLoaded};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum DefaultPolicy {
@@ -251,6 +251,8 @@ clickhouse_table_schema! {
tags: Vec<String> => DefaultPolicy::None,
tool_calls: Vec<String> => DefaultPolicy::None,
tool_call_names: Vec<String> => DefaultPolicy::None,
+ skills_available: Vec<SkillReference> => DefaultPolicy::None,
+ skills_resource_loaded: Vec<SkillResourceLoaded> => DefaultPolicy::None,
metadata_names: Vec<String> => @metadata_names DefaultPolicy::None,
metadata_values: Vec<String> => @metadata_values DefaultPolicy::None,
experiment_metadata_names: Vec<String> => DefaultPolicy::None,
diff --git a/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.down.sql b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.down.sql
new file mode 100644
index 0000000000..c2f787f7d0
--- /dev/null
+++ b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.down.sql
@@ -0,0 +1,84 @@
+ALTER TABLE events_core_mv {CLICKHOUSE_CLUSTER_CLAUSE} MODIFY QUERY
+SELECT
+ project_id,
+ trace_id,
+ span_id,
+ parent_span_id,
+ start_time,
+ end_time,
+ name,
+ type,
+ environment,
+ version,
+ release,
+ trace_name,
+ user_id,
+ session_id,
+ tags,
+ level,
+ status_message,
+ completion_start_time,
+ is_app_root,
+ bookmarked,
+ public,
+ prompt_id,
+ prompt_name,
+ prompt_version,
+ model_id,
+ provided_model_name,
+ model_parameters,
+ provided_usage_details,
+ usage_details,
+ provided_cost_details,
+ cost_details,
+ usage_pricing_tier_id,
+ usage_pricing_tier_name,
+ tool_definitions,
+ tool_calls,
+ tool_call_names,
+ leftUTF8(input, 200) as input,
+ leftUTF8(output, 200) as output,
+ metadata_names,
+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,
+ evaluator_id,
+ evaluation_rule_id,
+ evaluator_execution_is_test,
+ experiment_id,
+ experiment_name,
+ experiment_metadata_names,
+ experiment_metadata_values,
+ experiment_description,
+ experiment_dataset_id,
+ experiment_item_id,
+ experiment_item_version,
+ experiment_item_expected_output,
+ experiment_item_metadata_names,
+ experiment_item_metadata_values,
+ experiment_item_root_span_id,
+ source,
+ service_name,
+ service_version,
+ scope_name,
+ scope_version,
+ telemetry_sdk_language,
+ telemetry_sdk_name,
+ telemetry_sdk_version,
+ blob_storage_file_path,
+ event_bytes,
+ created_at,
+ updated_at,
+ event_ts,
+ is_deleted,
+ ingestion_api_key,
+ ingestion_sdk_name,
+ ingestion_sdk_version
+FROM events_full
+SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core {CLICKHOUSE_CLUSTER_CLAUSE}
+ DROP COLUMN IF EXISTS skills_resource_loaded,
+ DROP COLUMN IF EXISTS skills_available
+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};
+ALTER TABLE events_full {CLICKHOUSE_CLUSTER_CLAUSE}
+ DROP COLUMN IF EXISTS skills_resource_loaded,
+ DROP COLUMN IF EXISTS skills_available
+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};
diff --git a/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sql b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sql
new file mode 100644
index 0000000000..f20dfd2f3d
--- /dev/null
+++ b/packages/shared/clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sql
@@ -0,0 +1,104 @@
+ALTER TABLE events_full {CLICKHOUSE_CLUSTER_CLAUSE}
+ ADD COLUMN IF NOT EXISTS skills_available Array(Tuple(
+ langfuseSkillId Nullable(String),
+ skillVersion Nullable(UInt32),
+ skillName String
+ )) DEFAULT [],
+ ADD COLUMN IF NOT EXISTS skills_resource_loaded Array(Tuple(
+ langfuseSkillId Nullable(String),
+ skillVersion Nullable(UInt32),
+ skillName String,
+ filePath String
+ )) DEFAULT []
+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};
+ALTER TABLE events_core {CLICKHOUSE_CLUSTER_CLAUSE}
+ ADD COLUMN IF NOT EXISTS skills_available Array(Tuple(
+ langfuseSkillId Nullable(String),
+ skillVersion Nullable(UInt32),
+ skillName String
+ )) DEFAULT [],
+ ADD COLUMN IF NOT EXISTS skills_resource_loaded Array(Tuple(
+ langfuseSkillId Nullable(String),
+ skillVersion Nullable(UInt32),
+ skillName String,
+ filePath String
+ )) DEFAULT []
+ SETTINGS enable_full_text_index = 1{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};
+ALTER TABLE events_core_mv {CLICKHOUSE_CLUSTER_CLAUSE} MODIFY QUERY
+SELECT
+ project_id,
+ trace_id,
+ span_id,
+ parent_span_id,
+ start_time,
+ end_time,
+ name,
+ type,
+ environment,
+ version,
+ release,
+ trace_name,
+ user_id,
+ session_id,
+ tags,
+ level,
+ status_message,
+ completion_start_time,
+ is_app_root,
+ bookmarked,
+ public,
+ prompt_id,
+ prompt_name,
+ prompt_version,
+ model_id,
+ provided_model_name,
+ model_parameters,
+ provided_usage_details,
+ usage_details,
+ provided_cost_details,
+ cost_details,
+ usage_pricing_tier_id,
+ usage_pricing_tier_name,
+ tool_definitions,
+ tool_calls,
+ tool_call_names,
+ skills_available,
+ skills_resource_loaded,
+ leftUTF8(input, 200) as input,
+ leftUTF8(output, 200) as output,
+ metadata_names,
+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,
+ evaluator_id,
+ evaluation_rule_id,
+ evaluator_execution_is_test,
+ experiment_id,
+ experiment_name,
+ experiment_metadata_names,
+ experiment_metadata_values,
+ experiment_description,
+ experiment_dataset_id,
+ experiment_item_id,
+ experiment_item_version,
+ experiment_item_expected_output,
+ experiment_item_metadata_names,
+ experiment_item_metadata_values,
+ experiment_item_root_span_id,
+ source,
+ service_name,
+ service_version,
+ scope_name,
+ scope_version,
+ telemetry_sdk_language,
+ telemetry_sdk_name,
+ telemetry_sdk_version,
+ blob_storage_file_path,
+ event_bytes,
+ created_at,
+ updated_at,
+ event_ts,
+ is_deleted,
+ ingestion_api_key,
+ ingestion_sdk_name,
+ ingestion_sdk_version
+FROM events_full
+SETTINGS enable_full_text_index = 1;
diff --git a/packages/shared/src/domain/observations.ts b/packages/shared/src/domain/observations.ts
index 64536ebb8e..c120763fdd 100644
--- a/packages/shared/src/domain/observations.ts
+++ b/packages/shared/src/domain/observations.ts
@@ -1,6 +1,10 @@
import z from "zod";
import { jsonSchema } from "../utils/zod";
import { MetadataDomain } from "./traces";
+import {
+ SkillsAvailableSchema,
+ SkillsResourceLoadedSchema,
+} from "../features/skills/trace";
export const ObservationType = {
SPAN: "SPAN",
@@ -98,6 +102,8 @@ export const ObservationSchema = z.object({
toolDefinitions: z.record(z.string(), z.string()).nullable(),
toolCalls: z.array(z.string()).nullable(),
toolCallNames: z.array(z.string()).nullable(),
+ skillsAvailable: SkillsAvailableSchema.optional(),
+ skillsResourceLoaded: SkillsResourceLoadedSchema.optional(),
});
export type Observation = z.infer<typeof ObservationSchema>;
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index b2d7da2efb..62195e3aa1 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -160,6 +160,12 @@ const EnvSchema = z.object({
.default(20_000),
LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(3600), // 1h
+ LANGFUSE_CACHE_SKILL_ENABLED: z.enum(["true", "false"]).default("true"),
+ LANGFUSE_CACHE_SKILL_TTL_SECONDS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(3600), // 1h
CLICKHOUSE_URL: z.url(),
CLICKHOUSE_READ_ONLY_URL: z.url().optional(),
CLICKHOUSE_EVENTS_READ_ONLY_URL: z.url().optional(),
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.ts
index 1d3828f4ac..2945e0fc96 100644
--- a/packages/shared/src/features/query/dataModel.ts
+++ b/packages/shared/src/features/query/dataModel.ts
@@ -1382,6 +1382,20 @@ export const eventsObservationsView: ViewDeclarationType = {
type: "string",
description: "Month of the observation start_time in YYYY-MM format.",
},
+ skillNamesAvailable: {
+ sql: "arrayDistinct(events_observations.skills_available.skillName)",
+ alias: "skillNamesAvailable",
+ type: "arrayString",
+ explodeArray: true,
+ description: "Names of skills made available.",
+ },
+ skillsNamesLoaded: {
+ sql: "arrayDistinct(events_observations.skills_resource_loaded.skillName)",
+ alias: "skillsNamesLoaded",
+ type: "arrayString",
+ explodeArray: true,
+ description: "Names of skills made available.",
+ },
toolNames: {
sql: "mapKeys(events_observations.tool_definitions)",
alias: "toolNames",
diff --git a/packages/shared/src/features/skills/constants.ts b/packages/shared/src/features/skills/constants.ts
index e7ef695f1f..a0a1c95587 100644
--- a/packages/shared/src/features/skills/constants.ts
+++ b/packages/shared/src/features/skills/constants.ts
@@ -3,3 +3,16 @@ export const SKILL_PRODUCTION_LABEL = "production";
export const MAX_SKILL_FILES = 200;
export const MAX_SKILL_PATH_LENGTH = 512;
export const MAX_SKILL_BYTES = 1_000_000;
+
+export const MAX_SKILL_CATALOG_JSON_BYTES = 64 * 1024;
+export const MAX_SKILL_TOOL_ARGS_JSON_BYTES = 16 * 1024;
+export const MAX_SKILL_ERROR_JSON_BYTES = 4 * 1024;
+
+export const SKILL_RESOURCE_TOOL_NAMES = new Set([
+ "skill",
+ "Skill",
+ "loadSkill",
+ "skill_read",
+ "readFile",
+ "Read",
+]);
diff --git a/packages/shared/src/features/skills/trace.ts b/packages/shared/src/features/skills/trace.ts
new file mode 100644
index 0000000000..4afe748509
--- /dev/null
+++ b/packages/shared/src/features/skills/trace.ts
@@ -0,0 +1,28 @@
+import { z } from "zod/v4";
+import { SkillFilePathSchema } from "./types";
+
+export const MAX_SKILLS_PER_OBSERVATION = 100;
+
+export const SkillsAvailableSchema = z
+ .array(
+ z.object({
+ langfuseSkillId: z.string().min(1).max(256).nullish(),
+ skillName: z.string().min(1).max(128),
+ skillVersion: z.number().int().min(1).max(2_147_483_647).nullish(),
+ }),
+ )
+ .max(MAX_SKILLS_PER_OBSERVATION);
+
+export const SkillsResourceLoadedSchema = z
+ .array(
+ z.object({
+ langfuseSkillId: z.string().min(1).max(256).nullish(),
+ filePath: SkillFilePathSchema,
+ skillName: z.string().min(1).max(128),
+ skillVersion: z.number().int().min(1).max(2_147_483_647).nullish(),
+ }),
+ )
+ .max(MAX_SKILLS_PER_OBSERVATION);
+
+export type SkillsAvailable = z.infer<typeof SkillsAvailableSchema>;
+export type SkillsResourceLoaded = z.infer<typeof SkillsResourceLoadedSchema>;
diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts
index 363158452d..c6627d5a93 100644
--- a/packages/shared/src/index.ts
+++ b/packages/shared/src/index.ts
@@ -99,6 +99,7 @@ export * from "./features/prompts/constants";
// skills
export * from "./features/skills/types";
+export * from "./features/skills/trace";
export * from "./features/skills/constants";
export {
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index 78086b12ec..5a24181239 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -22,6 +22,7 @@ export * from "./services/email/commentMention/sendCommentMentionEmail";
export * from "./services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail";
export * from "./services/email/posthogExportFailed/sendPostHogExportFailedEmail";
export * from "./services/PromptService";
+export * from "./services/SkillCacheService";
export * from "./services/PromptService/types";
export * from "./services/traces-ui-table-service";
export * from "./services/InMemoryFilterService";
diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.ts
index 325142ab59..9fe2f9b797 100644
--- a/packages/shared/src/server/llm/internalTraceEvents.ts
+++ b/packages/shared/src/server/llm/internalTraceEvents.ts
@@ -1,3 +1,7 @@
+import type {
+ SkillsAvailable,
+ SkillsResourceLoaded,
+} from "../../features/skills/trace";
import {
asBoolean,
asNumberRecord,
@@ -66,6 +70,8 @@ export type InternalTraceEventInput = {
promptId?: string;
promptName?: string;
promptVersion?: string;
+ skillsAvailable?: SkillsAvailable;
+ skillsResourceLoaded?: SkillsResourceLoaded;
modelId?: string;
modelName?: string;
modelParameters?: string | Record<string, unknown>;
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index 1a7d40da62..bad1965664 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -1,5 +1,19 @@
/* eslint-disable no-nested-ternary */
import { randomUUID } from "crypto";
+import {
+ MAX_SKILLS_PER_OBSERVATION,
+ SkillsAvailableSchema,
+ SkillsResourceLoadedSchema,
+ type SkillsAvailable,
+ type SkillsResourceLoaded,
+} from "../../features/skills/trace";
+import {
+ MAX_SKILL_CATALOG_JSON_BYTES,
+ MAX_SKILL_ERROR_JSON_BYTES,
+ MAX_SKILL_PATH_LENGTH,
+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ SKILL_RESOURCE_TOOL_NAMES,
+} from "../../features/skills/constants";
import {
ForbiddenError,
@@ -627,6 +641,14 @@ export class OtelIngestionProcessor {
span.status?.message ??
null,
+ ...this.extractSkills(
+ spanAttributes,
+ observationType,
+ this.extractName(span.name, spanAttributes),
+ input,
+ output,
+ span.status?.code,
+ ),
promptName: canLinkPrompt
? (spanAttributes?.[
LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME
@@ -1352,6 +1374,14 @@ export class OtelIngestionProcessor {
model: skipModelUsageAndCost
? undefined
: this.extractModelName(attributes),
+ ...this.extractSkills(
+ attributes,
+ mappedObservationType,
+ this.extractName(span.name, attributes),
+ input,
+ output,
+ span.status?.code,
+ ),
promptName: canLinkPrompt
? (attributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME] ??
attributes["langfuse.prompt.name"] ??
@@ -1895,6 +1925,7 @@ export class OtelIngestionProcessor {
LangfuseOtelSpanAttributes.TRACE_OUTPUT,
LangfuseOtelSpanAttributes.OBSERVATION_INPUT,
LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT,
+ LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE,
// Vercel AI SDK
"ai.prompt.messages",
"ai.prompt",
@@ -3904,6 +3935,203 @@ export class OtelIngestionProcessor {
}
}
+ private extractSkills(
+ attributes: Record<string, unknown>,
+ observationType: string,
+ toolName: string,
+ input: unknown,
+ output: unknown,
+ statusCode: number | undefined,
+ ): {
+ skillsAvailable?: SkillsAvailable;
+ skillsResourceLoaded?: SkillsResourceLoaded;
+ } {
+ const manifest =
+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE];
+ // Skill inference is enabled only when an availability attribute is reported.
+ if (!manifest) return {};
+ if (
+ observationType !== ObservationType.GENERATION &&
+ (observationType !== ObservationType.TOOL ||
+ !SKILL_RESOURCE_TOOL_NAMES.has(toolName) ||
+ output == null ||
+ statusCode === 2 ||
+ parseObservationLevel(
+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],
+ ) === ObservationLevel.ERROR)
+ ) {
+ return {};
+ }
+
+ let parsedManifest = Array.isArray(manifest)
+ ? manifest
+ : this.parseJsonPayloadWithMaxSize(
+ manifest,
+ MAX_SKILL_CATALOG_JSON_BYTES,
+ "skillsAvailable",
+ );
+ if (
+ Array.isArray(parsedManifest) &&
+ parsedManifest.length > MAX_SKILLS_PER_OBSERVATION
+ ) {
+ logger.warn("OTEL skill catalog dropped: item limit exceeded", {
+ projectId: this.projectId,
+ count: parsedManifest.length,
+ limit: MAX_SKILLS_PER_OBSERVATION,
+ });
+ parsedManifest = undefined;
+ }
+ const available = Array.isArray(parsedManifest)
+ ? SkillsAvailableSchema.safeParse(parsedManifest)
+ : undefined;
+ if (observationType === ObservationType.GENERATION) {
+ if (available?.success) return { skillsAvailable: available.data };
+ return {};
+ }
+
+ const candidate = this.extractSkillResource(
+ toolName,
+ input,
+ output,
+ available?.success ? available.data : [],
+ );
+ if (!candidate) return {};
+ if (
+ typeof candidate.filePath === "string" &&
+ candidate.filePath.length > MAX_SKILL_PATH_LENGTH
+ ) {
+ logger.warn(
+ "OTEL skill resource load dropped: path length limit exceeded",
+ {
+ projectId: this.projectId,
+ toolName,
+ length: candidate.filePath.length,
+ limit: MAX_SKILL_PATH_LENGTH,
+ },
+ );
+ return {};
+ }
+ const loaded = SkillsResourceLoadedSchema.element.safeParse(candidate);
+ if (!loaded.success) return {};
+
+ return {
+ ...(available?.success ? { skillsAvailable: available.data } : {}),
+ skillsResourceLoaded: [
+ { skillName: loaded.data.skillName, filePath: loaded.data.filePath },
+ ],
+ };
+ }
+
+ private extractSkillResource(
+ toolName: string,
+ input: unknown,
+ output: unknown,
+ available: SkillsAvailable,
+ ) {
+ const args = this.parseJsonPayloadWithMaxSize(
+ input,
+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ "toolInput",
+ );
+ if (!OtelIngestionProcessor.isPlainObject(args)) return;
+ const oversizedPath = [args.path, args.file_path].find(
+ (path) =>
+ typeof path === "string" &&
+ path.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ );
+ if (typeof oversizedPath === "string") {
+ logger.warn(
+ "OTEL skill resource load dropped: input path length limit exceeded",
+ {
+ projectId: this.projectId,
+ toolName,
+ length: oversizedPath.length,
+ limit: MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ },
+ );
+ return;
+ }
+
+ switch (toolName) {
+ case "skill":
+ case "Skill":
+ case "loadSkill":
+ if (this.isFailedSkillActivation(output)) return;
+ return {
+ skillName: args.name ?? args.skill,
+ filePath: "SKILL.md",
+ };
+ case "skill_read":
+ return {
+ skillName: args.skillName,
+ filePath: args.path,
+ };
+ case "readFile":
+ case "Read":
+ return this.extractSkillResourceFromPath(
+ args.path ?? args.file_path,
+ available,
+ );
+ default:
+ return;
+ }
+ }
+
+ private extractSkillResourceFromPath(
+ path: unknown,
+ available: SkillsAvailable,
+ ) {
+ if (typeof path !== "string") return;
+ const segments = path.split("/");
+ const matches = segments.filter((segment) =>
+ available.some((skill) => skill.skillName === segment),
+ );
+ if (matches.length !== 1) return;
+ const skillName = matches[0];
+ return {
+ skillName,
+ filePath: segments.slice(segments.indexOf(skillName) + 1).join("/"),
+ };
+ }
+
+ private isFailedSkillActivation(output: unknown): boolean {
+ const result = this.parseJsonPayloadWithMaxSize(
+ output,
+ MAX_SKILL_ERROR_JSON_BYTES,
+ "toolOutput",
+ );
+ const text = typeof result === "string" ? result : output;
+ return (
+ (OtelIngestionProcessor.isPlainObject(result) &&
+ (result.isError === true || result.error !== undefined)) ||
+ (typeof text === "string" &&
+ /^(?:Skill|File) ".*" not found/.test(
+ text.slice(0, MAX_SKILL_ERROR_JSON_BYTES),
+ ))
+ );
+ }
+
+ private parseJsonPayloadWithMaxSize(
+ payload: unknown,
+ maxBytes: number,
+ field: "skillsAvailable" | "toolInput" | "toolOutput",
+ ): unknown {
+ if (
+ typeof payload === "string" &&
+ (payload.length > maxBytes ||
+ Buffer.byteLength(payload, "utf8") > maxBytes)
+ ) {
+ logger.warn("OTEL skill payload parsing skipped: byte limit exceeded", {
+ projectId: this.projectId,
+ field,
+ limitBytes: maxBytes,
+ stringLength: payload.length,
+ });
+ return undefined;
+ }
+ return this.parseJsonPayload(payload);
+ }
+
/**
* Parses a payload (object or JSON string) and returns the object
* or `undefined` if invalid.
diff --git a/packages/shared/src/server/otel/attributes.ts b/packages/shared/src/server/otel/attributes.ts
index 21358cab9e..022c52b230 100644
--- a/packages/shared/src/server/otel/attributes.ts
+++ b/packages/shared/src/server/otel/attributes.ts
@@ -18,6 +18,7 @@ export enum LangfuseOtelSpanAttributes {
OBSERVATION_STATUS_MESSAGE = "langfuse.observation.status_message",
OBSERVATION_INPUT = "langfuse.observation.input",
OBSERVATION_OUTPUT = "langfuse.observation.output",
+ OBSERVATION_SKILLS_AVAILABLE = "langfuse.observation.skills_available",
// Langfuse-observation of type Generation attributes
OBSERVATION_COMPLETION_START_TIME = "langfuse.observation.completion_start_time",
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts b/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts
index cce72aafd2..d0d45dab18 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts
@@ -216,6 +216,16 @@ const EVENTS_FILTER_OPTION_DEFINITIONS = {
expression: eventsTableHasParentObservationSql,
sort: "booleanAsc",
},
+ skillNamesAvailable: {
+ kind: "array",
+ expression: "arrayDistinct(e.skills_available.skillName)",
+ sort: "countDesc",
+ },
+ skillsNamesLoaded: {
+ kind: "array",
+ expression: "arrayDistinct(e.skills_resource_loaded.skillName)",
+ sort: "countDesc",
+ },
toolNames: {
kind: "array",
expression: "mapKeys(e.tool_definitions)",
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index 3384fbfdab..a84d6dddfb 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -7,6 +7,10 @@ import {
eventsTableTraceNameAggregationSql,
eventsTableTraceNameSelectSql,
} from "../../../eventsTable";
+import type {
+ SkillsAvailable,
+ SkillsResourceLoaded,
+} from "../../../features/skills/trace";
import { expressionBuilder, sql, type SelectExpression } from "kysely";
import { renderClickhouseExpression } from "../../query-ast/compile";
@@ -184,6 +188,10 @@ const EVENTS_FIELDS = {
promptName: "e.prompt_name as prompt_name",
promptVersion: "e.prompt_version as prompt_version",
+ // Skill fields
+ skillsAvailable: 'e.skills_available as "skills_available"',
+ skillsResourceLoaded: 'e.skills_resource_loaded as "skills_resource_loaded"',
+
// Tool fields
toolDefinitions: "e.tool_definitions as tool_definitions",
toolCalls: "e.tool_calls as tool_calls",
@@ -376,6 +384,8 @@ const FIELD_SETS = {
"toolDefinitions",
"toolCalls",
"toolCallNames",
+ "skillsAvailable",
+ "skillsResourceLoaded",
],
byIdModel: [
"providedModelName",
@@ -554,6 +564,12 @@ type EventsColumnField = {
const EVENTS_FIELD_EXPRESSIONS = {
traceName: sql.raw<string>(eventsTableTraceNameSelectSql).as("trace_name"),
metadata: eventsMetadataMap.as("metadata"),
+ skillsAvailable: sql
+ .ref<SkillsAvailable>("e.skills_available")
+ .as("skills_available"),
+ skillsResourceLoaded: sql
+ .ref<SkillsResourceLoaded>("e.skills_resource_loaded")
+ .as("skills_resource_loaded"),
} satisfies Partial<
Record<Exclude<EventsFieldKey, EventsColumnField>, EventsSelection>
>;
diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
index 762b55b54a..07bb507d38 100644
--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
@@ -200,7 +200,7 @@ WHERE (e.project_id = {param1:String}) AND (e.span_id IN ({param2:Array(String)}
]
`;
-exports[`golden: events IO payload reads family > getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls 1`] = `
+exports[`golden: events IO payload reads family > getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls+skills 1`] = `
[
{
"fn": "queryClickhouse",
@@ -249,6 +249,8 @@ exports[`golden: events IO payload reads family > getObservationsBatchIOFromEven
arrayReverse(e.experiment_item_metadata_names),
arrayReverse(arrayMap(v -> leftUTF8(v, 5000), e.experiment_item_metadata_values))
) AS experiment_item_metadata,
+ e.skills_available AS skills_available,
+ e.skills_resource_loaded AS skills_resource_loaded,
e.tool_calls AS tool_calls,
e.tool_call_names AS tool_call_names,
mapFromArrays(
@@ -833,6 +835,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromE
e.prompt_id AS prompt_id,
e.prompt_name AS prompt_name,
e.prompt_version AS prompt_version,
+ e.skills_available AS skills_available,
+ e.skills_resource_loaded AS skills_resource_loaded,
e.tool_definitions AS tool_definitions,
e.tool_calls AS tool_calls,
e.tool_call_names AS tool_call_names,
@@ -900,6 +904,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromE
e.prompt_id AS prompt_id,
e.prompt_name AS prompt_name,
e.prompt_version AS prompt_version,
+ e.skills_available AS skills_available,
+ e.skills_resource_loaded AS skills_resource_loaded,
e.tool_definitions AS tool_definitions,
e.tool_calls AS tool_calls,
e.tool_call_names AS tool_call_names,
@@ -969,6 +975,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromE
e.prompt_id AS prompt_id,
e.prompt_name AS prompt_name,
e.prompt_version AS prompt_version,
+ e.skills_available AS skills_available,
+ e.skills_resource_loaded AS skills_resource_loaded,
e.tool_definitions AS tool_definitions,
e.tool_calls AS tool_calls,
e.tool_call_names AS tool_call_names,
@@ -1042,6 +1050,8 @@ exports[`golden: events point-reads & existence family > getObservationByIdFromE
e.prompt_id AS prompt_id,
e.prompt_name AS prompt_name,
e.prompt_version AS prompt_version,
+ e.skills_available AS skills_available,
+ e.skills_resource_loaded AS skills_resource_loaded,
e.tool_definitions AS tool_definitions,
e.tool_calls AS tool_calls,
e.tool_call_names AS tool_call_names,
diff --git a/packages/shared/src/server/repositories/definitions.ts b/packages/shared/src/server/repositories/definitions.ts
index 5a696023b5..2a46916adf 100644
--- a/packages/shared/src/server/repositories/definitions.ts
+++ b/packages/shared/src/server/repositories/definitions.ts
@@ -1,4 +1,8 @@
import z from "zod";
+import {
+ SkillsAvailableSchema,
+ SkillsResourceLoadedSchema,
+} from "../../features/skills/trace";
import type { IngestionAttribution } from "../ingestion/ingestionAttribution";
import { DEFAULT_TRACE_ENVIRONMENT } from "../ingestion/types";
import { toClickhouseDateTime } from "../clickhouse/client";
@@ -118,6 +122,8 @@ export const eventsObservationRecordBaseSchema =
export const eventsObservationRecordReadSchema =
observationRecordReadSchema.extend({
+ skills_available: SkillsAvailableSchema.optional(),
+ skills_resource_loaded: SkillsResourceLoadedSchema.optional(),
user_id: z.string().nullish(),
session_id: z.string().nullish(),
is_root_observation: z.boolean().optional(),
@@ -513,6 +519,8 @@ export const eventRecordBaseSchema = z.object({
tool_definitions: z.record(z.string(), z.string()).default({}),
tool_calls: z.array(z.string()).default([]),
tool_call_names: z.array(z.string()).default([]),
+ skills_available: SkillsAvailableSchema.optional(),
+ skills_resource_loaded: SkillsResourceLoadedSchema.optional(),
// I/O
input: z.string().nullish(),
diff --git a/packages/shared/src/server/repositories/events.golden.test.ts b/packages/shared/src/server/repositories/events.golden.test.ts
index faf8b4cbd1..6a9e7e0a51 100644
--- a/packages/shared/src/server/repositories/events.golden.test.ts
+++ b/packages/shared/src/server/repositories/events.golden.test.ts
@@ -399,13 +399,14 @@ describeWithClickhouse("golden: events IO payload reads family", () => {
expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
});
- it("getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls", async () => {
+ it("getObservationsBatchIOFromEventsTable full+ioCharLimit+experiment+toolCalls+skills", async () => {
await getObservationsBatchIOFromEventsTable({
...batchBase,
truncated: false,
ioCharLimit: 5000.7,
includeExperimentFields: true,
includeToolCallFields: true,
+ includeSkillFields: true,
});
expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
});
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 2a7100c71f..e8de269d2e 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -175,6 +175,10 @@ import {
} from "../../tableDefinitions";
import { tracesTableCols } from "../../tableDefinitions/tracesTable";
import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import type {
+ SkillsAvailable,
+ SkillsResourceLoaded,
+} from "../../features/skills/trace";
export type EventBatchIOStringOutput = {
id: string;
@@ -184,6 +188,11 @@ export type EventBatchIOStringOutput = {
metadata: MetadataDomain;
};
+export type EventBatchIOSkillFields = {
+ skillsAvailable: SkillsAvailable;
+ skillsResourceLoaded: SkillsResourceLoaded;
+};
+
/** Raw ClickHouse storage shape: name-less JSON strings, names in the parallel toolCallNames array. */
export type EventBatchIOToolCallFields = {
toolCalls: string[];
@@ -198,10 +207,12 @@ export type EventBatchIOWithExperimentOutput = EventBatchIOStringOutput & {
export type EventBatchIOResult<
TIncludeExperiment extends boolean,
TIncludeToolCalls extends boolean,
+ TIncludeSkills extends boolean = false,
> = (TIncludeExperiment extends true
? EventBatchIOWithExperimentOutput
: EventBatchIOStringOutput) &
- (TIncludeToolCalls extends true ? EventBatchIOToolCallFields : object);
+ (TIncludeToolCalls extends true ? EventBatchIOToolCallFields : object) &
+ (TIncludeSkills extends true ? EventBatchIOSkillFields : object);
const BATCH_IO_STRING_RENDERING_PROPS: RenderingProps = {
// Batch I/O truncation is handled in SQL via leftUTF8 for performance.
@@ -2721,6 +2732,7 @@ export const deleteEventsOlderThanDays = async (
export const getObservationsBatchIOFromEventsTable = async <
TIncludeExperiment extends boolean = false,
TIncludeToolCalls extends boolean = false,
+ TIncludeSkills extends boolean = false,
>(opts: {
projectId: string;
observations: Array<{
@@ -2741,8 +2753,11 @@ export const getObservationsBatchIOFromEventsTable = async <
includeExperimentFields?: TIncludeExperiment;
/** Opt-in: tool-call arrays can be large; only eval consumers need them. */
includeToolCallFields?: TIncludeToolCalls;
+ includeSkillFields?: TIncludeSkills;
}): Promise<
- Array<EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls>>
+ Array<
+ EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls, TIncludeSkills>
+ >
> => {
if (opts.observations.length === 0) {
return [];
@@ -2805,6 +2820,14 @@ export const getObservationsBatchIOFromEventsTable = async <
).as("experiment_item_metadata"),
]),
)
+ .$if(opts.includeSkillFields === true, (qb) =>
+ qb.select([
+ sql.ref<SkillsAvailable>("e.skills_available").as("skills_available"),
+ sql
+ .ref<SkillsResourceLoaded>("e.skills_resource_loaded")
+ .as("skills_resource_loaded"),
+ ]),
+ )
.$if(opts.includeToolCallFields === true, (qb) =>
qb.select([
"e.tool_calls as tool_calls",
@@ -2846,6 +2869,8 @@ export const getObservationsBatchIOFromEventsTable = async <
input: string | null;
output: string | null;
metadata: Record<string, string>;
+ skills_available?: SkillsAvailable;
+ skills_resource_loaded?: SkillsResourceLoaded;
tool_calls?: string[];
tool_call_names?: string[];
experiment_item_expected_output?: string | null;
@@ -2864,6 +2889,12 @@ export const getObservationsBatchIOFromEventsTable = async <
output: applyBatchIOStringRendering(r.output),
metadata:
r.metadata !== undefined ? parseMetadataCHRecordToDomain(r.metadata) : {},
+ ...(opts.includeSkillFields
+ ? {
+ skillsAvailable: r.skills_available ?? [],
+ skillsResourceLoaded: r.skills_resource_loaded ?? [],
+ }
+ : {}),
...(opts.includeToolCallFields
? {
toolCalls: r.tool_calls ?? [],
@@ -2879,7 +2910,9 @@ export const getObservationsBatchIOFromEventsTable = async <
: {},
}
: {}),
- })) as Array<EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls>>;
+ })) as Array<
+ EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls, TIncludeSkills>
+ >;
};
/**
diff --git a/packages/shared/src/server/repositories/observations_converters.ts b/packages/shared/src/server/repositories/observations_converters.ts
index e51b212c9c..8d215a7f73 100644
--- a/packages/shared/src/server/repositories/observations_converters.ts
+++ b/packages/shared/src/server/repositories/observations_converters.ts
@@ -408,6 +408,14 @@ export function convertEventsObservation(
renderingProps: RenderingProps = DEFAULT_RENDERING_PROPS,
complete: boolean,
): EventsObservation | PartialEventsObservation {
+ const skills = {
+ ...(record.skills_available !== undefined && {
+ skillsAvailable: record.skills_available,
+ }),
+ ...(record.skills_resource_loaded !== undefined && {
+ skillsResourceLoaded: record.skills_resource_loaded,
+ }),
+ };
if (complete) {
const baseObservation = convertObservationPartial(
record as ObservationRecordReadType,
@@ -416,6 +424,7 @@ export function convertEventsObservation(
);
return {
...baseObservation,
+ ...skills,
userId: record.user_id ?? null,
sessionId: record.session_id ?? null,
...(record.is_root_observation !== undefined && {
@@ -436,6 +445,7 @@ export function convertEventsObservation(
);
return {
...baseObservation,
+ ...skills,
...(record.user_id !== undefined && { userId: record.user_id }),
...(record.session_id !== undefined && { sessionId: record.session_id }),
...(record.is_root_observation !== undefined && {
diff --git a/packages/shared/src/server/services/SkillCacheService.ts b/packages/shared/src/server/services/SkillCacheService.ts
new file mode 100644
index 0000000000..891ad55549
--- /dev/null
+++ b/packages/shared/src/server/services/SkillCacheService.ts
@@ -0,0 +1,104 @@
+import type { Redis, Cluster } from "ioredis";
+import type { PrismaClient, Skill } from "../../db";
+import { env } from "../../env";
+import { logger } from "../logger";
+
+type SkillVersion = Pick<Skill, "name" | "version">;
+type SkillReference = Pick<Skill, "id" | "name" | "version">;
+
+export class SkillCacheService {
+ private readonly cacheEnabled: boolean;
+
+ constructor(
+ private readonly prisma: PrismaClient,
+ private readonly redis: Redis | Cluster | null,
+ ) {
+ this.cacheEnabled =
+ Boolean(redis) && env.LANGFUSE_CACHE_SKILL_ENABLED === "true";
+ }
+
+ async getReferences({
+ projectId,
+ versions,
+ }: {
+ projectId: string;
+ versions: SkillVersion[];
+ }): Promise<SkillReference[]> {
+ const unique = new Map<string, SkillVersion>();
+ for (const version of versions) {
+ unique.set(JSON.stringify([version.name, version.version]), version);
+ }
+ versions = Array.from(unique.values());
+ if (versions.length === 0) return [];
+
+ let cachedIds: (string | null)[] = [];
+ if (this.cacheEnabled && this.redis) {
+ try {
+ const keys = versions.map((version) =>
+ this.getCacheKey(projectId, version),
+ );
+ cachedIds = await this.redis.mget(...keys);
+ } catch (error) {
+ logger.error("[SkillCacheService] Error reading cache", error);
+ }
+ }
+
+ const missing = versions.filter((_, index) => !cachedIds[index]);
+ const fetched = missing.length
+ ? await this.prisma.skill.findMany({
+ where: { projectId, OR: missing },
+ select: { id: true, name: true, version: true },
+ })
+ : [];
+
+ if (this.cacheEnabled && this.redis && fetched.length) {
+ try {
+ const pipeline = this.redis.pipeline();
+ for (const skill of fetched) {
+ pipeline.set(
+ this.getCacheKey(projectId, skill),
+ skill.id,
+ "EX",
+ env.LANGFUSE_CACHE_SKILL_TTL_SECONDS,
+ );
+ }
+ const results = await pipeline.exec();
+ const error = results?.find(([error]) => error)?.[0];
+ if (error) throw error;
+ } catch (error) {
+ logger.error("[SkillCacheService] Error writing cache", error);
+ }
+ }
+
+ const cached = versions.flatMap((version, index) => {
+ const id = cachedIds[index];
+ return id ? [{ ...version, id }] : [];
+ });
+ return cached.concat(fetched);
+ }
+
+ async invalidateCache({
+ projectId,
+ versions,
+ }: {
+ projectId: string;
+ versions: SkillVersion[];
+ }): Promise<void> {
+ if (!this.cacheEnabled || !this.redis || versions.length === 0) return;
+ try {
+ await this.redis.del(
+ ...versions.map((version) => this.getCacheKey(projectId, version)),
+ );
+ } catch (error) {
+ logger.error("[SkillCacheService] Error invalidating cache", error);
+ }
+ }
+
+ private getCacheKey(
+ projectId: string,
+ { name, version }: SkillVersion,
+ ): string {
+ // A project hash tag keeps batch reads and writes in one Redis Cluster slot.
+ return `skill_reference:{${encodeURIComponent(projectId)}}:${encodeURIComponent(name)}:version:${version}`;
+ }
+}
diff --git a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts
index 110f1e7e97..66a99c4034 100644
--- a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts
+++ b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts
@@ -136,6 +136,16 @@ const viewFilterDefinitions: Record<
"level",
sourceSpec("Status", { uiTableId: "level", aliases: ["Level"] }),
),
+ defineField(
+ "skillNamesAvailable",
+ sourceSpec("Skill Names (Available)", {
+ uiTableId: "skillNamesAvailable",
+ }),
+ ),
+ defineField(
+ "skillsNamesLoaded",
+ sourceSpec("Skill Names (Loaded)", { uiTableId: "skillsNamesLoaded" }),
+ ),
defineField(
"toolNames",
sourceSpec("Tool Names (Available)", { uiTableId: "toolNames" }),
diff --git a/web/src/features/events/server/eventsRouter.ts b/web/src/features/events/server/eventsRouter.ts
index 8d6517fb0a..9bd1c092e1 100644
--- a/web/src/features/events/server/eventsRouter.ts
+++ b/web/src/features/events/server/eventsRouter.ts
@@ -13,6 +13,8 @@ import {
} from "@/src/server/api/trpc";
import {
type OrderByState,
+ type SkillsAvailable,
+ type SkillsResourceLoaded,
normalizeOrderByForTable,
paginationZod,
singleFilterList,
@@ -74,6 +76,8 @@ export type EventBatchIOOutput = {
input: string | null;
output: string | null;
metadata: MetadataDomainClient;
+ skillsAvailable?: SkillsAvailable;
+ skillsResourceLoaded?: SkillsResourceLoaded;
};
export type GetAllEventsInput = z.infer<typeof GetAllEventsInput>;
@@ -123,6 +127,7 @@ const BatchIOInput = zodSchema.object({
// rendering anyway (IO_TABLE_CHAR_LIMIT).
ioCharLimit: zodSchema.number().int().positive().max(10_000).optional(),
includeToolCalls: zodSchema.boolean().optional(), // Defaults to false; tool-call arrays can be large
+ includeSkills: zodSchema.boolean().optional(),
// Opts into trace-level auth (public traces) in protectedGetEventsTraceProcedure
traceId: zodSchema.string().optional(),
});
@@ -349,6 +354,7 @@ export const eventsRouter = createTRPCRouter({
truncated: input.truncated,
ioCharLimit: input.ioCharLimit,
includeToolCallFields: input.includeToolCalls,
+ includeSkillFields: input.includeSkills,
});
return batchIO.map(toDomainWithStringifiedMetadata);
@@ -374,6 +380,7 @@ export const eventsRouter = createTRPCRouter({
truncated: input.truncated,
ioCharLimit: input.ioCharLimit,
includeToolCallFields: input.includeToolCalls,
+ includeSkillFields: input.includeSkills,
});
return batchIO.map(toDomainWithStringifiedMetadata);
@@ -398,6 +405,7 @@ export const eventsRouter = createTRPCRouter({
ioCharLimit: input.ioCharLimit,
includeExperimentFields: true,
includeToolCallFields: input.includeToolCalls,
+ includeSkillFields: input.includeSkills,
});
return batchIO.map(toDomainWithStringifiedMetadata);
diff --git a/web/src/features/events/server/eventsService.ts b/web/src/features/events/server/eventsService.ts
index 04bbc9c3b1..37ad1f7573 100644
--- a/web/src/features/events/server/eventsService.ts
+++ b/web/src/features/events/server/eventsService.ts
@@ -161,6 +161,8 @@ const EVENT_FILTER_OPTION_COLUMNS = [
"isRootObservation",
"toolNames",
"calledToolNames",
+ "skillNamesAvailable",
+ "skillsNamesLoaded",
"metadataKeys",
] as const satisfies readonly EventFilterOptionColumn[];
@@ -472,6 +474,8 @@ const EVENT_FILTER_VALUE_ONLY_COLUMNS = new Set<EventFilterOptionColumn>([
"traceTags",
"toolNames",
"calledToolNames",
+ "skillNamesAvailable",
+ "skillsNamesLoaded",
]);
const EVENT_FILTER_OPTIONS_NON_PARTICIPATING_COLUMNS = new Set([
@@ -923,6 +927,7 @@ export async function getEventMetadataValues(
interface GetEventBatchIOParams<
TIncludeExperiment extends boolean = false,
TIncludeToolCalls extends boolean = false,
+ TIncludeSkills extends boolean = false,
> {
projectId: string;
observations: Array<{
@@ -937,6 +942,7 @@ interface GetEventBatchIOParams<
includeExperimentFields?: TIncludeExperiment;
/** Opt-in: tool-call arrays can be large; only eval consumers need them. */
includeToolCallFields?: TIncludeToolCalls;
+ includeSkillFields?: TIncludeSkills;
}
/**
@@ -945,9 +951,18 @@ interface GetEventBatchIOParams<
export async function getEventBatchIO<
TIncludeExperiment extends boolean = false,
TIncludeToolCalls extends boolean = false,
+ TIncludeSkills extends boolean = false,
>(
- params: GetEventBatchIOParams<TIncludeExperiment, TIncludeToolCalls>,
-): Promise<Array<EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls>>> {
+ params: GetEventBatchIOParams<
+ TIncludeExperiment,
+ TIncludeToolCalls,
+ TIncludeSkills
+ >,
+): Promise<
+ Array<
+ EventBatchIOResult<TIncludeExperiment, TIncludeToolCalls, TIncludeSkills>
+ >
+> {
return getObservationsBatchIOFromEventsTable({
projectId: params.projectId,
observations: params.observations,
@@ -958,5 +973,6 @@ export async function getEventBatchIO<
sessionId: params.sessionId,
includeExperimentFields: params.includeExperimentFields,
includeToolCallFields: params.includeToolCallFields,
+ includeSkillFields: params.includeSkillFields,
});
}
diff --git a/web/src/features/metrics/components/MetricsFilterBuilder.tsx b/web/src/features/metrics/components/MetricsFilterBuilder.tsx
index 1455d59324..07dfeb0ecf 100644
--- a/web/src/features/metrics/components/MetricsFilterBuilder.tsx
+++ b/web/src/features/metrics/components/MetricsFilterBuilder.tsx
@@ -74,6 +74,8 @@ const eventsFilterOptionsColumns = [
"experimentName",
"isRootObservation",
"calledToolNames",
+ "skillNamesAvailable",
+ "skillsNamesLoaded",
"metadataKeys",
"scores_avg",
"score_categories",
@@ -388,6 +390,8 @@ const buildV2FilterColumnsParams = ({
modelOptions: filterOptions?.providedModelName ?? [],
toolNamesOptions: slowFilterOptions?.toolNames ?? [],
calledToolNamesOptions: filterOptions?.calledToolNames ?? [],
+ skillNamesAvailableOptions: filterOptions?.skillNamesAvailable ?? [],
+ skillsNamesLoadedOptions: filterOptions?.skillsNamesLoaded ?? [],
observationLevelOptions,
experimentNameOptions: filterOptions?.experimentName ?? [],
experimentDatasetOptions:
diff --git a/web/src/features/metrics/metricsFilterColumns.ts b/web/src/features/metrics/metricsFilterColumns.ts
index b9a6d4ad0e..9b4fb23cc4 100644
--- a/web/src/features/metrics/metricsFilterColumns.ts
+++ b/web/src/features/metrics/metricsFilterColumns.ts
@@ -17,6 +17,8 @@ export type GetMetricsFilterColumnsParams = {
modelOptions: SingleValueOption[];
toolNamesOptions: SingleValueOption[];
calledToolNamesOptions: SingleValueOption[];
+ skillNamesAvailableOptions?: SingleValueOption[];
+ skillsNamesLoadedOptions?: SingleValueOption[];
observationLevelOptions: SingleValueOption[];
experimentNameOptions: SingleValueOption[];
experimentDatasetOptions: SingleValueOption[];
@@ -46,6 +48,8 @@ const getMetricsFilterColumnSpecs = ({
modelOptions,
toolNamesOptions,
calledToolNamesOptions,
+ skillNamesAvailableOptions = [],
+ skillsNamesLoadedOptions = [],
observationLevelOptions,
experimentNameOptions,
experimentDatasetOptions,
@@ -232,9 +236,29 @@ const getMetricsFilterColumnSpecs = ({
if (selectedView === "observations") {
filterColumns.push(
- // v2-only filter columns (experiment data only exists in events table)
+ // These filters require the v2 events view.
...(viewVersion === "v2"
? [
+ {
+ column: {
+ name: "Skill Names (Available)",
+ id: "skillNamesAvailable",
+ type: "arrayOptions",
+ options: skillNamesAvailableOptions,
+ internal: "internalValue",
+ },
+ customSelect: true,
+ } satisfies MetricsFilterColumnSpec,
+ {
+ column: {
+ name: "Skill Names (Loaded)",
+ id: "skillsNamesLoaded",
+ type: "arrayOptions",
+ options: skillsNamesLoadedOptions,
+ internal: "internalValue",
+ },
+ customSelect: true,
+ } satisfies MetricsFilterColumnSpec,
{
column: {
name: "Experiment Name",
diff --git a/web/src/features/public-api/types/observations.ts b/web/src/features/public-api/types/observations.ts
index 337bbd65a5..2e1ea6f2a6 100644
--- a/web/src/features/public-api/types/observations.ts
+++ b/web/src/features/public-api/types/observations.ts
@@ -168,6 +168,10 @@ export const transformDbToApiObservation = (
toolCallNames,
+ // Exclude skill data from the legacy public API.
+ skillsAvailable,
+ skillsResourceLoaded,
+
// Exclude publish/bookmark flags from V1 public observations API.
// V2 observations already exposes these on the events-based contract.
bookmarked,
diff --git a/web/src/features/skills/server/skill-service.ts b/web/src/features/skills/server/skill-service.ts
index f5bf1f80a2..164f946a24 100644
--- a/web/src/features/skills/server/skill-service.ts
+++ b/web/src/features/skills/server/skill-service.ts
@@ -19,6 +19,7 @@ import {
type SkillSelector,
} from "@langfuse/shared";
import type { Prisma, PrismaClient, SkillBlob } from "@langfuse/shared/src/db";
+import { SkillCacheService, redis } from "@langfuse/shared/src/server";
import type { ProjectAuthedContext } from "@/src/server/api/trpc";
import type { AuthorizationContext } from "@/src/features/auth/policy/types";
import { auditLog } from "@/src/features/audit-logs/server";
@@ -123,7 +124,11 @@ function serializeVersion(skill: SkillWithRelations) {
}
export class SkillService {
- constructor(private readonly prisma: PrismaClient) {}
+ private readonly skillCacheService: SkillCacheService;
+
+ constructor(private readonly prisma: PrismaClient) {
+ this.skillCacheService = new SkillCacheService(prisma, redis);
+ }
async skillVersions(params: {
projectId: string;
@@ -590,7 +595,7 @@ export class SkillService {
actor: SkillActor;
}): Promise<void> {
const where = { projectId: params.projectId, name: params.name };
- await this.prisma.$transaction(async (tx) => {
+ const versions = await this.prisma.$transaction(async (tx) => {
await this.lockSkill(tx, params.projectId, params.name);
const versions = await tx.skill.findMany({
where,
@@ -621,6 +626,11 @@ export class SkillService {
tx,
);
await tx.skill.deleteMany({ where });
+ return versions.map(({ version }) => ({ name: params.name, version }));
+ });
+ await this.skillCacheService.invalidateCache({
+ projectId: params.projectId,
+ versions,
});
}
@@ -691,6 +701,10 @@ export class SkillService {
}
}
});
+ await this.skillCacheService.invalidateCache({
+ projectId: params.projectId,
+ versions: [{ name: params.name, version: params.version }],
+ });
}
private async prepareVersionFiles(params: {
diff --git a/web/src/features/traces/components/DetailAttributesTab.clienttest.tsx b/web/src/features/traces/components/DetailAttributesTab.clienttest.tsx
index 285e385eae..0312a945ae 100644
--- a/web/src/features/traces/components/DetailAttributesTab.clienttest.tsx
+++ b/web/src/features/traces/components/DetailAttributesTab.clienttest.tsx
@@ -9,6 +9,9 @@ import { MediaEnabledFields } from "@/src/features/media/validation";
vi.mock("@/src/components/ui/PrettyJsonView", () => ({
PrettyJsonView: vi.fn(() => null),
}));
+vi.mock("@/src/features/feature-flags", () => ({
+ useInternalFeaturesEnabled: () => false,
+}));
vi.mock("./ObservationDetailView/AttributeRowActions", () => ({
AttributeRowActions: () => null,
}));
diff --git a/web/src/features/traces/components/DetailAttributesTab.tsx b/web/src/features/traces/components/DetailAttributesTab.tsx
index ab3c4995db..e1f01be6a8 100644
--- a/web/src/features/traces/components/DetailAttributesTab.tsx
+++ b/web/src/features/traces/components/DetailAttributesTab.tsx
@@ -1,9 +1,14 @@
/** Trace metadata, or observation attributes, model parameters and metadata. */
import { useMemo } from "react";
+import {
+ type SkillsAvailable,
+ type SkillsResourceLoaded,
+} from "@langfuse/shared";
import { type MediaReturnType } from "@/src/features/media";
import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";
+import { useInternalFeaturesEnabled } from "@/src/features/feature-flags";
import { cn } from "@/src/utils/tailwind";
import { IO_SECTIONS_FLUSH_CLASS } from "@/src/features/traces/constants/ioSectionClasses";
import { type MetadataFilterActions } from "@/src/components/table/ValueCell";
@@ -31,6 +36,8 @@ export function DetailAttributesTab({
attributes: Record<string, unknown>;
attributesAnchorTime: Date;
modelParameters: Record<string, unknown> | null;
+ skillsAvailable?: SkillsAvailable;
+ skillsResourceLoaded?: SkillsResourceLoaded;
};
metadata: unknown;
/** Avoids re-parsing. */
@@ -48,6 +55,12 @@ export function DetailAttributesTab({
projectId,
filterTarget: observation ? "observations" : "traces",
};
+ const internalFeaturesEnabled = useInternalFeaturesEnabled();
+ const skillsAvailable = observation?.skillsAvailable ?? [];
+ const skillsResourceLoaded = observation?.skillsResourceLoaded ?? [];
+ const hasSkills =
+ internalFeaturesEnabled &&
+ (skillsAvailable.length > 0 || skillsResourceLoaded.length > 0);
const hasAttributes =
observation && Object.keys(observation.attributes).length > 0;
const hasMetadata =
@@ -120,7 +133,24 @@ export function DetailAttributesTab({
)}
</div>
) : null}
- {!hasAttributes && !observation?.modelParameters && !hasMetadata ? (
+ {hasSkills ? (
+ <>
+ <PrettyJsonView
+ title="Available skills"
+ json={skillsAvailable}
+ currentView={currentView}
+ />
+ <PrettyJsonView
+ title="Loaded resources"
+ json={skillsResourceLoaded}
+ currentView={currentView}
+ />
+ </>
+ ) : null}
+ {!hasSkills &&
+ !hasAttributes &&
+ !observation?.modelParameters &&
+ !hasMetadata ? (
<p className="text-muted-foreground text-base">
{observation
? "No attributes, model parameters or metadata on this observation."
diff --git a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx
index 91347f8e59..483b8243b3 100644
--- a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx
+++ b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx
@@ -553,6 +553,8 @@ export function ConnectedObservationDetailView({
attributes,
attributesAnchorTime: observation.startTime,
modelParameters,
+ skillsAvailable: observationWithIO?.skillsAvailable,
+ skillsResourceLoaded: observationWithIO?.skillsResourceLoaded,
}}
metadata={observationWithIOCompat.data?.metadata ?? undefined}
parsedMetadata={parsedMetadata}
diff --git a/web/src/features/traces/hooks/useParsedObservation.ts b/web/src/features/traces/hooks/useParsedObservation.ts
index e7fcf2d74c..6f55a8b2c3 100644
--- a/web/src/features/traces/hooks/useParsedObservation.ts
+++ b/web/src/features/traces/hooks/useParsedObservation.ts
@@ -75,12 +75,13 @@ export function useParsedObservation({
minStartTime: startTime ?? new Date(0),
maxStartTime: startTime ?? new Date(),
truncated: false,
+ includeSkills: true,
},
{
...sendAsPostOption,
enabled: isV4,
staleTime: 5 * 60 * 1000, // 5 minutes
- select: (data) => data[0], // Extract single result from batch
+ select: (data): EventBatchIOOutput | undefined => data[0],
},
);
@@ -91,6 +92,8 @@ export function useParsedObservation({
...baseObservation,
input: eventsQuery.data.input,
output: eventsQuery.data.output,
+ skillsAvailable: eventsQuery.data.skillsAvailable,
+ skillsResourceLoaded: eventsQuery.data.skillsResourceLoaded,
// Stringify metadata to match ObservationReturnTypeWithMetadata format
metadata: stringifyMetadata(eventsQuery.data.metadata),
} satisfies ObservationWithStringifiedIO;
diff --git a/worker/src/__tests__/nativeCodec.integration.test.ts b/worker/src/__tests__/nativeCodec.integration.test.ts
index 98a6f3f396..a37787d8ab 100644
--- a/worker/src/__tests__/nativeCodec.integration.test.ts
+++ b/worker/src/__tests__/nativeCodec.integration.test.ts
@@ -30,10 +30,6 @@ describe.runIf(
});
columns = await result.json<Column>();
expect(columns.length).toBeGreaterThan(0);
- const normalizeType = (type: string) =>
- type
- .replaceAll(/LowCardinality\(([^()]*)\)/g, "$1")
- .replaceAll(/\s/g, "");
// Names and types come from the same Rust declaration as JS reads and column writes.
// Check the full insertable set, including columns that captured traffic omits.
const liveColumns = columns.filter(
@@ -325,6 +321,31 @@ function preparedRowArbitrary(columns: Column[]): Arbitrary<Row> {
});
}
+// Native tuple values are positional; compare element types without their server-side names.
+function normalizeType(type: string): string {
+ const tokens = (
+ type.match(
+ /'(?:\\.|''|[^'\\])*'|`(?:\\.|``|[^`\\])*`|"(?:\\.|""|[^"\\])*"|[A-Za-z_]\w*|\s+|./gs,
+ ) ?? []
+ ).filter((token) => !/^\s+$/.test(token));
+ const parents: string[] = [];
+ return tokens
+ .filter((token, index) => {
+ const previous = tokens[index - 1];
+ const tupleElement =
+ parents.at(-1) === "Tuple" && (previous === "(" || previous === ",");
+ if (token === "(") parents.push(previous);
+ else if (token === ")") parents.pop();
+ return !(
+ tupleElement &&
+ /^(?:[A-Za-z_]\w*|`.*`|".*")$/s.test(token) &&
+ /^[A-Za-z_]\w*$/.test(tokens[index + 1] ?? "")
+ );
+ })
+ .join("")
+ .replaceAll(/LowCardinality\(([^()]*)\)/g, "$1");
+}
+
function columnValueArbitrary(type: string): Arbitrary<unknown> {
const normalized = type
.replaceAll(/LowCardinality\(([^()]*)\)/g, "$1")
@@ -333,6 +354,19 @@ function columnValueArbitrary(type: string): Arbitrary<unknown> {
fc.fullUnicodeString({ maxLength: 24 }),
fc.constantFrom("", "true"),
);
+ if (normalized.startsWith("Array(Tuple(")) {
+ return fc.array(
+ fc.record({
+ langfuseSkillId: fc.option(text, { nil: null }),
+ skillVersion: fc.option(fc.integer({ min: 1, max: 4_294_967_295 }), {
+ nil: null,
+ }),
+ skillName: text,
+ ...(normalized.includes("filePathString") ? { filePath: text } : {}),
+ }),
+ { maxLength: 4 },
+ );
+ }
const nullable = normalized.match(/^Nullable\((.*)\)$/);
if (nullable)
return fc.option(columnValueArbitrary(nullable[1]), { nil: null });
diff --git a/worker/src/features/database-read-stream/observation-stream.ts b/worker/src/features/database-read-stream/observation-stream.ts
index 41f02247ca..c838e0e8da 100644
--- a/worker/src/features/database-read-stream/observation-stream.ts
+++ b/worker/src/features/database-read-stream/observation-stream.ts
@@ -657,6 +657,8 @@ const getObservationStreamFromEvents = async (
release: _release,
bookmarked: _bookmarked,
public: _public,
+ skillsAvailable: _skillsAvailable,
+ skillsResourceLoaded: _skillsResourceLoaded,
userId,
traceName,
...observationFields
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 7fc90692df..0895572483 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -6,11 +6,14 @@ import {
InvalidRequestError,
LangfuseNotFoundError,
Model,
+ MAX_SKILLS_PER_OBSERVATION,
ObservationLevel,
PrismaClient,
Prompt,
safeJsonParse,
type JsonNested,
+ type SkillsAvailable,
+ type SkillsResourceLoaded,
} from "@langfuse/shared";
import {
ClickhouseClientType,
@@ -31,6 +34,7 @@ import {
ObservationRecordInsertType,
observationRecordReadSchema,
PromptService,
+ SkillCacheService,
QueueJobs,
recordIncrement,
ScoreEventType,
@@ -225,6 +229,7 @@ const immutableEntityKeys: {
export class IngestionService {
private promptService: PromptService;
+ private skillCacheService: SkillCacheService;
constructor(
private redis: Redis | Cluster,
@@ -233,6 +238,7 @@ export class IngestionService {
private clickhouseClient: ClickhouseClientType,
) {
this.promptService = new PromptService(prisma, redis);
+ this.skillCacheService = new SkillCacheService(prisma, redis);
}
public async mergeAndWrite(params: MergeAndWriteParams): Promise<void> {
@@ -335,8 +341,8 @@ export class IngestionService {
Object.keys(eventData.providedUsageDetails ?? {}).length > 0 ||
Object.keys(eventData.providedCostDetails ?? {}).length > 0;
- // Perform lookups for prompt and model/usage enrichment
- const [prompt, generationUsage] = await Promise.all([
+ // Enrich prompt, usage and skill references concurrently.
+ const [prompt, generationUsage, skillReferences] = await Promise.all([
// Lookup prompt by name and version
eventData.promptName && eventData.promptVersion
? this.promptService.getPrompt({
@@ -372,6 +378,12 @@ export class IngestionService {
},
})
: null,
+ this.resolveSkillReferences({
+ projectId: eventData.projectId,
+ type: eventData.type ?? "SPAN",
+ available: eventData.skillsAvailable,
+ loaded: eventData.skillsResourceLoaded,
+ }),
]);
const now = toClickhouseDateTime();
@@ -424,6 +436,8 @@ export class IngestionService {
prompt_id: prompt?.id || "",
prompt_name: eventData.promptName,
prompt_version: parseUInt16(eventData.promptVersion),
+ skills_available: skillReferences.skills_available,
+ skills_resource_loaded: skillReferences.skills_resource_loaded,
// Model
model_id: generationUsage?.internal_model_id || "",
@@ -1235,6 +1249,77 @@ export class IngestionService {
return parsedObservationRecord;
}
+ private async resolveSkillReferences({
+ projectId,
+ type,
+ available = [],
+ loaded = [],
+ }: {
+ projectId: string;
+ type: EventRecordInsertType["type"];
+ available?: SkillsAvailable;
+ loaded?: SkillsResourceLoaded;
+ }): Promise<{
+ skills_available: SkillsAvailable;
+ skills_resource_loaded: SkillsResourceLoaded;
+ }> {
+ // Bound internal callers too, before building the lookup or its cache key.
+ if (
+ available.length > MAX_SKILLS_PER_OBSERVATION ||
+ loaded.length > MAX_SKILLS_PER_OBSERVATION
+ ) {
+ logger.warn("Skill references truncated: item limit exceeded", {
+ projectId,
+ availableCount: available.length,
+ loadedCount: loaded.length,
+ limit: MAX_SKILLS_PER_OBSERVATION,
+ });
+ }
+ available = available.slice(0, MAX_SKILLS_PER_OBSERVATION);
+ loaded = loaded.slice(0, MAX_SKILLS_PER_OBSERVATION);
+ const versions = available.flatMap(({ skillName, skillVersion }) =>
+ skillVersion == null ? [] : [{ name: skillName, version: skillVersion }],
+ );
+ const skills = await this.skillCacheService.getReferences({
+ projectId,
+ versions,
+ });
+
+ const availableSkills = available.map(({ skillName, skillVersion }) => {
+ const skill = skills.find(
+ (skill) => skill.name === skillName && skill.version === skillVersion,
+ );
+ return {
+ skillName,
+ skillVersion: skill?.version ?? skillVersion,
+ langfuseSkillId: skill?.id,
+ };
+ });
+
+ const loadedResources = loaded.map(
+ ({ skillName, filePath, skillVersion }) => {
+ const candidates = availableSkills.filter(
+ (skill) => skill.skillName === skillName,
+ );
+ if (candidates.length !== 1)
+ return { skillName, filePath, skillVersion };
+
+ const skill = candidates[0];
+ return {
+ skillName,
+ filePath,
+ langfuseSkillId: skill?.langfuseSkillId,
+ skillVersion: skill.skillVersion ?? skillVersion,
+ };
+ },
+ );
+
+ return {
+ skills_available: type === "GENERATION" ? availableSkills : [],
+ skills_resource_loaded: loadedResources,
+ };
+ }
+
private mergeRecords<T extends InsertRecord>(
records: T[],
immutableEntityKeys: string[],
diff --git a/worker/src/services/IngestionService/prepareNativeEvent.ts b/worker/src/services/IngestionService/prepareNativeEvent.ts
index c4c9ef9a20..54b018cc79 100644
--- a/worker/src/services/IngestionService/prepareNativeEvent.ts
+++ b/worker/src/services/IngestionService/prepareNativeEvent.ts
@@ -3,7 +3,8 @@ import type { EventRecordInsertType } from "@langfuse/shared/src/server";
/**
* Snapshot a row after overflow handling and byte accounting into Rust. Serialize model parameters
- * for their ClickHouse String column; keep the caller's structured row available for evaluations.
+ * for their ClickHouse String column.
+ * Keep the caller's structured row available for evaluations.
*/
export function prepareNativeEvent(
record: Omit<EventRecordInsertType, "model_parameters"> & {