Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(skills): simplify skill load inference (#18863)

tobi_kl939e7fb1122c
diff --git a/packages/shared/src/server/ingestion/extractSkillsBackend.ts b/packages/shared/src/server/ingestion/extractSkillsBackend.ts
new file mode 100644
index 0000000000..ff5d1ffb8d
--- /dev/null
+++ b/packages/shared/src/server/ingestion/extractSkillsBackend.ts
@@ -0,0 +1,228 @@
+import {
+ MAX_SKILLS_PER_OBSERVATION,
+ SkillsAvailableSchema,
+ SkillsResourceLoadedSchema,
+ type SkillsAvailable,
+ type SkillsResourceLoaded,
+} from "../../features/skills/trace";
+import {
+ MAX_SKILL_CATALOG_JSON_BYTES,
+ MAX_SKILL_ERROR_JSON_BYTES,
+ MAX_SKILL_PATH_LENGTH,
+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ SKILL_RESOURCE_TOOL_NAMES,
+} from "../../features/skills/constants";
+import { ObservationType } from "../../domain/observations";
+import { logger } from "../logger";
+
+export function extractSkillsFromObservation({
+ skillsAvailable: manifest,
+ observationType,
+ name,
+ input,
+ output,
+ isError,
+ projectId,
+}: {
+ skillsAvailable: unknown;
+ observationType: string;
+ name: string;
+ input: unknown;
+ output: unknown;
+ isError: boolean;
+ projectId: string;
+}): {
+ skillsAvailable?: SkillsAvailable;
+ skillsResourceLoaded?: SkillsResourceLoaded;
+} {
+ // Skill inference is enabled only when an availability attribute is reported.
+ if (!manifest) return {};
+ if (
+ observationType !== ObservationType.GENERATION &&
+ (observationType !== ObservationType.TOOL ||
+ !SKILL_RESOURCE_TOOL_NAMES.has(name) ||
+ output == null ||
+ isError)
+ ) {
+ return {};
+ }
+
+ let parsedManifest = Array.isArray(manifest)
+ ? manifest
+ : parseJsonPayloadWithMaxSize(
+ manifest,
+ MAX_SKILL_CATALOG_JSON_BYTES,
+ "skillsAvailable",
+ projectId,
+ );
+ if (
+ Array.isArray(parsedManifest) &&
+ parsedManifest.length > MAX_SKILLS_PER_OBSERVATION
+ ) {
+ logger.warn("OTEL skill catalog dropped: item limit exceeded", {
+ projectId,
+ count: parsedManifest.length,
+ limit: MAX_SKILLS_PER_OBSERVATION,
+ });
+ parsedManifest = undefined;
+ }
+ const available = Array.isArray(parsedManifest)
+ ? SkillsAvailableSchema.safeParse(parsedManifest)
+ : undefined;
+ if (observationType === ObservationType.GENERATION) {
+ if (available?.success) return { skillsAvailable: available.data };
+ return {};
+ }
+
+ const candidate = extractSkillResource(
+ name,
+ input,
+ output,
+ available?.success ? available.data : [],
+ projectId,
+ );
+ if (!candidate) return {};
+ if (
+ typeof candidate.filePath === "string" &&
+ candidate.filePath.length > MAX_SKILL_PATH_LENGTH
+ ) {
+ logger.warn(
+ "OTEL skill resource load dropped: path length limit exceeded",
+ {
+ projectId,
+ name,
+ length: candidate.filePath.length,
+ limit: MAX_SKILL_PATH_LENGTH,
+ },
+ );
+ return {};
+ }
+ const loaded = SkillsResourceLoadedSchema.element.safeParse(candidate);
+ if (!loaded.success) return {};
+
+ return {
+ ...(available?.success ? { skillsAvailable: available.data } : {}),
+ skillsResourceLoaded: [
+ { skillName: loaded.data.skillName, filePath: loaded.data.filePath },
+ ],
+ };
+}
+
+function extractSkillResource(
+ toolName: string,
+ input: unknown,
+ output: unknown,
+ available: SkillsAvailable,
+ projectId: string,
+) {
+ const args = parseJsonPayloadWithMaxSize(
+ input,
+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ "toolInput",
+ projectId,
+ );
+ if (!isPlainObject(args)) return;
+ const oversizedPath = [args.path, args.file_path].find(
+ (path) =>
+ typeof path === "string" && path.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ );
+ if (typeof oversizedPath === "string") {
+ logger.warn(
+ "OTEL skill resource load dropped: input path length limit exceeded",
+ {
+ projectId,
+ toolName,
+ length: oversizedPath.length,
+ limit: MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ },
+ );
+ return;
+ }
+
+ switch (toolName) {
+ case "skill":
+ case "Skill":
+ case "loadSkill":
+ if (isFailedSkillActivation(output, projectId)) return;
+ return {
+ skillName: args.name ?? args.skill,
+ filePath: "SKILL.md",
+ };
+ case "skill_read":
+ return {
+ skillName: args.skillName,
+ filePath: args.path,
+ };
+ case "readFile":
+ case "Read":
+ return extractSkillResourceFromPath(
+ args.path ?? args.file_path,
+ available,
+ );
+ default:
+ return;
+ }
+}
+
+function extractSkillResourceFromPath(
+ path: unknown,
+ available: SkillsAvailable,
+) {
+ if (typeof path !== "string") return;
+ const segments = path.split("/");
+ const matches = segments.filter((segment) =>
+ available.some((skill) => skill.skillName === segment),
+ );
+ if (matches.length !== 1) return;
+ const skillName = matches[0];
+ return {
+ skillName,
+ filePath: segments.slice(segments.indexOf(skillName) + 1).join("/"),
+ };
+}
+
+function isFailedSkillActivation(output: unknown, projectId: string): boolean {
+ const result = parseJsonPayloadWithMaxSize(
+ output,
+ MAX_SKILL_ERROR_JSON_BYTES,
+ "toolOutput",
+ projectId,
+ );
+ return (
+ isPlainObject(result) &&
+ (result.isError === true || result.error !== undefined)
+ );
+}
+
+function parseJsonPayloadWithMaxSize(
+ payload: unknown,
+ maxBytes: number,
+ field: "skillsAvailable" | "toolInput" | "toolOutput",
+ projectId: string,
+): unknown {
+ if (
+ typeof payload === "string" &&
+ (payload.length > maxBytes || Buffer.byteLength(payload, "utf8") > maxBytes)
+ ) {
+ logger.warn("OTEL skill payload parsing skipped: byte limit exceeded", {
+ projectId,
+ field,
+ limitBytes: maxBytes,
+ stringLength: payload.length,
+ });
+ return undefined;
+ }
+ if (payload && typeof payload === "object") return payload;
+ if (typeof payload === "string") {
+ try {
+ return JSON.parse(payload);
+ } catch {
+ return undefined;
+ }
+ }
+ return undefined;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index bad1965664..09c351cb06 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -1,20 +1,6 @@
/* eslint-disable no-nested-ternary */
import { randomUUID } from "crypto";
-import {
- MAX_SKILLS_PER_OBSERVATION,
- SkillsAvailableSchema,
- SkillsResourceLoadedSchema,
- type SkillsAvailable,
- type SkillsResourceLoaded,
-} from "../../features/skills/trace";
-import {
- MAX_SKILL_CATALOG_JSON_BYTES,
- MAX_SKILL_ERROR_JSON_BYTES,
- MAX_SKILL_PATH_LENGTH,
- MAX_SKILL_TOOL_ARGS_JSON_BYTES,
- SKILL_RESOURCE_TOOL_NAMES,
-} from "../../features/skills/constants";
-
+import { extractSkillsFromObservation } from "../ingestion/extractSkillsBackend";
import {
ForbiddenError,
ObservationLevel,
@@ -597,18 +583,33 @@ export class OtelIngestionProcessor {
const canLinkPrompt =
observationType === ObservationType.GENERATION;
+ const name = this.extractName(
+ span.name,
+ spanAttributes,
+ observationType,
+ scopeSpan.scope?.name,
+ );
+
+ const level =
+ parseObservationLevel(
+ spanAttributes[
+ LangfuseOtelSpanAttributes.OBSERVATION_LEVEL
+ ],
+ ) ??
+ (span.status?.code === 2
+ ? ObservationLevel.ERROR
+ : scopeSpan?.scope?.name === "livekit-agents" &&
+ LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)
+ ? ObservationLevel.DEBUG
+ : ObservationLevel.DEFAULT);
+
events.push({
projectId: this.projectId,
traceId,
spanId,
parentSpanId,
- name: this.extractName(
- span.name,
- spanAttributes,
- observationType,
- scopeSpan.scope?.name,
- ),
+ name,
type: observationType,
environment: this.extractEnvironment(
spanAttributes,
@@ -622,18 +623,7 @@ export class OtelIngestionProcessor {
startTimeISO,
endTimeISO,
- level:
- parseObservationLevel(
- spanAttributes[
- LangfuseOtelSpanAttributes.OBSERVATION_LEVEL
- ],
- ) ??
- (span.status?.code === 2
- ? ObservationLevel.ERROR
- : scopeSpan?.scope?.name === "livekit-agents" &&
- LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)
- ? ObservationLevel.DEBUG
- : ObservationLevel.DEFAULT),
+ level,
statusMessage:
spanAttributes[
LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE
@@ -641,14 +631,18 @@ export class OtelIngestionProcessor {
span.status?.message ??
null,
- ...this.extractSkills(
- spanAttributes,
+ ...extractSkillsFromObservation({
+ skillsAvailable:
+ spanAttributes[
+ LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE
+ ],
observationType,
- this.extractName(span.name, spanAttributes),
+ name,
input,
output,
- span.status?.code,
- ),
+ isError: level === ObservationLevel.ERROR,
+ projectId: this.projectId,
+ }),
promptName: canLinkPrompt
? (spanAttributes?.[
LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME
@@ -1331,16 +1325,28 @@ export class OtelIngestionProcessor {
// Prompts can only be linked to GENERATION observations
const canLinkPrompt = mappedObservationType === ObservationType.GENERATION;
+ const name = this.extractName(
+ span.name,
+ attributes,
+ mappedObservationType,
+ instrumentationScopeName,
+ );
+ const level =
+ parseObservationLevel(
+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],
+ ) ??
+ (span.status?.code === 2
+ ? ObservationLevel.ERROR
+ : scopeSpan?.scope?.name === "livekit-agents" &&
+ LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)
+ ? ObservationLevel.DEBUG
+ : ObservationLevel.DEFAULT);
+
const observation = {
id: this.parseId(span.spanId?.data ?? span.spanId),
traceId,
parentObservationId,
- name: this.extractName(
- span.name,
- attributes,
- mappedObservationType,
- instrumentationScopeName,
- ),
+ name,
startTime: startTimeISO,
endTime: endTimeISO,
environment: this.extractEnvironment(attributes, resourceAttributes),
@@ -1349,16 +1355,7 @@ export class OtelIngestionProcessor {
startTimeISO,
),
metadata: normalizedToolMetadata.metadata,
- level:
- parseObservationLevel(
- attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],
- ) ??
- (span.status?.code === 2
- ? ObservationLevel.ERROR
- : scopeSpan?.scope?.name === "livekit-agents" &&
- LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)
- ? ObservationLevel.DEBUG
- : ObservationLevel.DEFAULT),
+ level,
statusMessage:
attributes[LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE] ??
span.status?.message ??
@@ -1374,14 +1371,16 @@ export class OtelIngestionProcessor {
model: skipModelUsageAndCost
? undefined
: this.extractModelName(attributes),
- ...this.extractSkills(
- attributes,
- mappedObservationType,
- this.extractName(span.name, attributes),
+ ...extractSkillsFromObservation({
+ skillsAvailable:
+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE],
+ observationType: mappedObservationType,
+ name,
input,
output,
- span.status?.code,
- ),
+ isError: level === ObservationLevel.ERROR,
+ projectId: this.projectId,
+ }),
promptName: canLinkPrompt
? (attributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME] ??
attributes["langfuse.prompt.name"] ??
@@ -3935,203 +3934,6 @@ export class OtelIngestionProcessor {
}
}
- private extractSkills(
- attributes: Record<string, unknown>,
- observationType: string,
- toolName: string,
- input: unknown,
- output: unknown,
- statusCode: number | undefined,
- ): {
- skillsAvailable?: SkillsAvailable;
- skillsResourceLoaded?: SkillsResourceLoaded;
- } {
- const manifest =
- attributes[LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE];
- // Skill inference is enabled only when an availability attribute is reported.
- if (!manifest) return {};
- if (
- observationType !== ObservationType.GENERATION &&
- (observationType !== ObservationType.TOOL ||
- !SKILL_RESOURCE_TOOL_NAMES.has(toolName) ||
- output == null ||
- statusCode === 2 ||
- parseObservationLevel(
- attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],
- ) === ObservationLevel.ERROR)
- ) {
- return {};
- }
-
- let parsedManifest = Array.isArray(manifest)
- ? manifest
- : this.parseJsonPayloadWithMaxSize(
- manifest,
- MAX_SKILL_CATALOG_JSON_BYTES,
- "skillsAvailable",
- );
- if (
- Array.isArray(parsedManifest) &&
- parsedManifest.length > MAX_SKILLS_PER_OBSERVATION
- ) {
- logger.warn("OTEL skill catalog dropped: item limit exceeded", {
- projectId: this.projectId,
- count: parsedManifest.length,
- limit: MAX_SKILLS_PER_OBSERVATION,
- });
- parsedManifest = undefined;
- }
- const available = Array.isArray(parsedManifest)
- ? SkillsAvailableSchema.safeParse(parsedManifest)
- : undefined;
- if (observationType === ObservationType.GENERATION) {
- if (available?.success) return { skillsAvailable: available.data };
- return {};
- }
-
- const candidate = this.extractSkillResource(
- toolName,
- input,
- output,
- available?.success ? available.data : [],
- );
- if (!candidate) return {};
- if (
- typeof candidate.filePath === "string" &&
- candidate.filePath.length > MAX_SKILL_PATH_LENGTH
- ) {
- logger.warn(
- "OTEL skill resource load dropped: path length limit exceeded",
- {
- projectId: this.projectId,
- toolName,
- length: candidate.filePath.length,
- limit: MAX_SKILL_PATH_LENGTH,
- },
- );
- return {};
- }
- const loaded = SkillsResourceLoadedSchema.element.safeParse(candidate);
- if (!loaded.success) return {};
-
- return {
- ...(available?.success ? { skillsAvailable: available.data } : {}),
- skillsResourceLoaded: [
- { skillName: loaded.data.skillName, filePath: loaded.data.filePath },
- ],
- };
- }
-
- private extractSkillResource(
- toolName: string,
- input: unknown,
- output: unknown,
- available: SkillsAvailable,
- ) {
- const args = this.parseJsonPayloadWithMaxSize(
- input,
- MAX_SKILL_TOOL_ARGS_JSON_BYTES,
- "toolInput",
- );
- if (!OtelIngestionProcessor.isPlainObject(args)) return;
- const oversizedPath = [args.path, args.file_path].find(
- (path) =>
- typeof path === "string" &&
- path.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES,
- );
- if (typeof oversizedPath === "string") {
- logger.warn(
- "OTEL skill resource load dropped: input path length limit exceeded",
- {
- projectId: this.projectId,
- toolName,
- length: oversizedPath.length,
- limit: MAX_SKILL_TOOL_ARGS_JSON_BYTES,
- },
- );
- return;
- }
-
- switch (toolName) {
- case "skill":
- case "Skill":
- case "loadSkill":
- if (this.isFailedSkillActivation(output)) return;
- return {
- skillName: args.name ?? args.skill,
- filePath: "SKILL.md",
- };
- case "skill_read":
- return {
- skillName: args.skillName,
- filePath: args.path,
- };
- case "readFile":
- case "Read":
- return this.extractSkillResourceFromPath(
- args.path ?? args.file_path,
- available,
- );
- default:
- return;
- }
- }
-
- private extractSkillResourceFromPath(
- path: unknown,
- available: SkillsAvailable,
- ) {
- if (typeof path !== "string") return;
- const segments = path.split("/");
- const matches = segments.filter((segment) =>
- available.some((skill) => skill.skillName === segment),
- );
- if (matches.length !== 1) return;
- const skillName = matches[0];
- return {
- skillName,
- filePath: segments.slice(segments.indexOf(skillName) + 1).join("/"),
- };
- }
-
- private isFailedSkillActivation(output: unknown): boolean {
- const result = this.parseJsonPayloadWithMaxSize(
- output,
- MAX_SKILL_ERROR_JSON_BYTES,
- "toolOutput",
- );
- const text = typeof result === "string" ? result : output;
- return (
- (OtelIngestionProcessor.isPlainObject(result) &&
- (result.isError === true || result.error !== undefined)) ||
- (typeof text === "string" &&
- /^(?:Skill|File) ".*" not found/.test(
- text.slice(0, MAX_SKILL_ERROR_JSON_BYTES),
- ))
- );
- }
-
- private parseJsonPayloadWithMaxSize(
- payload: unknown,
- maxBytes: number,
- field: "skillsAvailable" | "toolInput" | "toolOutput",
- ): unknown {
- if (
- typeof payload === "string" &&
- (payload.length > maxBytes ||
- Buffer.byteLength(payload, "utf8") > maxBytes)
- ) {
- logger.warn("OTEL skill payload parsing skipped: byte limit exceeded", {
- projectId: this.projectId,
- field,
- limitBytes: maxBytes,
- stringLength: payload.length,
- });
- return undefined;
- }
- return this.parseJsonPayload(payload);
- }
-
/**
* Parses a payload (object or JSON string) and returns the object
* or `undefined` if invalid.