🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(skills): simplify skill load inference (#18863)
tobi_kl939e7fb1122c
diff --git a/packages/shared/src/server/ingestion/extractSkillsBackend.ts b/packages/shared/src/server/ingestion/extractSkillsBackend.tsnew file mode 100644index 0000000000..ff5d1ffb8d--- /dev/null+++ b/packages/shared/src/server/ingestion/extractSkillsBackend.ts@@ -0,0 +1,228 @@+import {+ MAX_SKILLS_PER_OBSERVATION,+ SkillsAvailableSchema,+ SkillsResourceLoadedSchema,+ type SkillsAvailable,+ type SkillsResourceLoaded,+} from "../../features/skills/trace";+import {+ MAX_SKILL_CATALOG_JSON_BYTES,+ MAX_SKILL_ERROR_JSON_BYTES,+ MAX_SKILL_PATH_LENGTH,+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ SKILL_RESOURCE_TOOL_NAMES,+} from "../../features/skills/constants";+import { ObservationType } from "../../domain/observations";+import { logger } from "../logger";++export function extractSkillsFromObservation({+ skillsAvailable: manifest,+ observationType,+ name,+ input,+ output,+ isError,+ projectId,+}: {+ skillsAvailable: unknown;+ observationType: string;+ name: string;+ input: unknown;+ output: unknown;+ isError: boolean;+ projectId: string;+}): {+ skillsAvailable?: SkillsAvailable;+ skillsResourceLoaded?: SkillsResourceLoaded;+} {+ // Skill inference is enabled only when an availability attribute is reported.+ if (!manifest) return {};+ if (+ observationType !== ObservationType.GENERATION &&+ (observationType !== ObservationType.TOOL ||+ !SKILL_RESOURCE_TOOL_NAMES.has(name) ||+ output == null ||+ isError)+ ) {+ return {};+ }++ let parsedManifest = Array.isArray(manifest)+ ? manifest+ : parseJsonPayloadWithMaxSize(+ manifest,+ MAX_SKILL_CATALOG_JSON_BYTES,+ "skillsAvailable",+ projectId,+ );+ if (+ Array.isArray(parsedManifest) &&+ parsedManifest.length > MAX_SKILLS_PER_OBSERVATION+ ) {+ logger.warn("OTEL skill catalog dropped: item limit exceeded", {+ projectId,+ count: parsedManifest.length,+ limit: MAX_SKILLS_PER_OBSERVATION,+ });+ parsedManifest = undefined;+ }+ const available = Array.isArray(parsedManifest)+ ? SkillsAvailableSchema.safeParse(parsedManifest)+ : undefined;+ if (observationType === ObservationType.GENERATION) {+ if (available?.success) return { skillsAvailable: available.data };+ return {};+ }++ const candidate = extractSkillResource(+ name,+ input,+ output,+ available?.success ? available.data : [],+ projectId,+ );+ if (!candidate) return {};+ if (+ typeof candidate.filePath === "string" &&+ candidate.filePath.length > MAX_SKILL_PATH_LENGTH+ ) {+ logger.warn(+ "OTEL skill resource load dropped: path length limit exceeded",+ {+ projectId,+ name,+ length: candidate.filePath.length,+ limit: MAX_SKILL_PATH_LENGTH,+ },+ );+ return {};+ }+ const loaded = SkillsResourceLoadedSchema.element.safeParse(candidate);+ if (!loaded.success) return {};++ return {+ ...(available?.success ? { skillsAvailable: available.data } : {}),+ skillsResourceLoaded: [+ { skillName: loaded.data.skillName, filePath: loaded.data.filePath },+ ],+ };+}++function extractSkillResource(+ toolName: string,+ input: unknown,+ output: unknown,+ available: SkillsAvailable,+ projectId: string,+) {+ const args = parseJsonPayloadWithMaxSize(+ input,+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ "toolInput",+ projectId,+ );+ if (!isPlainObject(args)) return;+ const oversizedPath = [args.path, args.file_path].find(+ (path) =>+ typeof path === "string" && path.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ );+ if (typeof oversizedPath === "string") {+ logger.warn(+ "OTEL skill resource load dropped: input path length limit exceeded",+ {+ projectId,+ toolName,+ length: oversizedPath.length,+ limit: MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ },+ );+ return;+ }++ switch (toolName) {+ case "skill":+ case "Skill":+ case "loadSkill":+ if (isFailedSkillActivation(output, projectId)) return;+ return {+ skillName: args.name ?? args.skill,+ filePath: "SKILL.md",+ };+ case "skill_read":+ return {+ skillName: args.skillName,+ filePath: args.path,+ };+ case "readFile":+ case "Read":+ return extractSkillResourceFromPath(+ args.path ?? args.file_path,+ available,+ );+ default:+ return;+ }+}++function extractSkillResourceFromPath(+ path: unknown,+ available: SkillsAvailable,+) {+ if (typeof path !== "string") return;+ const segments = path.split("/");+ const matches = segments.filter((segment) =>+ available.some((skill) => skill.skillName === segment),+ );+ if (matches.length !== 1) return;+ const skillName = matches[0];+ return {+ skillName,+ filePath: segments.slice(segments.indexOf(skillName) + 1).join("/"),+ };+}++function isFailedSkillActivation(output: unknown, projectId: string): boolean {+ const result = parseJsonPayloadWithMaxSize(+ output,+ MAX_SKILL_ERROR_JSON_BYTES,+ "toolOutput",+ projectId,+ );+ return (+ isPlainObject(result) &&+ (result.isError === true || result.error !== undefined)+ );+}++function parseJsonPayloadWithMaxSize(+ payload: unknown,+ maxBytes: number,+ field: "skillsAvailable" | "toolInput" | "toolOutput",+ projectId: string,+): unknown {+ if (+ typeof payload === "string" &&+ (payload.length > maxBytes || Buffer.byteLength(payload, "utf8") > maxBytes)+ ) {+ logger.warn("OTEL skill payload parsing skipped: byte limit exceeded", {+ projectId,+ field,+ limitBytes: maxBytes,+ stringLength: payload.length,+ });+ return undefined;+ }+ if (payload && typeof payload === "object") return payload;+ if (typeof payload === "string") {+ try {+ return JSON.parse(payload);+ } catch {+ return undefined;+ }+ }+ return undefined;+}++function isPlainObject(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex bad1965664..09c351cb06 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -1,20 +1,6 @@/* eslint-disable no-nested-ternary */import { randomUUID } from "crypto";-import {- MAX_SKILLS_PER_OBSERVATION,- SkillsAvailableSchema,- SkillsResourceLoadedSchema,- type SkillsAvailable,- type SkillsResourceLoaded,-} from "../../features/skills/trace";-import {- MAX_SKILL_CATALOG_JSON_BYTES,- MAX_SKILL_ERROR_JSON_BYTES,- MAX_SKILL_PATH_LENGTH,- MAX_SKILL_TOOL_ARGS_JSON_BYTES,- SKILL_RESOURCE_TOOL_NAMES,-} from "../../features/skills/constants";-+import { extractSkillsFromObservation } from "../ingestion/extractSkillsBackend";import {ForbiddenError,ObservationLevel,@@ -597,18 +583,33 @@ export class OtelIngestionProcessor {const canLinkPrompt =observationType === ObservationType.GENERATION;+ const name = this.extractName(+ span.name,+ spanAttributes,+ observationType,+ scopeSpan.scope?.name,+ );++ const level =+ parseObservationLevel(+ spanAttributes[+ LangfuseOtelSpanAttributes.OBSERVATION_LEVEL+ ],+ ) ??+ (span.status?.code === 2+ ? ObservationLevel.ERROR+ : scopeSpan?.scope?.name === "livekit-agents" &&+ LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)+ ? ObservationLevel.DEBUG+ : ObservationLevel.DEFAULT);+events.push({projectId: this.projectId,traceId,spanId,parentSpanId,- name: this.extractName(- span.name,- spanAttributes,- observationType,- scopeSpan.scope?.name,- ),+ name,type: observationType,environment: this.extractEnvironment(spanAttributes,@@ -622,18 +623,7 @@ export class OtelIngestionProcessor {startTimeISO,endTimeISO,- level:- parseObservationLevel(- spanAttributes[- LangfuseOtelSpanAttributes.OBSERVATION_LEVEL- ],- ) ??- (span.status?.code === 2- ? ObservationLevel.ERROR- : scopeSpan?.scope?.name === "livekit-agents" &&- LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)- ? ObservationLevel.DEBUG- : ObservationLevel.DEFAULT),+ level,statusMessage:spanAttributes[LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE@@ -641,14 +631,18 @@ export class OtelIngestionProcessor {span.status?.message ??null,- ...this.extractSkills(- spanAttributes,+ ...extractSkillsFromObservation({+ skillsAvailable:+ spanAttributes[+ LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE+ ],observationType,- this.extractName(span.name, spanAttributes),+ name,input,output,- span.status?.code,- ),+ isError: level === ObservationLevel.ERROR,+ projectId: this.projectId,+ }),promptName: canLinkPrompt? (spanAttributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME@@ -1331,16 +1325,28 @@ export class OtelIngestionProcessor {// Prompts can only be linked to GENERATION observationsconst canLinkPrompt = mappedObservationType === ObservationType.GENERATION;+ const name = this.extractName(+ span.name,+ attributes,+ mappedObservationType,+ instrumentationScopeName,+ );+ const level =+ parseObservationLevel(+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],+ ) ??+ (span.status?.code === 2+ ? ObservationLevel.ERROR+ : scopeSpan?.scope?.name === "livekit-agents" &&+ LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)+ ? ObservationLevel.DEBUG+ : ObservationLevel.DEFAULT);+const observation = {id: this.parseId(span.spanId?.data ?? span.spanId),traceId,parentObservationId,- name: this.extractName(- span.name,- attributes,- mappedObservationType,- instrumentationScopeName,- ),+ name,startTime: startTimeISO,endTime: endTimeISO,environment: this.extractEnvironment(attributes, resourceAttributes),@@ -1349,16 +1355,7 @@ export class OtelIngestionProcessor {startTimeISO,),metadata: normalizedToolMetadata.metadata,- level:- parseObservationLevel(- attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],- ) ??- (span.status?.code === 2- ? ObservationLevel.ERROR- : scopeSpan?.scope?.name === "livekit-agents" &&- LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)- ? ObservationLevel.DEBUG- : ObservationLevel.DEFAULT),+ level,statusMessage:attributes[LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE] ??span.status?.message ??@@ -1374,14 +1371,16 @@ export class OtelIngestionProcessor {model: skipModelUsageAndCost? undefined: this.extractModelName(attributes),- ...this.extractSkills(- attributes,- mappedObservationType,- this.extractName(span.name, attributes),+ ...extractSkillsFromObservation({+ skillsAvailable:+ attributes[LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE],+ observationType: mappedObservationType,+ name,input,output,- span.status?.code,- ),+ isError: level === ObservationLevel.ERROR,+ projectId: this.projectId,+ }),promptName: canLinkPrompt? (attributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME] ??attributes["langfuse.prompt.name"] ??@@ -3935,203 +3934,6 @@ export class OtelIngestionProcessor {}}- private extractSkills(- attributes: Record<string, unknown>,- observationType: string,- toolName: string,- input: unknown,- output: unknown,- statusCode: number | undefined,- ): {- skillsAvailable?: SkillsAvailable;- skillsResourceLoaded?: SkillsResourceLoaded;- } {- const manifest =- attributes[LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE];- // Skill inference is enabled only when an availability attribute is reported.- if (!manifest) return {};- if (- observationType !== ObservationType.GENERATION &&- (observationType !== ObservationType.TOOL ||- !SKILL_RESOURCE_TOOL_NAMES.has(toolName) ||- output == null ||- statusCode === 2 ||- parseObservationLevel(- attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],- ) === ObservationLevel.ERROR)- ) {- return {};- }-- let parsedManifest = Array.isArray(manifest)- ? manifest- : this.parseJsonPayloadWithMaxSize(- manifest,- MAX_SKILL_CATALOG_JSON_BYTES,- "skillsAvailable",- );- if (- Array.isArray(parsedManifest) &&- parsedManifest.length > MAX_SKILLS_PER_OBSERVATION- ) {- logger.warn("OTEL skill catalog dropped: item limit exceeded", {- projectId: this.projectId,- count: parsedManifest.length,- limit: MAX_SKILLS_PER_OBSERVATION,- });- parsedManifest = undefined;- }- const available = Array.isArray(parsedManifest)- ? SkillsAvailableSchema.safeParse(parsedManifest)- : undefined;- if (observationType === ObservationType.GENERATION) {- if (available?.success) return { skillsAvailable: available.data };- return {};- }-- const candidate = this.extractSkillResource(- toolName,- input,- output,- available?.success ? available.data : [],- );- if (!candidate) return {};- if (- typeof candidate.filePath === "string" &&- candidate.filePath.length > MAX_SKILL_PATH_LENGTH- ) {- logger.warn(- "OTEL skill resource load dropped: path length limit exceeded",- {- projectId: this.projectId,- toolName,- length: candidate.filePath.length,- limit: MAX_SKILL_PATH_LENGTH,- },- );- return {};- }- const loaded = SkillsResourceLoadedSchema.element.safeParse(candidate);- if (!loaded.success) return {};-- return {- ...(available?.success ? { skillsAvailable: available.data } : {}),- skillsResourceLoaded: [- { skillName: loaded.data.skillName, filePath: loaded.data.filePath },- ],- };- }-- private extractSkillResource(- toolName: string,- input: unknown,- output: unknown,- available: SkillsAvailable,- ) {- const args = this.parseJsonPayloadWithMaxSize(- input,- MAX_SKILL_TOOL_ARGS_JSON_BYTES,- "toolInput",- );- if (!OtelIngestionProcessor.isPlainObject(args)) return;- const oversizedPath = [args.path, args.file_path].find(- (path) =>- typeof path === "string" &&- path.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES,- );- if (typeof oversizedPath === "string") {- logger.warn(- "OTEL skill resource load dropped: input path length limit exceeded",- {- projectId: this.projectId,- toolName,- length: oversizedPath.length,- limit: MAX_SKILL_TOOL_ARGS_JSON_BYTES,- },- );- return;- }-- switch (toolName) {- case "skill":- case "Skill":- case "loadSkill":- if (this.isFailedSkillActivation(output)) return;- return {- skillName: args.name ?? args.skill,- filePath: "SKILL.md",- };- case "skill_read":- return {- skillName: args.skillName,- filePath: args.path,- };- case "readFile":- case "Read":- return this.extractSkillResourceFromPath(- args.path ?? args.file_path,- available,- );- default:- return;- }- }-- private extractSkillResourceFromPath(- path: unknown,- available: SkillsAvailable,- ) {- if (typeof path !== "string") return;- const segments = path.split("/");- const matches = segments.filter((segment) =>- available.some((skill) => skill.skillName === segment),- );- if (matches.length !== 1) return;- const skillName = matches[0];- return {- skillName,- filePath: segments.slice(segments.indexOf(skillName) + 1).join("/"),- };- }-- private isFailedSkillActivation(output: unknown): boolean {- const result = this.parseJsonPayloadWithMaxSize(- output,- MAX_SKILL_ERROR_JSON_BYTES,- "toolOutput",- );- const text = typeof result === "string" ? result : output;- return (- (OtelIngestionProcessor.isPlainObject(result) &&- (result.isError === true || result.error !== undefined)) ||- (typeof text === "string" &&- /^(?:Skill|File) ".*" not found/.test(- text.slice(0, MAX_SKILL_ERROR_JSON_BYTES),- ))- );- }-- private parseJsonPayloadWithMaxSize(- payload: unknown,- maxBytes: number,- field: "skillsAvailable" | "toolInput" | "toolOutput",- ): unknown {- if (- typeof payload === "string" &&- (payload.length > maxBytes ||- Buffer.byteLength(payload, "utf8") > maxBytes)- ) {- logger.warn("OTEL skill payload parsing skipped: byte limit exceeded", {- projectId: this.projectId,- field,- limitBytes: maxBytes,- stringLength: payload.length,- });- return undefined;- }- return this.parseJsonPayload(payload);- }-/*** Parses a payload (object or JSON string) and returns the object* or `undefined` if invalid.