🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(ingestion): accept name/version skill catalogs (#19087)
tobi_kle653b99d4b70
diff --git a/packages/shared/src/features/skills/trace.ts b/packages/shared/src/features/skills/trace.tsindex 4afe748509..0a7b37b0f8 100644--- a/packages/shared/src/features/skills/trace.ts+++ b/packages/shared/src/features/skills/trace.ts@@ -13,6 +13,26 @@ export const SkillsAvailableSchema = z).max(MAX_SKILLS_PER_OBSERVATION);+export const SkillsAvailableInputSchema = z+ .array(+ z.union([+ z.object({+ name: SkillsAvailableSchema.element.shape.skillName,+ version: SkillsAvailableSchema.element.shape.skillVersion,+ }),+ z+ .object({+ skillName: SkillsAvailableSchema.element.shape.skillName,+ skillVersion: SkillsAvailableSchema.element.shape.skillVersion,+ })+ .transform(({ skillName, skillVersion }) => ({+ name: skillName,+ version: skillVersion,+ })),+ ]),+ )+ .max(MAX_SKILLS_PER_OBSERVATION);+export const SkillsResourceLoadedSchema = z.array(z.object({@@ -24,5 +44,6 @@ export const SkillsResourceLoadedSchema = z).max(MAX_SKILLS_PER_OBSERVATION);+export type SkillsAvailableInput = z.infer<typeof SkillsAvailableInputSchema>;export type SkillsAvailable = z.infer<typeof SkillsAvailableSchema>;export type SkillsResourceLoaded = z.infer<typeof SkillsResourceLoadedSchema>;diff --git a/packages/shared/src/server/ingestion/extractSkillsBackend.ts b/packages/shared/src/server/ingestion/extractSkillsBackend.tsindex ff5d1ffb8d..4912f230bf 100644--- a/packages/shared/src/server/ingestion/extractSkillsBackend.ts+++ b/packages/shared/src/server/ingestion/extractSkillsBackend.ts@@ -1,8 +1,8 @@import {MAX_SKILLS_PER_OBSERVATION,- SkillsAvailableSchema,+ SkillsAvailableInputSchema,SkillsResourceLoadedSchema,- type SkillsAvailable,+ type SkillsAvailableInput,type SkillsResourceLoaded,} from "../../features/skills/trace";import {@@ -32,7 +32,7 @@ export function extractSkillsFromObservation({isError: boolean;projectId: string;}): {- skillsAvailable?: SkillsAvailable;+ skillsAvailable?: SkillsAvailableInput;skillsResourceLoaded?: SkillsResourceLoaded;} {// Skill inference is enabled only when an availability attribute is reported.@@ -67,7 +67,7 @@ export function extractSkillsFromObservation({parsedManifest = undefined;}const available = Array.isArray(parsedManifest)- ? SkillsAvailableSchema.safeParse(parsedManifest)+ ? SkillsAvailableInputSchema.safeParse(parsedManifest): undefined;if (observationType === ObservationType.GENERATION) {if (available?.success) return { skillsAvailable: available.data };@@ -112,7 +112,7 @@ function extractSkillResource(toolName: string,input: unknown,output: unknown,- available: SkillsAvailable,+ available: SkillsAvailableInput,projectId: string,) {const args = parseJsonPayloadWithMaxSize(@@ -166,12 +166,12 @@ function extractSkillResource(function extractSkillResourceFromPath(path: unknown,- available: SkillsAvailable,+ available: SkillsAvailableInput,) {if (typeof path !== "string") return;const segments = path.split("/");const matches = segments.filter((segment) =>- available.some((skill) => skill.skillName === segment),+ available.some((skill) => skill.name === segment),);if (matches.length !== 1) return;const skillName = matches[0];diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.tsindex 9fe2f9b797..0096715bdb 100644--- a/packages/shared/src/server/llm/internalTraceEvents.ts+++ b/packages/shared/src/server/llm/internalTraceEvents.ts@@ -1,5 +1,5 @@import type {- SkillsAvailable,+ SkillsAvailableInput,SkillsResourceLoaded,} from "../../features/skills/trace";import {@@ -70,7 +70,7 @@ export type InternalTraceEventInput = {promptId?: string;promptName?: string;promptVersion?: string;- skillsAvailable?: SkillsAvailable;+ skillsAvailable?: SkillsAvailableInput;skillsResourceLoaded?: SkillsResourceLoaded;modelId?: string;modelName?: string;diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 0895572483..10a5587c86 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -13,6 +13,7 @@ import {safeJsonParse,type JsonNested,type SkillsAvailable,+ type SkillsAvailableInput,type SkillsResourceLoaded,} from "@langfuse/shared";import {@@ -1257,7 +1258,7 @@ export class IngestionService {}: {projectId: string;type: EventRecordInsertType["type"];- available?: SkillsAvailable;+ available?: SkillsAvailableInput;loaded?: SkillsResourceLoaded;}): Promise<{skills_available: SkillsAvailable;@@ -1277,21 +1278,21 @@ export class IngestionService {}available = available.slice(0, MAX_SKILLS_PER_OBSERVATION);loaded = loaded.slice(0, MAX_SKILLS_PER_OBSERVATION);- const versions = available.flatMap(({ skillName, skillVersion }) =>- skillVersion == null ? [] : [{ name: skillName, version: skillVersion }],+ const versions = available.flatMap(({ name, version }) =>+ version == null ? [] : [{ name, version }],);const skills = await this.skillCacheService.getReferences({projectId,versions,});- const availableSkills = available.map(({ skillName, skillVersion }) => {+ const availableSkills = available.map(({ name, version }) => {const skill = skills.find(- (skill) => skill.name === skillName && skill.version === skillVersion,+ (skill) => skill.name === name && skill.version === version,);return {- skillName,- skillVersion: skill?.version ?? skillVersion,+ name,+ version: skill?.version ?? version,langfuseSkillId: skill?.id,};});@@ -1299,7 +1300,7 @@ export class IngestionService {const loadedResources = loaded.map(({ skillName, filePath, skillVersion }) => {const candidates = availableSkills.filter(- (skill) => skill.skillName === skillName,+ (skill) => skill.name === skillName,);if (candidates.length !== 1)return { skillName, filePath, skillVersion };@@ -1309,13 +1310,20 @@ export class IngestionService {skillName,filePath,langfuseSkillId: skill?.langfuseSkillId,- skillVersion: skill.skillVersion ?? skillVersion,+ skillVersion: skill.version ?? skillVersion,};},);return {- skills_available: type === "GENERATION" ? availableSkills : [],+ skills_available:+ type === "GENERATION"+ ? availableSkills.map(({ name, version, langfuseSkillId }) => ({+ skillName: name,+ skillVersion: version,+ langfuseSkillId,+ }))+ : [],skills_resource_loaded: loadedResources,};}