Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(ingestion): accept name/version skill catalogs (#19087)

tobi_kle653b99d4b70
diff --git a/packages/shared/src/features/skills/trace.ts b/packages/shared/src/features/skills/trace.ts
index 4afe748509..0a7b37b0f8 100644
--- a/packages/shared/src/features/skills/trace.ts
+++ b/packages/shared/src/features/skills/trace.ts
@@ -13,6 +13,26 @@ export const SkillsAvailableSchema = z
)
.max(MAX_SKILLS_PER_OBSERVATION);
+export const SkillsAvailableInputSchema = z
+ .array(
+ z.union([
+ z.object({
+ name: SkillsAvailableSchema.element.shape.skillName,
+ version: SkillsAvailableSchema.element.shape.skillVersion,
+ }),
+ z
+ .object({
+ skillName: SkillsAvailableSchema.element.shape.skillName,
+ skillVersion: SkillsAvailableSchema.element.shape.skillVersion,
+ })
+ .transform(({ skillName, skillVersion }) => ({
+ name: skillName,
+ version: skillVersion,
+ })),
+ ]),
+ )
+ .max(MAX_SKILLS_PER_OBSERVATION);
+
export const SkillsResourceLoadedSchema = z
.array(
z.object({
@@ -24,5 +44,6 @@ export const SkillsResourceLoadedSchema = z
)
.max(MAX_SKILLS_PER_OBSERVATION);
+export type SkillsAvailableInput = z.infer<typeof SkillsAvailableInputSchema>;
export type SkillsAvailable = z.infer<typeof SkillsAvailableSchema>;
export type SkillsResourceLoaded = z.infer<typeof SkillsResourceLoadedSchema>;
diff --git a/packages/shared/src/server/ingestion/extractSkillsBackend.ts b/packages/shared/src/server/ingestion/extractSkillsBackend.ts
index ff5d1ffb8d..4912f230bf 100644
--- a/packages/shared/src/server/ingestion/extractSkillsBackend.ts
+++ b/packages/shared/src/server/ingestion/extractSkillsBackend.ts
@@ -1,8 +1,8 @@
import {
MAX_SKILLS_PER_OBSERVATION,
- SkillsAvailableSchema,
+ SkillsAvailableInputSchema,
SkillsResourceLoadedSchema,
- type SkillsAvailable,
+ type SkillsAvailableInput,
type SkillsResourceLoaded,
} from "../../features/skills/trace";
import {
@@ -32,7 +32,7 @@ export function extractSkillsFromObservation({
isError: boolean;
projectId: string;
}): {
- skillsAvailable?: SkillsAvailable;
+ skillsAvailable?: SkillsAvailableInput;
skillsResourceLoaded?: SkillsResourceLoaded;
} {
// Skill inference is enabled only when an availability attribute is reported.
@@ -67,7 +67,7 @@ export function extractSkillsFromObservation({
parsedManifest = undefined;
}
const available = Array.isArray(parsedManifest)
- ? SkillsAvailableSchema.safeParse(parsedManifest)
+ ? SkillsAvailableInputSchema.safeParse(parsedManifest)
: undefined;
if (observationType === ObservationType.GENERATION) {
if (available?.success) return { skillsAvailable: available.data };
@@ -112,7 +112,7 @@ function extractSkillResource(
toolName: string,
input: unknown,
output: unknown,
- available: SkillsAvailable,
+ available: SkillsAvailableInput,
projectId: string,
) {
const args = parseJsonPayloadWithMaxSize(
@@ -166,12 +166,12 @@ function extractSkillResource(
function extractSkillResourceFromPath(
path: unknown,
- available: SkillsAvailable,
+ available: SkillsAvailableInput,
) {
if (typeof path !== "string") return;
const segments = path.split("/");
const matches = segments.filter((segment) =>
- available.some((skill) => skill.skillName === segment),
+ available.some((skill) => skill.name === segment),
);
if (matches.length !== 1) return;
const skillName = matches[0];
diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.ts
index 9fe2f9b797..0096715bdb 100644
--- a/packages/shared/src/server/llm/internalTraceEvents.ts
+++ b/packages/shared/src/server/llm/internalTraceEvents.ts
@@ -1,5 +1,5 @@
import type {
- SkillsAvailable,
+ SkillsAvailableInput,
SkillsResourceLoaded,
} from "../../features/skills/trace";
import {
@@ -70,7 +70,7 @@ export type InternalTraceEventInput = {
promptId?: string;
promptName?: string;
promptVersion?: string;
- skillsAvailable?: SkillsAvailable;
+ skillsAvailable?: SkillsAvailableInput;
skillsResourceLoaded?: SkillsResourceLoaded;
modelId?: string;
modelName?: string;
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 0895572483..10a5587c86 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -13,6 +13,7 @@ import {
safeJsonParse,
type JsonNested,
type SkillsAvailable,
+ type SkillsAvailableInput,
type SkillsResourceLoaded,
} from "@langfuse/shared";
import {
@@ -1257,7 +1258,7 @@ export class IngestionService {
}: {
projectId: string;
type: EventRecordInsertType["type"];
- available?: SkillsAvailable;
+ available?: SkillsAvailableInput;
loaded?: SkillsResourceLoaded;
}): Promise<{
skills_available: SkillsAvailable;
@@ -1277,21 +1278,21 @@ export class IngestionService {
}
available = available.slice(0, MAX_SKILLS_PER_OBSERVATION);
loaded = loaded.slice(0, MAX_SKILLS_PER_OBSERVATION);
- const versions = available.flatMap(({ skillName, skillVersion }) =>
- skillVersion == null ? [] : [{ name: skillName, version: skillVersion }],
+ const versions = available.flatMap(({ name, version }) =>
+ version == null ? [] : [{ name, version }],
);
const skills = await this.skillCacheService.getReferences({
projectId,
versions,
});
- const availableSkills = available.map(({ skillName, skillVersion }) => {
+ const availableSkills = available.map(({ name, version }) => {
const skill = skills.find(
- (skill) => skill.name === skillName && skill.version === skillVersion,
+ (skill) => skill.name === name && skill.version === version,
);
return {
- skillName,
- skillVersion: skill?.version ?? skillVersion,
+ name,
+ version: skill?.version ?? version,
langfuseSkillId: skill?.id,
};
});
@@ -1299,7 +1300,7 @@ export class IngestionService {
const loadedResources = loaded.map(
({ skillName, filePath, skillVersion }) => {
const candidates = availableSkills.filter(
- (skill) => skill.skillName === skillName,
+ (skill) => skill.name === skillName,
);
if (candidates.length !== 1)
return { skillName, filePath, skillVersion };
@@ -1309,13 +1310,20 @@ export class IngestionService {
skillName,
filePath,
langfuseSkillId: skill?.langfuseSkillId,
- skillVersion: skill.skillVersion ?? skillVersion,
+ skillVersion: skill.version ?? skillVersion,
};
},
);
return {
- skills_available: type === "GENERATION" ? availableSkills : [],
+ skills_available:
+ type === "GENERATION"
+ ? availableSkills.map(({ name, version, langfuseSkillId }) => ({
+ skillName: name,
+ skillVersion: version,
+ langfuseSkillId,
+ }))
+ : [],
skills_resource_loaded: loadedResources,
};
}