🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(tracing): show requested skill loads and file paths (#19239)
tobi_klc83198b03f6b
diff --git a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsxindex 96c196fbb7..b2accb6188 100644--- a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx+++ b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx@@ -358,7 +358,11 @@ export function IOPreviewPretty({) : null}{showData && skills && skills.items.length > 0 ? (- <SectionSkills projectId={projectId} skills={skills} />+ <SectionSkills+ projectId={projectId}+ skills={skills}+ toolCallsByName={toolCallsByName}+ />) : null}{shouldRenderMessages ? (diff --git a/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsx b/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsxindex 1eccbabb45..0030f77bfc 100644--- a/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsx+++ b/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsx@@ -3,7 +3,69 @@ import { expect, userEvent, within } from "storybook/test";import preview from "../../../../../../.storybook/preview";import { SectionSkills } from "./SectionSkills";-const meta = preview.meta({ component: SectionSkills });+const meta = preview.meta({+ component: SectionSkills,+ args: { toolCallsByName: new Map() },+});++export const LoadRequested = meta.story({+ args: {+ projectId: "storybook-project",+ skills: {+ kind: "available",+ items: [+ {+ skillName: "refund-policy",+ langfuseSkillId: "refund",+ skillVersion: 3,+ },+ {+ skillName: "shipping-policy",+ langfuseSkillId: "shipping",+ skillVersion: 1,+ },+ { skillName: "account-security" },+ ],+ },+ toolCallsByName: new Map([+ [+ "loadSkill",+ [+ {+ name: "loadSkill",+ invocationNumber: 1,+ arguments: { name: "refund-policy", version: 3 },+ },+ ],+ ],+ [+ "Read",+ [+ {+ name: "Read",+ invocationNumber: 2,+ arguments: { file_path: "/skills/account-security/SKILL.md" },+ },+ ],+ ],+ [+ "loadSkillResource",+ [+ {+ name: "loadSkillResource",+ invocationNumber: 3,+ arguments: { name: "refund-policy", path: "references/refunds.md" },+ },+ {+ name: "loadSkillResource",+ invocationNumber: 4,+ arguments: { name: "refund-policy", path: "references/refunds.md" },+ },+ ],+ ],+ ]),+ },+});export const Available = meta.story({args: {@@ -186,12 +248,109 @@ export const Loaded = meta.story({export const LongListStartsCollapsed = meta.story({name: "(Test) Long List Starts Collapsed",- args: Available.input.args,+ args: {+ ...Available.input.args,+ toolCallsByName: LoadRequested.input.args.toolCallsByName,+ },play: async ({ canvasElement }) => {const canvas = within(canvasElement);+ const summary = canvas.getByRole("button", { name: "Expand Skills" });+ await expect(within(summary).getByText("10")).toBeVisible();+ await expect(canvas.queryByRole("tab")).not.toBeInTheDocument();+ await userEvent.click(summary);await userEvent.click(- canvas.getByRole("button", { name: "Expand Available skills" }),+ canvas.getByRole("tab", { name: /^Not requested\s*8$/ }),);await expect(canvas.getByText("product-guide")).toBeVisible();+ await userEvent.click(+ canvas.getByRole("button", { name: "Collapse Skills" }),+ );+ await expect(within(summary).getByText("10")).toBeVisible();+ },+});++export const RequestedLoadsMatchAvailableRows = meta.story({+ name: "(Test) Requested Loads Match Available Rows",+ args: LoadRequested.input.args,+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);+ await expect(+ canvas.getByRole("tab", { name: /^Requested\s*2$/ }),+ ).toHaveAttribute("aria-selected", "true");+ await expect(+ canvas.queryByRole("link", { name: "shipping-policy" }),+ ).toBeNull();+ await expect(+ canvas.getByRole("link", { name: "refund-policy" }),+ ).toHaveAttribute(+ "href",+ "/project/storybook-project/skills/refund-policy?version=3",+ );+ await userEvent.click(+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),+ );+ await expect(canvas.getByText("SKILL.md")).toBeVisible();+ await expect(canvas.getAllByText("references/refunds.md")).toHaveLength(1);+ await expect(canvas.getByText("2 files")).toBeVisible();+ await userEvent.click(+ canvas.getByRole("button", { name: "Collapse Skills" }),+ );+ await userEvent.click(+ canvas.getByRole("button", { name: "Expand Skills" }),+ );+ await expect(canvas.getByText("references/refunds.md")).toBeVisible();+ await userEvent.click(+ canvas.getByRole("button", {+ name: "Files requested for account-security",+ }),+ );+ await expect(+ canvas.getByText("/skills/account-security/SKILL.md"),+ ).toBeVisible();+ await expect(+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),+ ).toHaveAttribute("aria-expanded", "false");+ await userEvent.click(+ canvas.getByRole("tab", { name: /^Not requested\s*1$/ }),+ );+ await expect(+ canvas.getByRole("link", { name: "shipping-policy" }),+ ).toBeVisible();+ await expect(+ canvas.queryByRole("link", { name: "refund-policy" }),+ ).toBeNull();+ await expect(+ canvas.queryByRole("button", { name: /Files requested/ }),+ ).toBeNull();+ await userEvent.click(canvas.getByRole("tab", { name: /^Requested\s*2$/ }));+ await expect(+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),+ ).toHaveAttribute("aria-expanded", "false");+ },+});++export const AllRequested = meta.story({+ name: "(Test) All Requested",+ args: {+ ...LoadRequested.input.args,+ skills: {+ kind: "available",+ items: [+ {+ skillName: "refund-policy",+ langfuseSkillId: "refund",+ skillVersion: 3,+ },+ ],+ },+ },+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);+ await expect(canvas.queryByRole("tab")).toBeNull();+ await userEvent.click(+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),+ );+ await expect(canvas.getByText("SKILL.md")).toBeVisible();+ await expect(canvas.getByText("references/refunds.md")).toBeVisible();},});diff --git a/web/src/features/traces/components/IOPreview/components/SectionSkills.tsx b/web/src/features/traces/components/IOPreview/components/SectionSkills.tsxindex 6aad1b36b2..ad49fcd704 100644--- a/web/src/features/traces/components/IOPreview/components/SectionSkills.tsx+++ b/web/src/features/traces/components/IOPreview/components/SectionSkills.tsx@@ -16,21 +16,78 @@ import {} from "@/src/components/ui/collapsible";import { IOSection } from "@/src/components/ui/IOSection";import { cn } from "@/src/utils/tailwind";+import { getSkillLoadRequests } from "../../../fns/getSkillLoadRequests";+import type { ToolCallInvocation } from "../../../hooks/useChatMLParser";-export function SectionSkills({ projectId, skills }: SectionSkillsProps) {+export function SectionSkills({+ projectId,+ skills,+ toolCallsByName,+}: SectionSkillsProps) {const [expandedSkillKey, setExpandedSkillKey] = useState("");+ const [activeTab, setActiveTab] = useState("requested");const groups = groupSkills(skills.items);+ const [collapsed, setCollapsed] = useState(groups.length > 5);+ const loadRequests = getSkillLoadRequests(+ skills.kind === "available" ? groups.map((group) => group.skill) : [],+ toolCallsByName,+ );const singleLoadedSkill =skills.kind === "loaded" && groups.length === 1 ? groups[0] : undefined;- const title =- skills.kind === "available" ? "Available skills" : "Loaded skills";+ const title = skills.kind === "available" ? "Skills" : "Loaded skills";+ const requestedGroups = groups.filter((group) =>+ loadRequests.has(group.skill),+ );+ const notRequestedGroups = groups.filter(+ (group) => !loadRequests.has(group.skill),+ );+ const hasSwitch = requestedGroups.length > 0 && notRequestedGroups.length > 0;+ const tabGroups =+ activeTab === "requested" ? requestedGroups : notRequestedGroups;+ const visibleGroups = hasSwitch ? tabGroups : groups;return (<IOSectiontitle={singleLoadedSkill ? "Loaded skill" : title}- count={singleLoadedSkill ? undefined : groups.length}+ count={+ singleLoadedSkill || (hasSwitch && !collapsed)+ ? undefined+ : groups.length+ }variant="table"- defaultCollapsed={groups.length > 5}+ collapsed={collapsed}+ onCollapsedChange={setCollapsed}+ tabs={+ hasSwitch+ ? {+ value: activeTab,+ onValueChange: (value) => {+ setActiveTab(value);+ setExpandedSkillKey("");+ },+ items: [+ {+ value: "requested",+ label: "Requested",+ count: requestedGroups.length,+ },+ {+ value: "not-requested",+ label: "Not requested",+ count: notRequestedGroups.length,+ },+ ],+ }+ : undefined+ }+ info={+ loadRequests.size > 0+ ? {+ label: "About skill load requests",+ text: `${loadRequests.size} of ${groups.length} available skills requested by this generation's output tool calls. A request does not confirm that the skill was loaded successfully.`,+ }+ : undefined+ }>{singleLoadedSkill ? (<div className="min-w-0">@@ -47,14 +104,22 @@ export function SectionSkills({ projectId, skills }: SectionSkillsProps) {</div>) : (<ul className="divide-border flex flex-col divide-y">- {groups.map((group, index) => (+ {visibleGroups.map((group, index) => (<SkillRowkey={group.key}projectId={projectId}- group={group}+ group={+ skills.kind === "available"+ ? {+ ...group,+ files: loadRequests.get(group.skill) ?? new Set<string>(),+ }+ : group+ }rowNumber={index + 1}expanded={expandedSkillKey === group.key}onExpandedChange={setExpandedSkillKey}+ fileState={skills.kind === "available" ? "requested" : "loaded"}/>))}</ul>@@ -69,6 +134,7 @@ function SkillRow({rowNumber,expanded,onExpandedChange,+ fileState,}: SkillRowProps) {const { skill, files } = group;@@ -88,7 +154,7 @@ function SkillRow({{files.size > 0 ? (<CollapsibleTriggerclassName="hover:bg-muted/20 flex shrink-0 cursor-pointer items-center rounded-sm"- aria-label={`Files loaded for ${skill.skillName}`}+ aria-label={`Files ${fileState} for ${skill.skillName}`}><DropdownIndicator direction={expanded ? "down" : "right"} /></CollapsibleTrigger>@@ -179,6 +245,7 @@ function groupSkills(export interface SectionSkillsProps {projectId: string;+ toolCallsByName: Map<string, ToolCallInvocation[]>;skills:| { kind: "available"; items: SkillsAvailable }| { kind: "loaded"; items: SkillsResourceLoaded };@@ -190,6 +257,7 @@ type SkillRowProps = {rowNumber: number;expanded: boolean;onExpandedChange: (key: string) => void;+ fileState: "requested" | "loaded";};type SkillGroup = {diff --git a/web/src/features/traces/fns/getSkillLoadRequests.ts b/web/src/features/traces/fns/getSkillLoadRequests.tsnew file mode 100644index 0000000000..bb8d0272a7--- /dev/null+++ b/web/src/features/traces/fns/getSkillLoadRequests.ts@@ -0,0 +1,104 @@+import {+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,+ SKILL_RESOURCE_TOOL_NAMES,+ SkillSelectorSchema,+ type SkillsAvailable,+} from "@langfuse/shared";+import type { ToolCallInvocation } from "../hooks/useChatMLParser";++const SKILL_LOAD_TOOLS = new Set(+ Array.from(SKILL_RESOURCE_TOOL_NAMES).concat([+ "loadSkillResource",+ "langfuse_loadSkill",+ "langfuse_loadSkillResource",+ ]),+);++function parseArguments(value: unknown): Record<string, unknown> | undefined {+ if (typeof value === "string") {+ if (value.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES) return;+ try {+ value = JSON.parse(value);+ } catch {+ return;+ }+ }+ if (value === null || typeof value !== "object" || Array.isArray(value)) {+ return;+ }+ return value as Record<string, unknown>;+}++function getSkillFileFromPath(path: unknown, skills: SkillsAvailable) {+ if (typeof path !== "string") return;+ const segments = path.split("/");+ if (segments.includes("..")) return;+ const matches = segments.filter((segment) =>+ skills.some((skill) => skill.skillName === segment),+ );+ if (matches.length !== 1) return;+ const index = segments.indexOf(matches[0]);+ // Generic file reads must identify a resource beneath a skills directory.+ if (segments[index - 1] !== "skills") return;+ if (!segments.slice(index + 1).join("/")) return;+ return { name: matches[0], filePath: path };+}++function getRequestedSkillFile(+ toolName: string,+ args: Record<string, unknown>,+ skills: SkillsAvailable,+) {+ if (toolName === "readFile" || toolName === "Read") {+ return getSkillFileFromPath(args.path ?? args.file_path, skills);+ }+ const name =+ toolName === "skill_read" ? args.skillName : (args.name ?? args.skill);+ if (typeof name !== "string") return;+ const isResourceRead =+ toolName === "skill_read" || toolName.endsWith("loadSkillResource");+ if (!isResourceRead) return { name, filePath: "SKILL.md" };+ if (typeof args.path !== "string" || !args.path.trim()) return;+ return { name, filePath: args.path };+}++function findUniqueSkill(+ skills: SkillsAvailable,+ name: string,+ version: unknown,+) {+ const selector = SkillSelectorSchema.safeParse({ version });+ if (!selector.success) return;+ const candidates = skills.filter(+ (skill) =>+ skill.skillName === name &&+ (selector.data.version === undefined ||+ skill.skillVersion === selector.data.version),+ );+ return candidates.length === 1 ? candidates[0] : undefined;+}++/** Matches output-side load requests to unambiguous available skill identities. */+export function getSkillLoadRequests(+ skills: SkillsAvailable,+ toolCallsByName: Map<string, ToolCallInvocation[]>,+): Map<SkillsAvailable[number], Set<string>> {+ const requests = new Map<SkillsAvailable[number], Set<string>>();++ for (const [toolName, calls] of toolCallsByName) {+ if (!SKILL_LOAD_TOOLS.has(toolName)) continue;+ for (const call of calls) {+ const args = parseArguments(call.arguments);+ if (!args) continue;+ const requested = getRequestedSkillFile(toolName, args, skills);+ if (!requested) continue;+ const skill = findUniqueSkill(skills, requested.name, args.version);+ if (!skill) continue;+ const skillRequests = requests.get(skill) ?? new Set<string>();+ skillRequests.add(requested.filePath);+ requests.set(skill, skillRequests);+ }+ }++ return requests;+}diff --git a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts b/web/src/features/traces/hooks/useChatMLParser.clienttest.tsindex e50b468a59..71790a56d7 100644--- a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts+++ b/web/src/features/traces/hooks/useChatMLParser.clienttest.ts@@ -2,6 +2,7 @@ import { renderHook } from "@testing-library/react";import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";import { toIOPreview } from "../parsers/toIOPreview";import { hasThinkingContent } from "../fns/chatMessageUtils";+import { getSkillLoadRequests } from "../fns/getSkillLoadRequests";import {parseChatML,type ChatMLParserResult,@@ -27,6 +28,162 @@ const parserImplementations = [] as const;describe("useChatMLParser", () => {+ it.each(parserImplementations)(+ "$name parser highlights only skills requested by the current generation",+ ({ parse }) => {+ const available = [+ { skillName: "refund-policy", skillVersion: 3 },+ { skillName: "shipping-policy", skillVersion: 1 },+ { skillName: "account-security" },+ ];+ const result = parse(+ [+ {+ role: "assistant",+ tool_calls: [+ {+ id: "historical",+ name: "loadSkill",+ arguments: { name: "shipping-policy" },+ },+ ],+ },+ ],+ [+ {+ role: "assistant",+ tool_calls: [+ {+ id: "load-refund",+ name: "loadSkill",+ arguments: '{"name":"refund-policy","version":"3"}',+ },+ {+ id: "read-security",+ name: "Read",+ arguments: { file_path: "/skills/account-security/SKILL.md" },+ },+ {+ id: "read-refund-resource",+ name: "loadSkillResource",+ arguments: {+ name: "refund-policy",+ version: "3",+ path: "references/refunds.md",+ },+ },+ ],+ },+ ],+ );++ const requests = getSkillLoadRequests(available, result.toolCallsByName);+ expect(requests.has(available[1])).toBe(false);+ expect(requests.get(available[0])).toEqual(+ new Set(["SKILL.md", "references/refunds.md"]),+ );+ expect(requests.get(available[2])).toEqual(+ new Set(["/skills/account-security/SKILL.md"]),+ );+ },+ );++ it("does not infer a skill load from malformed, unrelated, or ambiguous calls", () => {+ const available = [+ { skillName: "refund-policy", skillVersion: 2 },+ { skillName: "refund-policy", skillVersion: 3 },+ { skillName: "shipping-policy" },+ ];+ const calls = [+ { name: "loadSkill", arguments: { name: "refund-policy" } },+ { name: "loadSkill", arguments: { name: "refund-policy", version: 4 } },+ { name: "loadSkill", arguments: '{"name":' },+ { name: "loadSkill", arguments: ["shipping-policy"] },+ { name: "loadSkill", arguments: { name: "unknown-skill" } },+ { name: "loadSkillResource", arguments: { name: "shipping-policy" } },+ { name: "search", arguments: { name: "shipping-policy" } },+ {+ name: "Read",+ arguments: { path: "/skills/shipping-policy-backup/SKILL.md" },+ },+ {+ name: "Read",+ arguments: { path: "/skills/shipping-policy/../other/SKILL.md" },+ },+ { name: "Read", arguments: { path: "/skills/shipping-policy/" } },+ {+ name: "Read",+ arguments: { path: "/repo/shipping-policy/notes.txt" },+ },+ {+ name: "readFile",+ arguments: { path: "/repo/shipping-policy/notes.txt" },+ },+ {+ name: "Read",+ arguments: { path: "/skills/shipping-policy/refund-policy/SKILL.md" },+ },+ ].map((call, index) => ({ ...call, invocationNumber: index + 1 }));+ const byName = new Map<string, typeof calls>();+ for (const call of calls)+ byName.set(call.name, [...(byName.get(call.name) ?? []), call]);++ expect(getSkillLoadRequests(available, byName).size).toBe(0);+ const explicitVersion = {+ name: "loadSkill",+ arguments: { name: "refund-policy", version: 3 },+ invocationNumber: 12,+ };+ byName.set("loadSkill", [explicitVersion]);+ expect(getSkillLoadRequests(available, byName).get(available[1])).toEqual(+ new Set(["SKILL.md"]),+ );+ });++ it.each(["invalid", "0", "-1", "3.5", null, {}])(+ "ignores an invalid requested skill version: %j",+ (version) => {+ const available = [{ skillName: "refund-policy", skillVersion: 3 }];+ const call = {+ name: "loadSkill",+ arguments: { name: "refund-policy", version },+ invocationNumber: 1,+ };+ expect(+ getSkillLoadRequests(available, new Map([[call.name, [call]]])).size,+ ).toBe(0);+ },+ );++ it.each([+ ["Skill", { skill: "refund-policy" }, "SKILL.md"],+ ["skill", { name: "refund-policy" }, "SKILL.md"],+ [+ "skill_read",+ { skillName: "refund-policy", path: "references/refunds.md" },+ "references/refunds.md",+ ],+ [+ "readFile",+ { path: "/skills/refund-policy/SKILL.md" },+ "/skills/refund-policy/SKILL.md",+ ],+ ["langfuse_loadSkill", { name: "refund-policy" }, "SKILL.md"],+ [+ "langfuse_loadSkillResource",+ { name: "refund-policy", path: "SKILL.md" },+ "SKILL.md",+ ],+ ])("recognizes the %s skill-loading adapter", (name, args, filePath) => {+ const available = [{ skillName: "refund-policy" }];+ const call = { name, arguments: args, invocationNumber: 1 };+ expect(+ getSkillLoadRequests(available, new Map([[name, [call]]])).get(+ available[0],+ ),+ ).toEqual(new Set([filePath]));+ });+it.each(parserImplementations)("$name parser groups output-side tool call arguments by tool name",({ parse, expectedArguments }) => {