Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(tracing): show requested skill loads and file paths (#19239)

tobi_klc83198b03f6b
diff --git a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
index 96c196fbb7..b2accb6188 100644
--- a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
+++ b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
@@ -358,7 +358,11 @@ export function IOPreviewPretty({
) : null}
{showData && skills && skills.items.length > 0 ? (
- <SectionSkills projectId={projectId} skills={skills} />
+ <SectionSkills
+ projectId={projectId}
+ skills={skills}
+ toolCallsByName={toolCallsByName}
+ />
) : null}
{shouldRenderMessages ? (
diff --git a/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsx b/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsx
index 1eccbabb45..0030f77bfc 100644
--- a/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsx
+++ b/web/src/features/traces/components/IOPreview/components/SectionSkills.stories.tsx
@@ -3,7 +3,69 @@ import { expect, userEvent, within } from "storybook/test";
import preview from "../../../../../../.storybook/preview";
import { SectionSkills } from "./SectionSkills";
-const meta = preview.meta({ component: SectionSkills });
+const meta = preview.meta({
+ component: SectionSkills,
+ args: { toolCallsByName: new Map() },
+});
+
+export const LoadRequested = meta.story({
+ args: {
+ projectId: "storybook-project",
+ skills: {
+ kind: "available",
+ items: [
+ {
+ skillName: "refund-policy",
+ langfuseSkillId: "refund",
+ skillVersion: 3,
+ },
+ {
+ skillName: "shipping-policy",
+ langfuseSkillId: "shipping",
+ skillVersion: 1,
+ },
+ { skillName: "account-security" },
+ ],
+ },
+ toolCallsByName: new Map([
+ [
+ "loadSkill",
+ [
+ {
+ name: "loadSkill",
+ invocationNumber: 1,
+ arguments: { name: "refund-policy", version: 3 },
+ },
+ ],
+ ],
+ [
+ "Read",
+ [
+ {
+ name: "Read",
+ invocationNumber: 2,
+ arguments: { file_path: "/skills/account-security/SKILL.md" },
+ },
+ ],
+ ],
+ [
+ "loadSkillResource",
+ [
+ {
+ name: "loadSkillResource",
+ invocationNumber: 3,
+ arguments: { name: "refund-policy", path: "references/refunds.md" },
+ },
+ {
+ name: "loadSkillResource",
+ invocationNumber: 4,
+ arguments: { name: "refund-policy", path: "references/refunds.md" },
+ },
+ ],
+ ],
+ ]),
+ },
+});
export const Available = meta.story({
args: {
@@ -186,12 +248,109 @@ export const Loaded = meta.story({
export const LongListStartsCollapsed = meta.story({
name: "(Test) Long List Starts Collapsed",
- args: Available.input.args,
+ args: {
+ ...Available.input.args,
+ toolCallsByName: LoadRequested.input.args.toolCallsByName,
+ },
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
+ const summary = canvas.getByRole("button", { name: "Expand Skills" });
+ await expect(within(summary).getByText("10")).toBeVisible();
+ await expect(canvas.queryByRole("tab")).not.toBeInTheDocument();
+ await userEvent.click(summary);
await userEvent.click(
- canvas.getByRole("button", { name: "Expand Available skills" }),
+ canvas.getByRole("tab", { name: /^Not requested\s*8$/ }),
);
await expect(canvas.getByText("product-guide")).toBeVisible();
+ await userEvent.click(
+ canvas.getByRole("button", { name: "Collapse Skills" }),
+ );
+ await expect(within(summary).getByText("10")).toBeVisible();
+ },
+});
+
+export const RequestedLoadsMatchAvailableRows = meta.story({
+ name: "(Test) Requested Loads Match Available Rows",
+ args: LoadRequested.input.args,
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ await expect(
+ canvas.getByRole("tab", { name: /^Requested\s*2$/ }),
+ ).toHaveAttribute("aria-selected", "true");
+ await expect(
+ canvas.queryByRole("link", { name: "shipping-policy" }),
+ ).toBeNull();
+ await expect(
+ canvas.getByRole("link", { name: "refund-policy" }),
+ ).toHaveAttribute(
+ "href",
+ "/project/storybook-project/skills/refund-policy?version=3",
+ );
+ await userEvent.click(
+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),
+ );
+ await expect(canvas.getByText("SKILL.md")).toBeVisible();
+ await expect(canvas.getAllByText("references/refunds.md")).toHaveLength(1);
+ await expect(canvas.getByText("2 files")).toBeVisible();
+ await userEvent.click(
+ canvas.getByRole("button", { name: "Collapse Skills" }),
+ );
+ await userEvent.click(
+ canvas.getByRole("button", { name: "Expand Skills" }),
+ );
+ await expect(canvas.getByText("references/refunds.md")).toBeVisible();
+ await userEvent.click(
+ canvas.getByRole("button", {
+ name: "Files requested for account-security",
+ }),
+ );
+ await expect(
+ canvas.getByText("/skills/account-security/SKILL.md"),
+ ).toBeVisible();
+ await expect(
+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),
+ ).toHaveAttribute("aria-expanded", "false");
+ await userEvent.click(
+ canvas.getByRole("tab", { name: /^Not requested\s*1$/ }),
+ );
+ await expect(
+ canvas.getByRole("link", { name: "shipping-policy" }),
+ ).toBeVisible();
+ await expect(
+ canvas.queryByRole("link", { name: "refund-policy" }),
+ ).toBeNull();
+ await expect(
+ canvas.queryByRole("button", { name: /Files requested/ }),
+ ).toBeNull();
+ await userEvent.click(canvas.getByRole("tab", { name: /^Requested\s*2$/ }));
+ await expect(
+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),
+ ).toHaveAttribute("aria-expanded", "false");
+ },
+});
+
+export const AllRequested = meta.story({
+ name: "(Test) All Requested",
+ args: {
+ ...LoadRequested.input.args,
+ skills: {
+ kind: "available",
+ items: [
+ {
+ skillName: "refund-policy",
+ langfuseSkillId: "refund",
+ skillVersion: 3,
+ },
+ ],
+ },
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ await expect(canvas.queryByRole("tab")).toBeNull();
+ await userEvent.click(
+ canvas.getByRole("button", { name: "Files requested for refund-policy" }),
+ );
+ await expect(canvas.getByText("SKILL.md")).toBeVisible();
+ await expect(canvas.getByText("references/refunds.md")).toBeVisible();
},
});
diff --git a/web/src/features/traces/components/IOPreview/components/SectionSkills.tsx b/web/src/features/traces/components/IOPreview/components/SectionSkills.tsx
index 6aad1b36b2..ad49fcd704 100644
--- a/web/src/features/traces/components/IOPreview/components/SectionSkills.tsx
+++ b/web/src/features/traces/components/IOPreview/components/SectionSkills.tsx
@@ -16,21 +16,78 @@ import {
} from "@/src/components/ui/collapsible";
import { IOSection } from "@/src/components/ui/IOSection";
import { cn } from "@/src/utils/tailwind";
+import { getSkillLoadRequests } from "../../../fns/getSkillLoadRequests";
+import type { ToolCallInvocation } from "../../../hooks/useChatMLParser";
-export function SectionSkills({ projectId, skills }: SectionSkillsProps) {
+export function SectionSkills({
+ projectId,
+ skills,
+ toolCallsByName,
+}: SectionSkillsProps) {
const [expandedSkillKey, setExpandedSkillKey] = useState("");
+ const [activeTab, setActiveTab] = useState("requested");
const groups = groupSkills(skills.items);
+ const [collapsed, setCollapsed] = useState(groups.length > 5);
+ const loadRequests = getSkillLoadRequests(
+ skills.kind === "available" ? groups.map((group) => group.skill) : [],
+ toolCallsByName,
+ );
const singleLoadedSkill =
skills.kind === "loaded" && groups.length === 1 ? groups[0] : undefined;
- const title =
- skills.kind === "available" ? "Available skills" : "Loaded skills";
+ const title = skills.kind === "available" ? "Skills" : "Loaded skills";
+ const requestedGroups = groups.filter((group) =>
+ loadRequests.has(group.skill),
+ );
+ const notRequestedGroups = groups.filter(
+ (group) => !loadRequests.has(group.skill),
+ );
+ const hasSwitch = requestedGroups.length > 0 && notRequestedGroups.length > 0;
+ const tabGroups =
+ activeTab === "requested" ? requestedGroups : notRequestedGroups;
+ const visibleGroups = hasSwitch ? tabGroups : groups;
return (
<IOSection
title={singleLoadedSkill ? "Loaded skill" : title}
- count={singleLoadedSkill ? undefined : groups.length}
+ count={
+ singleLoadedSkill || (hasSwitch && !collapsed)
+ ? undefined
+ : groups.length
+ }
variant="table"
- defaultCollapsed={groups.length > 5}
+ collapsed={collapsed}
+ onCollapsedChange={setCollapsed}
+ tabs={
+ hasSwitch
+ ? {
+ value: activeTab,
+ onValueChange: (value) => {
+ setActiveTab(value);
+ setExpandedSkillKey("");
+ },
+ items: [
+ {
+ value: "requested",
+ label: "Requested",
+ count: requestedGroups.length,
+ },
+ {
+ value: "not-requested",
+ label: "Not requested",
+ count: notRequestedGroups.length,
+ },
+ ],
+ }
+ : undefined
+ }
+ info={
+ loadRequests.size > 0
+ ? {
+ label: "About skill load requests",
+ text: `${loadRequests.size} of ${groups.length} available skills requested by this generation's output tool calls. A request does not confirm that the skill was loaded successfully.`,
+ }
+ : undefined
+ }
>
{singleLoadedSkill ? (
<div className="min-w-0">
@@ -47,14 +104,22 @@ export function SectionSkills({ projectId, skills }: SectionSkillsProps) {
</div>
) : (
<ul className="divide-border flex flex-col divide-y">
- {groups.map((group, index) => (
+ {visibleGroups.map((group, index) => (
<SkillRow
key={group.key}
projectId={projectId}
- group={group}
+ group={
+ skills.kind === "available"
+ ? {
+ ...group,
+ files: loadRequests.get(group.skill) ?? new Set<string>(),
+ }
+ : group
+ }
rowNumber={index + 1}
expanded={expandedSkillKey === group.key}
onExpandedChange={setExpandedSkillKey}
+ fileState={skills.kind === "available" ? "requested" : "loaded"}
/>
))}
</ul>
@@ -69,6 +134,7 @@ function SkillRow({
rowNumber,
expanded,
onExpandedChange,
+ fileState,
}: SkillRowProps) {
const { skill, files } = group;
@@ -88,7 +154,7 @@ function SkillRow({
{files.size > 0 ? (
<CollapsibleTrigger
className="hover:bg-muted/20 flex shrink-0 cursor-pointer items-center rounded-sm"
- aria-label={`Files loaded for ${skill.skillName}`}
+ aria-label={`Files ${fileState} for ${skill.skillName}`}
>
<DropdownIndicator direction={expanded ? "down" : "right"} />
</CollapsibleTrigger>
@@ -179,6 +245,7 @@ function groupSkills(
export interface SectionSkillsProps {
projectId: string;
+ toolCallsByName: Map<string, ToolCallInvocation[]>;
skills:
| { kind: "available"; items: SkillsAvailable }
| { kind: "loaded"; items: SkillsResourceLoaded };
@@ -190,6 +257,7 @@ type SkillRowProps = {
rowNumber: number;
expanded: boolean;
onExpandedChange: (key: string) => void;
+ fileState: "requested" | "loaded";
};
type SkillGroup = {
diff --git a/web/src/features/traces/fns/getSkillLoadRequests.ts b/web/src/features/traces/fns/getSkillLoadRequests.ts
new file mode 100644
index 0000000000..bb8d0272a7
--- /dev/null
+++ b/web/src/features/traces/fns/getSkillLoadRequests.ts
@@ -0,0 +1,104 @@
+import {
+ MAX_SKILL_TOOL_ARGS_JSON_BYTES,
+ SKILL_RESOURCE_TOOL_NAMES,
+ SkillSelectorSchema,
+ type SkillsAvailable,
+} from "@langfuse/shared";
+import type { ToolCallInvocation } from "../hooks/useChatMLParser";
+
+const SKILL_LOAD_TOOLS = new Set(
+ Array.from(SKILL_RESOURCE_TOOL_NAMES).concat([
+ "loadSkillResource",
+ "langfuse_loadSkill",
+ "langfuse_loadSkillResource",
+ ]),
+);
+
+function parseArguments(value: unknown): Record<string, unknown> | undefined {
+ if (typeof value === "string") {
+ if (value.length > MAX_SKILL_TOOL_ARGS_JSON_BYTES) return;
+ try {
+ value = JSON.parse(value);
+ } catch {
+ return;
+ }
+ }
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
+ return;
+ }
+ return value as Record<string, unknown>;
+}
+
+function getSkillFileFromPath(path: unknown, skills: SkillsAvailable) {
+ if (typeof path !== "string") return;
+ const segments = path.split("/");
+ if (segments.includes("..")) return;
+ const matches = segments.filter((segment) =>
+ skills.some((skill) => skill.skillName === segment),
+ );
+ if (matches.length !== 1) return;
+ const index = segments.indexOf(matches[0]);
+ // Generic file reads must identify a resource beneath a skills directory.
+ if (segments[index - 1] !== "skills") return;
+ if (!segments.slice(index + 1).join("/")) return;
+ return { name: matches[0], filePath: path };
+}
+
+function getRequestedSkillFile(
+ toolName: string,
+ args: Record<string, unknown>,
+ skills: SkillsAvailable,
+) {
+ if (toolName === "readFile" || toolName === "Read") {
+ return getSkillFileFromPath(args.path ?? args.file_path, skills);
+ }
+ const name =
+ toolName === "skill_read" ? args.skillName : (args.name ?? args.skill);
+ if (typeof name !== "string") return;
+ const isResourceRead =
+ toolName === "skill_read" || toolName.endsWith("loadSkillResource");
+ if (!isResourceRead) return { name, filePath: "SKILL.md" };
+ if (typeof args.path !== "string" || !args.path.trim()) return;
+ return { name, filePath: args.path };
+}
+
+function findUniqueSkill(
+ skills: SkillsAvailable,
+ name: string,
+ version: unknown,
+) {
+ const selector = SkillSelectorSchema.safeParse({ version });
+ if (!selector.success) return;
+ const candidates = skills.filter(
+ (skill) =>
+ skill.skillName === name &&
+ (selector.data.version === undefined ||
+ skill.skillVersion === selector.data.version),
+ );
+ return candidates.length === 1 ? candidates[0] : undefined;
+}
+
+/** Matches output-side load requests to unambiguous available skill identities. */
+export function getSkillLoadRequests(
+ skills: SkillsAvailable,
+ toolCallsByName: Map<string, ToolCallInvocation[]>,
+): Map<SkillsAvailable[number], Set<string>> {
+ const requests = new Map<SkillsAvailable[number], Set<string>>();
+
+ for (const [toolName, calls] of toolCallsByName) {
+ if (!SKILL_LOAD_TOOLS.has(toolName)) continue;
+ for (const call of calls) {
+ const args = parseArguments(call.arguments);
+ if (!args) continue;
+ const requested = getRequestedSkillFile(toolName, args, skills);
+ if (!requested) continue;
+ const skill = findUniqueSkill(skills, requested.name, args.version);
+ if (!skill) continue;
+ const skillRequests = requests.get(skill) ?? new Set<string>();
+ skillRequests.add(requested.filePath);
+ requests.set(skill, skillRequests);
+ }
+ }
+
+ return requests;
+}
diff --git a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts b/web/src/features/traces/hooks/useChatMLParser.clienttest.ts
index e50b468a59..71790a56d7 100644
--- a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts
+++ b/web/src/features/traces/hooks/useChatMLParser.clienttest.ts
@@ -2,6 +2,7 @@ import { renderHook } from "@testing-library/react";
import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";
import { toIOPreview } from "../parsers/toIOPreview";
import { hasThinkingContent } from "../fns/chatMessageUtils";
+import { getSkillLoadRequests } from "../fns/getSkillLoadRequests";
import {
parseChatML,
type ChatMLParserResult,
@@ -27,6 +28,162 @@ const parserImplementations = [
] as const;
describe("useChatMLParser", () => {
+ it.each(parserImplementations)(
+ "$name parser highlights only skills requested by the current generation",
+ ({ parse }) => {
+ const available = [
+ { skillName: "refund-policy", skillVersion: 3 },
+ { skillName: "shipping-policy", skillVersion: 1 },
+ { skillName: "account-security" },
+ ];
+ const result = parse(
+ [
+ {
+ role: "assistant",
+ tool_calls: [
+ {
+ id: "historical",
+ name: "loadSkill",
+ arguments: { name: "shipping-policy" },
+ },
+ ],
+ },
+ ],
+ [
+ {
+ role: "assistant",
+ tool_calls: [
+ {
+ id: "load-refund",
+ name: "loadSkill",
+ arguments: '{"name":"refund-policy","version":"3"}',
+ },
+ {
+ id: "read-security",
+ name: "Read",
+ arguments: { file_path: "/skills/account-security/SKILL.md" },
+ },
+ {
+ id: "read-refund-resource",
+ name: "loadSkillResource",
+ arguments: {
+ name: "refund-policy",
+ version: "3",
+ path: "references/refunds.md",
+ },
+ },
+ ],
+ },
+ ],
+ );
+
+ const requests = getSkillLoadRequests(available, result.toolCallsByName);
+ expect(requests.has(available[1])).toBe(false);
+ expect(requests.get(available[0])).toEqual(
+ new Set(["SKILL.md", "references/refunds.md"]),
+ );
+ expect(requests.get(available[2])).toEqual(
+ new Set(["/skills/account-security/SKILL.md"]),
+ );
+ },
+ );
+
+ it("does not infer a skill load from malformed, unrelated, or ambiguous calls", () => {
+ const available = [
+ { skillName: "refund-policy", skillVersion: 2 },
+ { skillName: "refund-policy", skillVersion: 3 },
+ { skillName: "shipping-policy" },
+ ];
+ const calls = [
+ { name: "loadSkill", arguments: { name: "refund-policy" } },
+ { name: "loadSkill", arguments: { name: "refund-policy", version: 4 } },
+ { name: "loadSkill", arguments: '{"name":' },
+ { name: "loadSkill", arguments: ["shipping-policy"] },
+ { name: "loadSkill", arguments: { name: "unknown-skill" } },
+ { name: "loadSkillResource", arguments: { name: "shipping-policy" } },
+ { name: "search", arguments: { name: "shipping-policy" } },
+ {
+ name: "Read",
+ arguments: { path: "/skills/shipping-policy-backup/SKILL.md" },
+ },
+ {
+ name: "Read",
+ arguments: { path: "/skills/shipping-policy/../other/SKILL.md" },
+ },
+ { name: "Read", arguments: { path: "/skills/shipping-policy/" } },
+ {
+ name: "Read",
+ arguments: { path: "/repo/shipping-policy/notes.txt" },
+ },
+ {
+ name: "readFile",
+ arguments: { path: "/repo/shipping-policy/notes.txt" },
+ },
+ {
+ name: "Read",
+ arguments: { path: "/skills/shipping-policy/refund-policy/SKILL.md" },
+ },
+ ].map((call, index) => ({ ...call, invocationNumber: index + 1 }));
+ const byName = new Map<string, typeof calls>();
+ for (const call of calls)
+ byName.set(call.name, [...(byName.get(call.name) ?? []), call]);
+
+ expect(getSkillLoadRequests(available, byName).size).toBe(0);
+ const explicitVersion = {
+ name: "loadSkill",
+ arguments: { name: "refund-policy", version: 3 },
+ invocationNumber: 12,
+ };
+ byName.set("loadSkill", [explicitVersion]);
+ expect(getSkillLoadRequests(available, byName).get(available[1])).toEqual(
+ new Set(["SKILL.md"]),
+ );
+ });
+
+ it.each(["invalid", "0", "-1", "3.5", null, {}])(
+ "ignores an invalid requested skill version: %j",
+ (version) => {
+ const available = [{ skillName: "refund-policy", skillVersion: 3 }];
+ const call = {
+ name: "loadSkill",
+ arguments: { name: "refund-policy", version },
+ invocationNumber: 1,
+ };
+ expect(
+ getSkillLoadRequests(available, new Map([[call.name, [call]]])).size,
+ ).toBe(0);
+ },
+ );
+
+ it.each([
+ ["Skill", { skill: "refund-policy" }, "SKILL.md"],
+ ["skill", { name: "refund-policy" }, "SKILL.md"],
+ [
+ "skill_read",
+ { skillName: "refund-policy", path: "references/refunds.md" },
+ "references/refunds.md",
+ ],
+ [
+ "readFile",
+ { path: "/skills/refund-policy/SKILL.md" },
+ "/skills/refund-policy/SKILL.md",
+ ],
+ ["langfuse_loadSkill", { name: "refund-policy" }, "SKILL.md"],
+ [
+ "langfuse_loadSkillResource",
+ { name: "refund-policy", path: "SKILL.md" },
+ "SKILL.md",
+ ],
+ ])("recognizes the %s skill-loading adapter", (name, args, filePath) => {
+ const available = [{ skillName: "refund-policy" }];
+ const call = { name, arguments: args, invocationNumber: 1 };
+ expect(
+ getSkillLoadRequests(available, new Map([[name, [call]]])).get(
+ available[0],
+ ),
+ ).toEqual(new Set([filePath]));
+ });
+
it.each(parserImplementations)(
"$name parser groups output-side tool call arguments by tool name",
({ parse, expectedArguments }) => {