Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(skills): add usage metrics tab (#18947)

tobi_kl5537191911ec
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.ts
index 2945e0fc96..b483d28347 100644
--- a/packages/shared/src/features/query/dataModel.ts
+++ b/packages/shared/src/features/query/dataModel.ts
@@ -1396,6 +1396,30 @@ export const eventsObservationsView: ViewDeclarationType = {
explodeArray: true,
description: "Names of skills made available.",
},
+ skillIdsAvailable: {
+ sql: "arrayDistinct(events_observations.skills_available.langfuseSkillId)",
+ alias: "skillIdsAvailable",
+ type: "arrayString",
+ explodeArray: true,
+ uiHidden: true,
+ description: "Langfuse IDs of skill versions made available.",
+ },
+ skillIdsLoaded: {
+ sql: "arrayDistinct(events_observations.skills_resource_loaded.langfuseSkillId)",
+ alias: "skillIdsLoaded",
+ type: "arrayString",
+ explodeArray: true,
+ uiHidden: true,
+ description: "Langfuse IDs of skill versions with loaded files.",
+ },
+ loadedFilePaths: {
+ sql: "arrayDistinct(events_observations.skills_resource_loaded.filePath)",
+ alias: "loadedFilePaths",
+ type: "arrayString",
+ explodeArray: true,
+ uiHidden: true,
+ description: "Paths of loaded skill files.",
+ },
toolNames: {
sql: "mapKeys(events_observations.tool_definitions)",
alias: "toolNames",
diff --git a/web/src/features/skills/components/SkillAnalytics.tsx b/web/src/features/skills/components/SkillAnalytics.tsx
new file mode 100644
index 0000000000..f3cc2d451d
--- /dev/null
+++ b/web/src/features/skills/components/SkillAnalytics.tsx
@@ -0,0 +1,177 @@
+import { RefreshCw } from "lucide-react";
+import {
+ Table,
+ type AsyncTableData,
+} from "@/src/components/design-system/table/Table";
+import { createTextTableColumn } from "@/src/components/design-system/table/columns/createTextTableColumn";
+import { createNumberTableColumn } from "@/src/components/design-system/table/columns/createNumberTableColumn";
+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
+import { StatusMessage } from "@/src/components/design-system/StatusMessage/StatusMessage";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { TableHeaderControls } from "@/src/components/table/table-header-controls";
+import { PageHeaderControlsPortal } from "@/src/components/layouts/page-header-controls-slot";
+import { useTableDateRange } from "@/src/hooks/useTableDateRange";
+import { rangeToString, type TimeRange } from "@/src/utils/date-range-utils";
+import { useSkillAnalytics } from "@/src/features/skills/hooks/useSkillAnalytics";
+type SkillAnalyticsProps = {
+ projectId: string;
+ skillId: string;
+ paths: string[];
+};
+type FileUsage = {
+ path: string;
+ loads: number;
+};
+
+export function SkillAnalytics(props: SkillAnalyticsProps) {
+ const { timeRange, setTimeRange } = useTableDateRange(props.projectId);
+ return (
+ <div className="flex min-h-0 flex-1 flex-col overflow-auto">
+ <TableHeaderControls timeRange={timeRange} setTimeRange={setTimeRange} />
+ <SkillAnalyticsDashboard
+ key={rangeToString(timeRange)}
+ {...props}
+ timeRange={timeRange}
+ />
+ </div>
+ );
+}
+
+function SkillAnalyticsDashboard({
+ projectId,
+ skillId,
+ paths,
+ timeRange,
+}: SkillAnalyticsProps & { timeRange: TimeRange }) {
+ const { available, loaded, files, handleRefresh } = useSkillAnalytics({
+ projectId,
+ skillId,
+ timeRange,
+ });
+ const fileMetrics = new Map(
+ (files.data ?? []).map((file) => [String(file.loadedFilePaths), file]),
+ );
+ const fileRows = paths
+ .map((path): FileUsage => {
+ const file = fileMetrics.get(path);
+ return {
+ path,
+ loads: Number(file?.count_count ?? 0),
+ };
+ })
+ .toSorted((a, b) => b.loads - a.loads || a.path.localeCompare(b.path));
+ const error = available.error ?? loaded.error ?? files.error;
+ const fileTableData = prepareFileTableData(
+ files.isPending,
+ files.error,
+ fileRows,
+ );
+ return (
+ <>
+ <PageHeaderControlsPortal>
+ <IconButton
+ icon={RefreshCw}
+ label="Refresh metrics"
+ title="Refresh metrics"
+ variant="outline"
+ onClick={handleRefresh}
+ disabled={
+ available.isFetching || loaded.isFetching || files.isFetching
+ }
+ />
+ </PageHeaderControlsPortal>
+ <div className="flex flex-col gap-5 p-4">
+ {error ? (
+ <StatusMessage
+ level="ERROR"
+ size="compact"
+ message={`Could not load some metrics: ${error.message}. Use Refresh metrics to retry.`}
+ />
+ ) : null}
+ <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
+ <Metric
+ label="Available"
+ value={
+ available.data
+ ? Number(available.data[0]?.count_count ?? 0)
+ : null
+ }
+ loading={available.isPending}
+ description="Observations offering this skill version"
+ />
+ <Metric
+ label="Loaded"
+ value={
+ loaded.data ? Number(loaded.data[0]?.count_count ?? 0) : null
+ }
+ loading={loaded.isPending}
+ description="Observations loading one or more files"
+ />
+ </div>
+ <section className="flex flex-col gap-3 rounded-lg border pt-4">
+ <div className="flex flex-col gap-1 px-4">
+ <h3 className="text-sm font-bold">File usage</h3>
+ </div>
+ <div className="ph-no-capture max-h-80 min-h-32 overflow-auto">
+ <Table<FileUsage>
+ tableName="Skill file usage"
+ columns={[
+ createTextTableColumn<FileUsage>({
+ accessorKey: "path",
+ header: "File",
+ size: 320,
+ }),
+ createNumberTableColumn<FileUsage>({
+ accessorKey: "loads",
+ header: "Loads",
+ size: 110,
+ formatter: formatCount,
+ }),
+ ]}
+ data={fileTableData}
+ />
+ </div>
+ </section>
+ </div>
+ </>
+ );
+}
+
+function Metric({
+ label,
+ value,
+ description,
+ loading,
+}: {
+ label: string;
+ value: number | null;
+ description: string;
+ loading: boolean;
+}) {
+ const displayValue = value === null ? "—" : formatCount(value);
+ return (
+ <div className="flex flex-col gap-2 rounded-lg border p-4">
+ <p className="text-muted-foreground text-xs">{label}</p>
+ {loading ? (
+ <Skeleton className="h-8 w-24" />
+ ) : (
+ <p className="text-2xl font-bold tabular-nums">{displayValue}</p>
+ )}
+ <p className="text-muted-foreground text-xs">{description}</p>
+ </div>
+ );
+}
+
+function prepareFileTableData(
+ loading: boolean,
+ error: { message: string } | null,
+ rows: FileUsage[],
+): AsyncTableData<FileUsage[]> {
+ if (loading) return { status: "loading" };
+ if (error) return { status: "error", error: error.message };
+ return { status: "success", data: rows };
+}
+
+function formatCount(value: number) {
+ return value.toLocaleString(undefined, { maximumFractionDigits: 0 });
+}
diff --git a/web/src/features/skills/components/SkillEditor.tsx b/web/src/features/skills/components/SkillEditor.tsx
index 531fc6ad62..ff2963556b 100644
--- a/web/src/features/skills/components/SkillEditor.tsx
+++ b/web/src/features/skills/components/SkillEditor.tsx
@@ -6,6 +6,7 @@ import {
useState,
} from "react";
import { useMediaQuery } from "react-responsive";
+import dynamic from "next/dynamic";
import { StringParam, useQueryParam, withDefault } from "use-query-params";
import { MAX_SKILL_FILES } from "@langfuse/shared";
import { useStore } from "zustand";
@@ -61,6 +62,7 @@ import {
import { SkillVersionHistory } from "@/src/features/skills/components/SkillVersionHistory";
import { SkillFileView } from "@/src/features/skills/components/SkillFileView";
import { useSkillFileContents } from "@/src/features/skills/hooks/useSkillFileContents";
+import { useLangfuseV4WriteMode } from "@/src/features/organizations/hooks";
import {
parseSkillFrontmatterMetadata,
SKILL_NAME_RULES,
@@ -68,6 +70,12 @@ import {
import { api } from "@/src/utils/api";
import { cn } from "@/src/utils/tailwind";
+const SkillAnalytics = dynamic(() =>
+ import("@/src/features/skills/components/SkillAnalytics").then(
+ (module) => module.SkillAnalytics,
+ ),
+);
+
export function SkillEditor({
projectId,
store,
@@ -84,7 +92,12 @@ export function SkillEditor({
canCreate: boolean;
view:
| { kind: "draft"; onDiscard: () => Promise<void> }
- | { kind: "version"; hasDraft: boolean; onEdit: () => Promise<void> };
+ | {
+ kind: "version";
+ skillId: string;
+ hasDraft: boolean;
+ onEdit: () => Promise<void>;
+ };
onCreated: (created: { name: string; version: number }) => Promise<void>;
history:
| { kind: "new" }
@@ -174,12 +187,18 @@ export function SkillEditor({
const [isSaving, setIsSaving] = useState(false);
const [isDownloading, setIsDownloading] = useState(false);
const isDraft = view.kind === "draft";
+ const v4WriteMode = useLangfuseV4WriteMode();
+ const canReadMetrics = v4WriteMode !== null && v4WriteMode !== "legacy";
const [selectedTab, setSelectedTab] = useQueryParam(
"tab",
withDefault(StringParam, "files"),
);
const activeTab =
- !isDraft && selectedTab === "linked-traces" ? "linked-traces" : "files";
+ !isDraft &&
+ (selectedTab === "linked-traces" ||
+ (selectedTab === "metrics" && canReadMetrics))
+ ? selectedTab
+ : "files";
const canEditFiles = isDraft && canCreate && !isSaving;
const isDesktop = useMediaQuery({ query: "(min-width: 768px)" });
const [treeState, setTreeState] = useState<SkillFileExplorerState>(() => ({
@@ -661,12 +680,28 @@ export function SkillEditor({
>
<Tabs.List variant="underline">
<Tabs.Trigger value="files" label="Files" />
+ <Tabs.Trigger
+ value="metrics"
+ label="Metrics"
+ disabled={isDraft || !canReadMetrics}
+ />
<Tabs.Trigger
value="linked-traces"
- label="Traces"
+ label="Observations"
disabled={isDraft}
/>
</Tabs.List>
+ {canReadMetrics &&
+ view.kind === "version" &&
+ baseVersion !== null ? (
+ <Tabs.Content value="metrics" layout="fill">
+ <SkillAnalytics
+ projectId={projectId}
+ skillId={view.skillId}
+ paths={Object.keys(store.getState().files).toSorted()}
+ />
+ </Tabs.Content>
+ ) : null}
{!isDraft && baseVersion !== null ? (
<Tabs.Content value="linked-traces" layout="fill">
<SkillLinkedTraces
diff --git a/web/src/features/skills/components/SkillEditorPage.tsx b/web/src/features/skills/components/SkillEditorPage.tsx
index 045c313a32..26c55e0059 100644
--- a/web/src/features/skills/components/SkillEditorPage.tsx
+++ b/web/src/features/skills/components/SkillEditorPage.tsx
@@ -255,6 +255,7 @@ function ExistingSkillSession({
}
: {
kind: "version",
+ skillId: skill.data.id,
hasDraft: draftStore !== null,
onEdit: selectDraft,
}
@@ -289,9 +290,10 @@ function ExistingSkillSession({
projectId,
skillName,
version: selectedVersion,
- ...(router.query.tab === "linked-traces"
+ ...(router.query.tab === "linked-traces" ||
+ router.query.tab === "metrics"
? {
- tab: "linked-traces",
+ tab: router.query.tab,
...(router.query.dateRange
? { dateRange: router.query.dateRange }
: {}),
diff --git a/web/src/features/skills/hooks/useSkillAnalytics.ts b/web/src/features/skills/hooks/useSkillAnalytics.ts
new file mode 100644
index 0000000000..7300fa99f6
--- /dev/null
+++ b/web/src/features/skills/hooks/useSkillAnalytics.ts
@@ -0,0 +1,84 @@
+import { useState } from "react";
+import { api, type RouterInputs } from "@/src/utils/api";
+import {
+ toAbsoluteTimeRange,
+ type TimeRange,
+} from "@/src/utils/date-range-utils";
+
+type MetricsInput = RouterInputs["dashboard"]["executeQuery"];
+type MetricsWindow = { fromTimestamp: string; toTimestamp: string };
+
+const queryOptions = { refetchOnWindowFocus: false, staleTime: 60_000 };
+
+export function useSkillAnalytics({
+ projectId,
+ skillId,
+ timeRange,
+}: {
+ projectId: string;
+ skillId: string;
+ timeRange: TimeRange;
+}) {
+ // Freeze relative time until the range changes or analytics are refreshed.
+ const [window, setWindow] = useState(() => getMetricsWindow(timeRange));
+ const available = api.dashboard.executeQuery.useQuery(
+ createMetricsQuery(projectId, skillId, window, "skillIdsAvailable"),
+ queryOptions,
+ );
+ const loaded = api.dashboard.executeQuery.useQuery(
+ createMetricsQuery(projectId, skillId, window, "skillIdsLoaded"),
+ queryOptions,
+ );
+ const files = api.dashboard.executeQuery.useQuery(
+ createMetricsQuery(projectId, skillId, window, "skillIdsLoaded", [
+ { field: "loadedFilePaths" },
+ ]),
+ queryOptions,
+ );
+
+ async function handleRefresh() {
+ if ("from" in timeRange) {
+ await Promise.all([
+ available.refetch(),
+ loaded.refetch(),
+ files.refetch(),
+ ]);
+ return;
+ }
+ setWindow(getMetricsWindow(timeRange));
+ }
+
+ return { available, loaded, files, handleRefresh };
+}
+
+function createMetricsQuery(
+ projectId: string,
+ skillId: string,
+ window: MetricsWindow,
+ column: "skillIdsAvailable" | "skillIdsLoaded",
+ dimensions: MetricsInput["query"]["dimensions"] = [],
+): MetricsInput {
+ return {
+ projectId,
+ version: "v2",
+ query: {
+ view: "observations",
+ dimensions,
+ metrics: [{ measure: "count", aggregation: "count" }],
+ filters: [
+ { type: "arrayOptions", column, operator: "any of", value: [skillId] },
+ ],
+ timeDimension: null,
+ ...window,
+ orderBy: null,
+ },
+ };
+}
+
+function getMetricsWindow(timeRange: TimeRange): MetricsWindow {
+ const range = toAbsoluteTimeRange(timeRange);
+ return {
+ fromTimestamp: (range?.from ?? new Date(0)).toISOString(),
+ toTimestamp: (range?.to ?? new Date()).toISOString(),
+ };
+}