🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(skills): add usage metrics tab (#18947)
tobi_kl5537191911ec
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.tsindex 2945e0fc96..b483d28347 100644--- a/packages/shared/src/features/query/dataModel.ts+++ b/packages/shared/src/features/query/dataModel.ts@@ -1396,6 +1396,30 @@ export const eventsObservationsView: ViewDeclarationType = {explodeArray: true,description: "Names of skills made available.",},+ skillIdsAvailable: {+ sql: "arrayDistinct(events_observations.skills_available.langfuseSkillId)",+ alias: "skillIdsAvailable",+ type: "arrayString",+ explodeArray: true,+ uiHidden: true,+ description: "Langfuse IDs of skill versions made available.",+ },+ skillIdsLoaded: {+ sql: "arrayDistinct(events_observations.skills_resource_loaded.langfuseSkillId)",+ alias: "skillIdsLoaded",+ type: "arrayString",+ explodeArray: true,+ uiHidden: true,+ description: "Langfuse IDs of skill versions with loaded files.",+ },+ loadedFilePaths: {+ sql: "arrayDistinct(events_observations.skills_resource_loaded.filePath)",+ alias: "loadedFilePaths",+ type: "arrayString",+ explodeArray: true,+ uiHidden: true,+ description: "Paths of loaded skill files.",+ },toolNames: {sql: "mapKeys(events_observations.tool_definitions)",alias: "toolNames",diff --git a/web/src/features/skills/components/SkillAnalytics.tsx b/web/src/features/skills/components/SkillAnalytics.tsxnew file mode 100644index 0000000000..f3cc2d451d--- /dev/null+++ b/web/src/features/skills/components/SkillAnalytics.tsx@@ -0,0 +1,177 @@+import { RefreshCw } from "lucide-react";+import {+ Table,+ type AsyncTableData,+} from "@/src/components/design-system/table/Table";+import { createTextTableColumn } from "@/src/components/design-system/table/columns/createTextTableColumn";+import { createNumberTableColumn } from "@/src/components/design-system/table/columns/createNumberTableColumn";+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";+import { StatusMessage } from "@/src/components/design-system/StatusMessage/StatusMessage";+import { Skeleton } from "@/src/components/ui/skeleton";+import { TableHeaderControls } from "@/src/components/table/table-header-controls";+import { PageHeaderControlsPortal } from "@/src/components/layouts/page-header-controls-slot";+import { useTableDateRange } from "@/src/hooks/useTableDateRange";+import { rangeToString, type TimeRange } from "@/src/utils/date-range-utils";+import { useSkillAnalytics } from "@/src/features/skills/hooks/useSkillAnalytics";+type SkillAnalyticsProps = {+ projectId: string;+ skillId: string;+ paths: string[];+};+type FileUsage = {+ path: string;+ loads: number;+};++export function SkillAnalytics(props: SkillAnalyticsProps) {+ const { timeRange, setTimeRange } = useTableDateRange(props.projectId);+ return (+ <div className="flex min-h-0 flex-1 flex-col overflow-auto">+ <TableHeaderControls timeRange={timeRange} setTimeRange={setTimeRange} />+ <SkillAnalyticsDashboard+ key={rangeToString(timeRange)}+ {...props}+ timeRange={timeRange}+ />+ </div>+ );+}++function SkillAnalyticsDashboard({+ projectId,+ skillId,+ paths,+ timeRange,+}: SkillAnalyticsProps & { timeRange: TimeRange }) {+ const { available, loaded, files, handleRefresh } = useSkillAnalytics({+ projectId,+ skillId,+ timeRange,+ });+ const fileMetrics = new Map(+ (files.data ?? []).map((file) => [String(file.loadedFilePaths), file]),+ );+ const fileRows = paths+ .map((path): FileUsage => {+ const file = fileMetrics.get(path);+ return {+ path,+ loads: Number(file?.count_count ?? 0),+ };+ })+ .toSorted((a, b) => b.loads - a.loads || a.path.localeCompare(b.path));+ const error = available.error ?? loaded.error ?? files.error;+ const fileTableData = prepareFileTableData(+ files.isPending,+ files.error,+ fileRows,+ );+ return (+ <>+ <PageHeaderControlsPortal>+ <IconButton+ icon={RefreshCw}+ label="Refresh metrics"+ title="Refresh metrics"+ variant="outline"+ onClick={handleRefresh}+ disabled={+ available.isFetching || loaded.isFetching || files.isFetching+ }+ />+ </PageHeaderControlsPortal>+ <div className="flex flex-col gap-5 p-4">+ {error ? (+ <StatusMessage+ level="ERROR"+ size="compact"+ message={`Could not load some metrics: ${error.message}. Use Refresh metrics to retry.`}+ />+ ) : null}+ <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">+ <Metric+ label="Available"+ value={+ available.data+ ? Number(available.data[0]?.count_count ?? 0)+ : null+ }+ loading={available.isPending}+ description="Observations offering this skill version"+ />+ <Metric+ label="Loaded"+ value={+ loaded.data ? Number(loaded.data[0]?.count_count ?? 0) : null+ }+ loading={loaded.isPending}+ description="Observations loading one or more files"+ />+ </div>+ <section className="flex flex-col gap-3 rounded-lg border pt-4">+ <div className="flex flex-col gap-1 px-4">+ <h3 className="text-sm font-bold">File usage</h3>+ </div>+ <div className="ph-no-capture max-h-80 min-h-32 overflow-auto">+ <Table<FileUsage>+ tableName="Skill file usage"+ columns={[+ createTextTableColumn<FileUsage>({+ accessorKey: "path",+ header: "File",+ size: 320,+ }),+ createNumberTableColumn<FileUsage>({+ accessorKey: "loads",+ header: "Loads",+ size: 110,+ formatter: formatCount,+ }),+ ]}+ data={fileTableData}+ />+ </div>+ </section>+ </div>+ </>+ );+}++function Metric({+ label,+ value,+ description,+ loading,+}: {+ label: string;+ value: number | null;+ description: string;+ loading: boolean;+}) {+ const displayValue = value === null ? "—" : formatCount(value);+ return (+ <div className="flex flex-col gap-2 rounded-lg border p-4">+ <p className="text-muted-foreground text-xs">{label}</p>+ {loading ? (+ <Skeleton className="h-8 w-24" />+ ) : (+ <p className="text-2xl font-bold tabular-nums">{displayValue}</p>+ )}+ <p className="text-muted-foreground text-xs">{description}</p>+ </div>+ );+}++function prepareFileTableData(+ loading: boolean,+ error: { message: string } | null,+ rows: FileUsage[],+): AsyncTableData<FileUsage[]> {+ if (loading) return { status: "loading" };+ if (error) return { status: "error", error: error.message };+ return { status: "success", data: rows };+}++function formatCount(value: number) {+ return value.toLocaleString(undefined, { maximumFractionDigits: 0 });+}diff --git a/web/src/features/skills/components/SkillEditor.tsx b/web/src/features/skills/components/SkillEditor.tsxindex 531fc6ad62..ff2963556b 100644--- a/web/src/features/skills/components/SkillEditor.tsx+++ b/web/src/features/skills/components/SkillEditor.tsx@@ -6,6 +6,7 @@ import {useState,} from "react";import { useMediaQuery } from "react-responsive";+import dynamic from "next/dynamic";import { StringParam, useQueryParam, withDefault } from "use-query-params";import { MAX_SKILL_FILES } from "@langfuse/shared";import { useStore } from "zustand";@@ -61,6 +62,7 @@ import {import { SkillVersionHistory } from "@/src/features/skills/components/SkillVersionHistory";import { SkillFileView } from "@/src/features/skills/components/SkillFileView";import { useSkillFileContents } from "@/src/features/skills/hooks/useSkillFileContents";+import { useLangfuseV4WriteMode } from "@/src/features/organizations/hooks";import {parseSkillFrontmatterMetadata,SKILL_NAME_RULES,@@ -68,6 +70,12 @@ import {import { api } from "@/src/utils/api";import { cn } from "@/src/utils/tailwind";+const SkillAnalytics = dynamic(() =>+ import("@/src/features/skills/components/SkillAnalytics").then(+ (module) => module.SkillAnalytics,+ ),+);+export function SkillEditor({projectId,store,@@ -84,7 +92,12 @@ export function SkillEditor({canCreate: boolean;view:| { kind: "draft"; onDiscard: () => Promise<void> }- | { kind: "version"; hasDraft: boolean; onEdit: () => Promise<void> };+ | {+ kind: "version";+ skillId: string;+ hasDraft: boolean;+ onEdit: () => Promise<void>;+ };onCreated: (created: { name: string; version: number }) => Promise<void>;history:| { kind: "new" }@@ -174,12 +187,18 @@ export function SkillEditor({const [isSaving, setIsSaving] = useState(false);const [isDownloading, setIsDownloading] = useState(false);const isDraft = view.kind === "draft";+ const v4WriteMode = useLangfuseV4WriteMode();+ const canReadMetrics = v4WriteMode !== null && v4WriteMode !== "legacy";const [selectedTab, setSelectedTab] = useQueryParam("tab",withDefault(StringParam, "files"),);const activeTab =- !isDraft && selectedTab === "linked-traces" ? "linked-traces" : "files";+ !isDraft &&+ (selectedTab === "linked-traces" ||+ (selectedTab === "metrics" && canReadMetrics))+ ? selectedTab+ : "files";const canEditFiles = isDraft && canCreate && !isSaving;const isDesktop = useMediaQuery({ query: "(min-width: 768px)" });const [treeState, setTreeState] = useState<SkillFileExplorerState>(() => ({@@ -661,12 +680,28 @@ export function SkillEditor({><Tabs.List variant="underline"><Tabs.Trigger value="files" label="Files" />+ <Tabs.Trigger+ value="metrics"+ label="Metrics"+ disabled={isDraft || !canReadMetrics}+ /><Tabs.Triggervalue="linked-traces"- label="Traces"+ label="Observations"disabled={isDraft}/></Tabs.List>+ {canReadMetrics &&+ view.kind === "version" &&+ baseVersion !== null ? (+ <Tabs.Content value="metrics" layout="fill">+ <SkillAnalytics+ projectId={projectId}+ skillId={view.skillId}+ paths={Object.keys(store.getState().files).toSorted()}+ />+ </Tabs.Content>+ ) : null}{!isDraft && baseVersion !== null ? (<Tabs.Content value="linked-traces" layout="fill"><SkillLinkedTracesdiff --git a/web/src/features/skills/components/SkillEditorPage.tsx b/web/src/features/skills/components/SkillEditorPage.tsxindex 045c313a32..26c55e0059 100644--- a/web/src/features/skills/components/SkillEditorPage.tsx+++ b/web/src/features/skills/components/SkillEditorPage.tsx@@ -255,6 +255,7 @@ function ExistingSkillSession({}: {kind: "version",+ skillId: skill.data.id,hasDraft: draftStore !== null,onEdit: selectDraft,}@@ -289,9 +290,10 @@ function ExistingSkillSession({projectId,skillName,version: selectedVersion,- ...(router.query.tab === "linked-traces"+ ...(router.query.tab === "linked-traces" ||+ router.query.tab === "metrics"? {- tab: "linked-traces",+ tab: router.query.tab,...(router.query.dateRange? { dateRange: router.query.dateRange }: {}),diff --git a/web/src/features/skills/hooks/useSkillAnalytics.ts b/web/src/features/skills/hooks/useSkillAnalytics.tsnew file mode 100644index 0000000000..7300fa99f6--- /dev/null+++ b/web/src/features/skills/hooks/useSkillAnalytics.ts@@ -0,0 +1,84 @@+import { useState } from "react";+import { api, type RouterInputs } from "@/src/utils/api";+import {+ toAbsoluteTimeRange,+ type TimeRange,+} from "@/src/utils/date-range-utils";++type MetricsInput = RouterInputs["dashboard"]["executeQuery"];+type MetricsWindow = { fromTimestamp: string; toTimestamp: string };++const queryOptions = { refetchOnWindowFocus: false, staleTime: 60_000 };++export function useSkillAnalytics({+ projectId,+ skillId,+ timeRange,+}: {+ projectId: string;+ skillId: string;+ timeRange: TimeRange;+}) {+ // Freeze relative time until the range changes or analytics are refreshed.+ const [window, setWindow] = useState(() => getMetricsWindow(timeRange));+ const available = api.dashboard.executeQuery.useQuery(+ createMetricsQuery(projectId, skillId, window, "skillIdsAvailable"),+ queryOptions,+ );+ const loaded = api.dashboard.executeQuery.useQuery(+ createMetricsQuery(projectId, skillId, window, "skillIdsLoaded"),+ queryOptions,+ );+ const files = api.dashboard.executeQuery.useQuery(+ createMetricsQuery(projectId, skillId, window, "skillIdsLoaded", [+ { field: "loadedFilePaths" },+ ]),+ queryOptions,+ );++ async function handleRefresh() {+ if ("from" in timeRange) {+ await Promise.all([+ available.refetch(),+ loaded.refetch(),+ files.refetch(),+ ]);+ return;+ }+ setWindow(getMetricsWindow(timeRange));+ }++ return { available, loaded, files, handleRefresh };+}++function createMetricsQuery(+ projectId: string,+ skillId: string,+ window: MetricsWindow,+ column: "skillIdsAvailable" | "skillIdsLoaded",+ dimensions: MetricsInput["query"]["dimensions"] = [],+): MetricsInput {+ return {+ projectId,+ version: "v2",+ query: {+ view: "observations",+ dimensions,+ metrics: [{ measure: "count", aggregation: "count" }],+ filters: [+ { type: "arrayOptions", column, operator: "any of", value: [skillId] },+ ],+ timeDimension: null,+ ...window,+ orderBy: null,+ },+ };+}++function getMetricsWindow(timeRange: TimeRange): MetricsWindow {+ const range = toAbsoluteTimeRange(timeRange);+ return {+ fromTimestamp: (range?.from ?? new Date(0)).toISOString(),+ toTimestamp: (range?.to ?? new Date()).toISOString(),+ };+}