🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(tracing): show prompt cache hit rate (#18669)
Lysander Kiesel90d994202fb7
diff --git a/web/src/features/traces/components/BreakdownTooltip.stories.tsx b/web/src/features/traces/components/BreakdownTooltip.stories.tsxindex d1515ac07a..1076f1e636 100644--- a/web/src/features/traces/components/BreakdownTooltip.stories.tsx+++ b/web/src/features/traces/components/BreakdownTooltip.stories.tsx@@ -1,4 +1,4 @@-import { expect, waitFor, within } from "storybook/test";+import { expect, userEvent, waitFor, within } from "storybook/test";import preview from "../../../../.storybook/preview";import { BreakdownTooltip } from "./BreakdownTooltip";@@ -54,6 +54,14 @@ const nearDecimalCostDetails = {total: 0.058342999997,};+const promptCacheUsage = {+ cache_read_input_tokens: 208_706,+ cache_creation_input_tokens: 41_313,+ input: 952,+ output: 2_254,+ total: 253_225,+};+const priceSource = {projectId: "project-1",modelId: "gpt-5.6/priority",@@ -73,6 +81,13 @@ export const Usage = meta.story({},});+export const WithCacheHitRate = meta.story({+ args: {+ details: promptCacheUsage,+ children: <span>253,225 tokens</span>,+ },+});+export const Cost = meta.story({args: {details: costDetails,@@ -148,7 +163,7 @@ async function openBreakdownTooltip(: null;if (!tooltip) throw new Error("Tooltip content was not rendered");- return { trigger, content: within(tooltip) };+ return { trigger, content: within(tooltip), tooltip };}export const TestLinksMatchedPricingTier = meta.story({@@ -264,3 +279,202 @@ export const TestShareOfTotal = meta.story({await expect(content.getByText("$0.04")).toBeInTheDocument();},});++export const TestCacheHitRate = meta.story({+ name: "(Test) Shows cache hit rate after input usage details",+ args: {+ details: promptCacheUsage,+ children: <span>253,225 tokens</span>,+ },+ play: async ({ canvasElement }) => {+ const { trigger, content, tooltip } = await openBreakdownTooltip(+ canvasElement,+ "253,225 tokens",+ );+ const text = tooltip.textContent ?? "";++ await expect(content.getByText("83.2%")).toBeInTheDocument();+ await expect(text.indexOf("Input usage")).toBeLessThan(+ text.indexOf("Cache hit rate"),+ );+ await expect(text.lastIndexOf("input")).toBeLessThan(+ text.indexOf("Cache hit rate"),+ );+ await expect(text.indexOf("Cache hit rate")).toBeLessThan(+ text.indexOf("Output usage"),+ );++ const help = content.getByRole("button", {+ name: "How cache hit rate is calculated",+ });+ await userEvent.hover(help);+ await expect(+ await within(canvasElement.ownerDocument.body).findByText(+ "Cache-read tokens divided by total input tokens.",+ ),+ ).toBeInTheDocument();+ await expect(content.getByText("Cache hit rate")).toBeInTheDocument();++ await userEvent.unhover(help);+ await waitFor(() =>+ expect(+ within(canvasElement.ownerDocument.body).queryByText(+ "Cache-read tokens divided by total input tokens.",+ ),+ ).not.toBeInTheDocument(),+ );+ trigger.focus();+ await userEvent.tab();+ await expect(+ canvasElement.ownerDocument.activeElement,+ ).toHaveAccessibleName("How cache hit rate is calculated");+ await expect(canvasElement.ownerDocument.activeElement?.tagName).toBe(+ "BUTTON",+ );+ await expect(+ await within(canvasElement.ownerDocument.body).findByText(+ "Cache-read tokens divided by total input tokens.",+ ),+ ).toBeInTheDocument();+ await expect(content.getByText("Cache hit rate")).toBeInTheDocument();+ },+});++export const TestZeroCacheHitRate = meta.story({+ name: "(Test) Shows zero cache hit rate when no tokens were read",+ args: {+ details: {+ cache_read_input_tokens: 0,+ cache_creation_input_tokens: 40,+ input: 60,+ output: 10,+ total: 110,+ },+ children: <span>110 tokens</span>,+ },+ play: async ({ canvasElement }) => {+ const { content } = await openBreakdownTooltip(canvasElement, "110 tokens");+ await expect(content.getByText("Cache hit rate")).toBeInTheDocument();+ await expect(content.getByText("0.0%")).toBeInTheDocument();+ },+});++export const TestCacheReadAliases = meta.story({+ name: "(Test) Distinguishes cache-read aliases from separate counts",+ render: () => (+ <div className="flex flex-col gap-2">+ <BreakdownTooltip+ details={{ input: 0, input_cached_tokens: 900, input_cache_read: 900 }}+ >+ <span>Bedrock usage with duplicate cache reads</span>+ </BreakdownTooltip>+ <BreakdownTooltip+ details={{+ input: 100,+ input_cached_tokens: 100,+ input_cached_text_tokens: 60,+ input_cached_audio_tokens: 40,+ }}+ >+ <span>Cached total with text and audio breakdown</span>+ </BreakdownTooltip>+ <BreakdownTooltip+ details={[+ { input: 50, input_cached_tokens: 50 },+ { input: 50, input_cache_read: 50 },+ ]}+ >+ <span>Different read keys across generations</span>+ </BreakdownTooltip>+ <BreakdownTooltip+ details={{+ input: 100,+ input_cached_text_tokens: 30,+ input_cached_audio_tokens: 70,+ }}+ >+ <span>Cached text and audio</span>+ </BreakdownTooltip>+ </div>+ ),+ play: async ({ canvasElement }) => {+ const ambiguous = await openBreakdownTooltip(+ canvasElement,+ "Bedrock usage with duplicate cache reads",+ );+ await expect(+ ambiguous.content.queryByText("Cache hit rate"),+ ).not.toBeInTheDocument();++ const totalAndBreakdown = await openBreakdownTooltip(+ canvasElement,+ "Cached total with text and audio breakdown",+ );+ await expect(+ totalAndBreakdown.content.queryByText("Cache hit rate"),+ ).not.toBeInTheDocument();++ const aggregated = await openBreakdownTooltip(+ canvasElement,+ "Different read keys across generations",+ );+ await expect(aggregated.content.getByText("50.0%")).toBeInTheDocument();++ const modalities = await openBreakdownTooltip(+ canvasElement,+ "Cached text and audio",+ );+ await expect(modalities.content.getByText("50.0%")).toBeInTheDocument();+ },+});++const hiddenCacheUsage = [+ ["no cache tokens", { input: 100, output: 20, total: 120 }],+ ["zero input", { cache_read_input_tokens: 0, input: 0, output: 1, total: 1 }],+ [+ "cache tokens without input",+ { input_cached_tokens: 100, output: 20, total: 120 },+ ],+ [+ "inclusive input and cache tokens",+ { input: 100, cache_read_input_tokens: 80, output: 50, total: 150 },+ ],+ [+ "unnormalized cached content",+ { input: 100, cached_content_token_count: 80, output: 10, total: 190 },+ ],+ [+ "unnormalized cache writes",+ { input: 90, cache_write_tokens: 10, output: 5, total: 105 },+ ],+] as const;++export const TestHiddenCacheHitRate = meta.story({+ name: "(Test) Hides cache hit rate for incomplete or inconsistent usage",+ render: () => (+ <div className="flex flex-col gap-2">+ {hiddenCacheUsage.map(([label, details]) => (+ <BreakdownTooltip key={label} details={details}>+ <span>{label}</span>+ </BreakdownTooltip>+ ))}+ <BreakdownTooltip details={promptCacheUsage} isCost>+ <span>cost breakdown</span>+ </BreakdownTooltip>+ </div>+ ),+ play: async ({ canvasElement }) => {+ for (const [label] of hiddenCacheUsage) {+ const { content } = await openBreakdownTooltip(canvasElement, label);+ await expect(+ content.queryByText("Cache hit rate"),+ ).not.toBeInTheDocument();+ }++ const { content } = await openBreakdownTooltip(+ canvasElement,+ "cost breakdown",+ );+ await expect(content.queryByText("Cache hit rate")).not.toBeInTheDocument();+ },+});diff --git a/web/src/features/traces/components/BreakdownTooltip.tsx b/web/src/features/traces/components/BreakdownTooltip.tsxindex 65fc66c98d..63b61a5d75 100644--- a/web/src/features/traces/components/BreakdownTooltip.tsx+++ b/web/src/features/traces/components/BreakdownTooltip.tsx@@ -7,8 +7,10 @@ import {import { useState } from "react";import Decimal from "decimal.js";import Link from "next/link";+import { CustomTooltip } from "@/src/components/design-system/CustomTooltip/CustomTooltip";+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";import { type Details } from "@/src/features/traces/fns/calculateAggregatedUsage";-import { ExternalLink } from "lucide-react";+import { ExternalLink, InfoIcon } from "lucide-react";import { centCostFormatter, usdFormatter } from "@/src/utils/numbers";import { cva, type VariantProps } from "class-variance-authority";import { cn } from "@/src/utils/tailwind";@@ -82,6 +84,7 @@ export const BreakdownTooltip = ({? usdFormatter(value, costFractionDigits, costFractionDigits): undefined;const shareOf = createShareOf(contributionEntries);+ const cacheHitRate = isCost ? null : getCacheHitRate(inputEntries, details);const displayedTotal =aggregatedDetails.total ??(isCost ? sumEntries(contributionEntries) : new Decimal(0));@@ -155,6 +158,11 @@ export const BreakdownTooltip = ({<Sectiontitle={isCost ? "Input cost" : "Input usage"}entries={inputEntries}+ lastRow={+ cacheHitRate === null ? null : (+ <CacheHitRateRow rate={cacheHitRate} />+ )+ }{...sectionProps}/>@@ -269,6 +277,7 @@ interface SectionProps {shareOf: (value: number | Decimal) => number;/** Hairline above the section, matching the total row. */separated?: boolean;+ lastRow?: React.ReactNode;}const Section = ({@@ -278,6 +287,7 @@ const Section = ({formatExactValue,shareOf,separated = false,+ lastRow,}: SectionProps) => {const sectionTotal = sumEntries(entries);@@ -305,10 +315,110 @@ const Section = ({share={shareOf(value ?? 0)}/>))}+ {lastRow}</div>);};+function CacheHitRateRow({ rate }: { rate: number }) {+ return (+ <div className="text-muted-foreground col-span-4 grid grid-cols-subgrid items-center text-sm leading-5">+ <span className="col-span-2 inline-flex items-center gap-1">+ Cache hit rate+ <CacheHitRateInfo />+ </span>+ <span className="text-foreground text-right font-mono text-xs tabular-nums">+ {formatCacheHitRate(rate)}+ </span>+ </div>+ );+}++function CacheHitRateInfo() {+ return (+ <CustomTooltip+ placement="top"+ content={+ <div className="space-y-1.5">+ <p>Cache-read tokens divided by total input tokens.</p>+ <p>+ For multi-turn interactions, above 90% is common when later turns+ keep a stable prefix, and about 70% for a single turn with a fixed+ rubric.+ </p>+ </div>+ }+ >+ {({ getTriggerProps }) => (+ <IconButton+ {...getTriggerProps()}+ icon={InfoIcon}+ label="How cache hit rate is calculated"+ size="xs"+ variant="ghost"+ />+ )}+ </CustomTooltip>+ );+}++function getCacheHitRate(+ entries: [string, number | undefined][],+ details: Details | Details[],+) {+ const usageDetails = Array.isArray(details) ? details : [details];+ const hasInconsistentTotal = usageDetails.some(({ total, ...buckets }) => {+ if (total === undefined) return false;+ return (+ !isUsableTokenCount(total) ||+ sumEntries(Object.entries(buckets)).gt(total)+ );+ });+ if (hasInconsistentTotal) return null;++ /** Generic read totals may overlap with each other or with modality counts. */+ const hasOverlappingCacheReads = usageDetails.some((detail) => {+ const readKeys = Object.entries(detail)+ .filter(+ ([key, value]) =>+ key.includes("input") && isCacheReadKey(key) && (value ?? 0) > 0,+ )+ .map(([key]) => key);+ return (+ readKeys.length > 1 &&+ readKeys.some(+ (key) => !/^input_cached_(text|audio|image)_tokens$/.test(key),+ )+ );+ });+ if (hasOverlappingCacheReads) return null;+ if (!entries.some(([key]) => /cache/i.test(key))) return null;+ if (!entries.some(([key]) => key === "input")) return null;+ if (entries.some(([, value]) => !isUsableTokenCount(value))) return null;++ const total = sumEntries(entries);+ if (total.lte(0)) return null;++ const cacheRead = sumEntries(entries.filter(([key]) => isCacheReadKey(key)));+ if (cacheRead.isNeg()) return null;++ const rate = cacheRead.div(total);+ return rate.toNumber();+}++function isUsableTokenCount(value: number | undefined) {+ return typeof value === "number" && Number.isFinite(value);+}++function isCacheReadKey(key: string) {+ if (/creation|cache_write/i.test(key)) return false;+ return /cache_read|cached/i.test(key);+}++function formatCacheHitRate(rate: number) {+ return `${(rate * 100).toFixed(1)}%`;+}+function sortEntriesByValue(entries: [string, number | undefined][]) {return entries.toSorted(([, a], [, b]) => (b ?? 0) - (a ?? 0));}