Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(tracing): show prompt cache hit rate (#18669)

Lysander Kiesel90d994202fb7
diff --git a/web/src/features/traces/components/BreakdownTooltip.stories.tsx b/web/src/features/traces/components/BreakdownTooltip.stories.tsx
index d1515ac07a..1076f1e636 100644
--- a/web/src/features/traces/components/BreakdownTooltip.stories.tsx
+++ b/web/src/features/traces/components/BreakdownTooltip.stories.tsx
@@ -1,4 +1,4 @@
-import { expect, waitFor, within } from "storybook/test";
+import { expect, userEvent, waitFor, within } from "storybook/test";
import preview from "../../../../.storybook/preview";
import { BreakdownTooltip } from "./BreakdownTooltip";
@@ -54,6 +54,14 @@ const nearDecimalCostDetails = {
total: 0.058342999997,
};
+const promptCacheUsage = {
+ cache_read_input_tokens: 208_706,
+ cache_creation_input_tokens: 41_313,
+ input: 952,
+ output: 2_254,
+ total: 253_225,
+};
+
const priceSource = {
projectId: "project-1",
modelId: "gpt-5.6/priority",
@@ -73,6 +81,13 @@ export const Usage = meta.story({
},
});
+export const WithCacheHitRate = meta.story({
+ args: {
+ details: promptCacheUsage,
+ children: <span>253,225 tokens</span>,
+ },
+});
+
export const Cost = meta.story({
args: {
details: costDetails,
@@ -148,7 +163,7 @@ async function openBreakdownTooltip(
: null;
if (!tooltip) throw new Error("Tooltip content was not rendered");
- return { trigger, content: within(tooltip) };
+ return { trigger, content: within(tooltip), tooltip };
}
export const TestLinksMatchedPricingTier = meta.story({
@@ -264,3 +279,202 @@ export const TestShareOfTotal = meta.story({
await expect(content.getByText("$0.04")).toBeInTheDocument();
},
});
+
+export const TestCacheHitRate = meta.story({
+ name: "(Test) Shows cache hit rate after input usage details",
+ args: {
+ details: promptCacheUsage,
+ children: <span>253,225 tokens</span>,
+ },
+ play: async ({ canvasElement }) => {
+ const { trigger, content, tooltip } = await openBreakdownTooltip(
+ canvasElement,
+ "253,225 tokens",
+ );
+ const text = tooltip.textContent ?? "";
+
+ await expect(content.getByText("83.2%")).toBeInTheDocument();
+ await expect(text.indexOf("Input usage")).toBeLessThan(
+ text.indexOf("Cache hit rate"),
+ );
+ await expect(text.lastIndexOf("input")).toBeLessThan(
+ text.indexOf("Cache hit rate"),
+ );
+ await expect(text.indexOf("Cache hit rate")).toBeLessThan(
+ text.indexOf("Output usage"),
+ );
+
+ const help = content.getByRole("button", {
+ name: "How cache hit rate is calculated",
+ });
+ await userEvent.hover(help);
+ await expect(
+ await within(canvasElement.ownerDocument.body).findByText(
+ "Cache-read tokens divided by total input tokens.",
+ ),
+ ).toBeInTheDocument();
+ await expect(content.getByText("Cache hit rate")).toBeInTheDocument();
+
+ await userEvent.unhover(help);
+ await waitFor(() =>
+ expect(
+ within(canvasElement.ownerDocument.body).queryByText(
+ "Cache-read tokens divided by total input tokens.",
+ ),
+ ).not.toBeInTheDocument(),
+ );
+ trigger.focus();
+ await userEvent.tab();
+ await expect(
+ canvasElement.ownerDocument.activeElement,
+ ).toHaveAccessibleName("How cache hit rate is calculated");
+ await expect(canvasElement.ownerDocument.activeElement?.tagName).toBe(
+ "BUTTON",
+ );
+ await expect(
+ await within(canvasElement.ownerDocument.body).findByText(
+ "Cache-read tokens divided by total input tokens.",
+ ),
+ ).toBeInTheDocument();
+ await expect(content.getByText("Cache hit rate")).toBeInTheDocument();
+ },
+});
+
+export const TestZeroCacheHitRate = meta.story({
+ name: "(Test) Shows zero cache hit rate when no tokens were read",
+ args: {
+ details: {
+ cache_read_input_tokens: 0,
+ cache_creation_input_tokens: 40,
+ input: 60,
+ output: 10,
+ total: 110,
+ },
+ children: <span>110 tokens</span>,
+ },
+ play: async ({ canvasElement }) => {
+ const { content } = await openBreakdownTooltip(canvasElement, "110 tokens");
+ await expect(content.getByText("Cache hit rate")).toBeInTheDocument();
+ await expect(content.getByText("0.0%")).toBeInTheDocument();
+ },
+});
+
+export const TestCacheReadAliases = meta.story({
+ name: "(Test) Distinguishes cache-read aliases from separate counts",
+ render: () => (
+ <div className="flex flex-col gap-2">
+ <BreakdownTooltip
+ details={{ input: 0, input_cached_tokens: 900, input_cache_read: 900 }}
+ >
+ <span>Bedrock usage with duplicate cache reads</span>
+ </BreakdownTooltip>
+ <BreakdownTooltip
+ details={{
+ input: 100,
+ input_cached_tokens: 100,
+ input_cached_text_tokens: 60,
+ input_cached_audio_tokens: 40,
+ }}
+ >
+ <span>Cached total with text and audio breakdown</span>
+ </BreakdownTooltip>
+ <BreakdownTooltip
+ details={[
+ { input: 50, input_cached_tokens: 50 },
+ { input: 50, input_cache_read: 50 },
+ ]}
+ >
+ <span>Different read keys across generations</span>
+ </BreakdownTooltip>
+ <BreakdownTooltip
+ details={{
+ input: 100,
+ input_cached_text_tokens: 30,
+ input_cached_audio_tokens: 70,
+ }}
+ >
+ <span>Cached text and audio</span>
+ </BreakdownTooltip>
+ </div>
+ ),
+ play: async ({ canvasElement }) => {
+ const ambiguous = await openBreakdownTooltip(
+ canvasElement,
+ "Bedrock usage with duplicate cache reads",
+ );
+ await expect(
+ ambiguous.content.queryByText("Cache hit rate"),
+ ).not.toBeInTheDocument();
+
+ const totalAndBreakdown = await openBreakdownTooltip(
+ canvasElement,
+ "Cached total with text and audio breakdown",
+ );
+ await expect(
+ totalAndBreakdown.content.queryByText("Cache hit rate"),
+ ).not.toBeInTheDocument();
+
+ const aggregated = await openBreakdownTooltip(
+ canvasElement,
+ "Different read keys across generations",
+ );
+ await expect(aggregated.content.getByText("50.0%")).toBeInTheDocument();
+
+ const modalities = await openBreakdownTooltip(
+ canvasElement,
+ "Cached text and audio",
+ );
+ await expect(modalities.content.getByText("50.0%")).toBeInTheDocument();
+ },
+});
+
+const hiddenCacheUsage = [
+ ["no cache tokens", { input: 100, output: 20, total: 120 }],
+ ["zero input", { cache_read_input_tokens: 0, input: 0, output: 1, total: 1 }],
+ [
+ "cache tokens without input",
+ { input_cached_tokens: 100, output: 20, total: 120 },
+ ],
+ [
+ "inclusive input and cache tokens",
+ { input: 100, cache_read_input_tokens: 80, output: 50, total: 150 },
+ ],
+ [
+ "unnormalized cached content",
+ { input: 100, cached_content_token_count: 80, output: 10, total: 190 },
+ ],
+ [
+ "unnormalized cache writes",
+ { input: 90, cache_write_tokens: 10, output: 5, total: 105 },
+ ],
+] as const;
+
+export const TestHiddenCacheHitRate = meta.story({
+ name: "(Test) Hides cache hit rate for incomplete or inconsistent usage",
+ render: () => (
+ <div className="flex flex-col gap-2">
+ {hiddenCacheUsage.map(([label, details]) => (
+ <BreakdownTooltip key={label} details={details}>
+ <span>{label}</span>
+ </BreakdownTooltip>
+ ))}
+ <BreakdownTooltip details={promptCacheUsage} isCost>
+ <span>cost breakdown</span>
+ </BreakdownTooltip>
+ </div>
+ ),
+ play: async ({ canvasElement }) => {
+ for (const [label] of hiddenCacheUsage) {
+ const { content } = await openBreakdownTooltip(canvasElement, label);
+ await expect(
+ content.queryByText("Cache hit rate"),
+ ).not.toBeInTheDocument();
+ }
+
+ const { content } = await openBreakdownTooltip(
+ canvasElement,
+ "cost breakdown",
+ );
+ await expect(content.queryByText("Cache hit rate")).not.toBeInTheDocument();
+ },
+});
diff --git a/web/src/features/traces/components/BreakdownTooltip.tsx b/web/src/features/traces/components/BreakdownTooltip.tsx
index 65fc66c98d..63b61a5d75 100644
--- a/web/src/features/traces/components/BreakdownTooltip.tsx
+++ b/web/src/features/traces/components/BreakdownTooltip.tsx
@@ -7,8 +7,10 @@ import {
import { useState } from "react";
import Decimal from "decimal.js";
import Link from "next/link";
+import { CustomTooltip } from "@/src/components/design-system/CustomTooltip/CustomTooltip";
+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
import { type Details } from "@/src/features/traces/fns/calculateAggregatedUsage";
-import { ExternalLink } from "lucide-react";
+import { ExternalLink, InfoIcon } from "lucide-react";
import { centCostFormatter, usdFormatter } from "@/src/utils/numbers";
import { cva, type VariantProps } from "class-variance-authority";
import { cn } from "@/src/utils/tailwind";
@@ -82,6 +84,7 @@ export const BreakdownTooltip = ({
? usdFormatter(value, costFractionDigits, costFractionDigits)
: undefined;
const shareOf = createShareOf(contributionEntries);
+ const cacheHitRate = isCost ? null : getCacheHitRate(inputEntries, details);
const displayedTotal =
aggregatedDetails.total ??
(isCost ? sumEntries(contributionEntries) : new Decimal(0));
@@ -155,6 +158,11 @@ export const BreakdownTooltip = ({
<Section
title={isCost ? "Input cost" : "Input usage"}
entries={inputEntries}
+ lastRow={
+ cacheHitRate === null ? null : (
+ <CacheHitRateRow rate={cacheHitRate} />
+ )
+ }
{...sectionProps}
/>
@@ -269,6 +277,7 @@ interface SectionProps {
shareOf: (value: number | Decimal) => number;
/** Hairline above the section, matching the total row. */
separated?: boolean;
+ lastRow?: React.ReactNode;
}
const Section = ({
@@ -278,6 +287,7 @@ const Section = ({
formatExactValue,
shareOf,
separated = false,
+ lastRow,
}: SectionProps) => {
const sectionTotal = sumEntries(entries);
@@ -305,10 +315,110 @@ const Section = ({
share={shareOf(value ?? 0)}
/>
))}
+ {lastRow}
</div>
);
};
+function CacheHitRateRow({ rate }: { rate: number }) {
+ return (
+ <div className="text-muted-foreground col-span-4 grid grid-cols-subgrid items-center text-sm leading-5">
+ <span className="col-span-2 inline-flex items-center gap-1">
+ Cache hit rate
+ <CacheHitRateInfo />
+ </span>
+ <span className="text-foreground text-right font-mono text-xs tabular-nums">
+ {formatCacheHitRate(rate)}
+ </span>
+ </div>
+ );
+}
+
+function CacheHitRateInfo() {
+ return (
+ <CustomTooltip
+ placement="top"
+ content={
+ <div className="space-y-1.5">
+ <p>Cache-read tokens divided by total input tokens.</p>
+ <p>
+ For multi-turn interactions, above 90% is common when later turns
+ keep a stable prefix, and about 70% for a single turn with a fixed
+ rubric.
+ </p>
+ </div>
+ }
+ >
+ {({ getTriggerProps }) => (
+ <IconButton
+ {...getTriggerProps()}
+ icon={InfoIcon}
+ label="How cache hit rate is calculated"
+ size="xs"
+ variant="ghost"
+ />
+ )}
+ </CustomTooltip>
+ );
+}
+
+function getCacheHitRate(
+ entries: [string, number | undefined][],
+ details: Details | Details[],
+) {
+ const usageDetails = Array.isArray(details) ? details : [details];
+ const hasInconsistentTotal = usageDetails.some(({ total, ...buckets }) => {
+ if (total === undefined) return false;
+ return (
+ !isUsableTokenCount(total) ||
+ sumEntries(Object.entries(buckets)).gt(total)
+ );
+ });
+ if (hasInconsistentTotal) return null;
+
+ /** Generic read totals may overlap with each other or with modality counts. */
+ const hasOverlappingCacheReads = usageDetails.some((detail) => {
+ const readKeys = Object.entries(detail)
+ .filter(
+ ([key, value]) =>
+ key.includes("input") && isCacheReadKey(key) && (value ?? 0) > 0,
+ )
+ .map(([key]) => key);
+ return (
+ readKeys.length > 1 &&
+ readKeys.some(
+ (key) => !/^input_cached_(text|audio|image)_tokens$/.test(key),
+ )
+ );
+ });
+ if (hasOverlappingCacheReads) return null;
+ if (!entries.some(([key]) => /cache/i.test(key))) return null;
+ if (!entries.some(([key]) => key === "input")) return null;
+ if (entries.some(([, value]) => !isUsableTokenCount(value))) return null;
+
+ const total = sumEntries(entries);
+ if (total.lte(0)) return null;
+
+ const cacheRead = sumEntries(entries.filter(([key]) => isCacheReadKey(key)));
+ if (cacheRead.isNeg()) return null;
+
+ const rate = cacheRead.div(total);
+ return rate.toNumber();
+}
+
+function isUsableTokenCount(value: number | undefined) {
+ return typeof value === "number" && Number.isFinite(value);
+}
+
+function isCacheReadKey(key: string) {
+ if (/creation|cache_write/i.test(key)) return false;
+ return /cache_read|cached/i.test(key);
+}
+
+function formatCacheHitRate(rate: number) {
+ return `${(rate * 100).toFixed(1)}%`;
+}
+
function sortEntriesByValue(entries: [string, number | undefined][]) {
return entries.toSorted(([, a], [, b]) => (b ?? 0) - (a ?? 0));
}