The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[BOT-1776] Show cache read tokens in usage analytics (#76847)
Tyler Pludeb0682e01441c
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_analytics/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_analytics/api.cljindex a0254b08209..d1065b15d53 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_analytics/api.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_analytics/api.clj@@ -36,6 +36,7 @@[:user_message_count ms/IntGreaterThanOrEqualToZero][:assistant_message_count ms/IntGreaterThanOrEqualToZero][:total_tokens ms/IntGreaterThanOrEqualToZero]+ [:cache_read_tokens ms/IntGreaterThanOrEqualToZero][:last_message_at [:maybe ms/TemporalInstant]][:profile_id [:maybe :string]][:search_count ms/IntGreaterThanOrEqualToZero]@@ -49,7 +50,7 @@(def ^:private SortColumn"Allow-list of columns the list endpoint can sort by."- [:enum "created_at" "message_count" "total_tokens" "user" "profile_id" "ip_address"])+ [:enum "created_at" "message_count" "total_tokens" "cache_read_tokens" "user" "profile_id" "ip_address"])(def ^:private SortDirection[:enum "asc" "desc"])diff --git a/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj b/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.cljindex 208450c79ac..fe2af84a044 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj@@ -74,13 +74,14 @@"Allow-list of API sort keys → vectors of HoneySQL ORDER BY expressions (sansdirection). A vector lets a single sort key emit multiple ORDER BY terms thatshare the same direction (e.g. user sort orders by first_name then last_name)."- {"created_at" [:c.created_at]- "message_count" [:message_count]- "total_tokens" [:total_tokens]- "user" [[:lower [:min :u.first_name]]- [:lower [:min :u.last_name]]]- "profile_id" [:profile_id]- "ip_address" [:c.ip_address]})+ {"created_at" [:c.created_at]+ "message_count" [:message_count]+ "total_tokens" [:total_tokens]+ "cache_read_tokens" [:cache_read_tokens]+ "user" [[:lower [:min :u.first_name]]+ [:lower [:min :u.last_name]]]+ "profile_id" [:profile_id]+ "ip_address" [:c.ip_address]})(def ^:private list-query"HoneySQL query that selects one row per conversation with the aggregate@@ -103,7 +104,18 @@[:= :mm.deleted_at nil]]:order-by [[:mm.created_at :asc] [:mm.id :asc]]:limit 1}- :profile_id]]+ :profile_id]+ ;; Cache tokens are only recorded per LLM call in `ai_usage_log`+ ;; (`metabot_message` stores prompt+completion only), so this is a+ ;; correlated subquery rather than another one-to-many join, which+ ;; would fan out against the `metabot_message` join and inflate+ ;; every aggregate above.+ [[:coalesce+ {:select [[[:sum :aul.cache_read_tokens]]]+ :from [[:ai_usage_log :aul]]+ :where [:= :aul.conversation_id :c.id]}+ 0]+ :cache_read_tokens]]:from [[:metabot_conversation :c]]:left-join [[:metabot_message :m] [:and[:= :m.conversation_id :c.id]@@ -123,6 +135,7 @@:user_message_count (:user_message_count row):assistant_message_count (:assistant_message_count row):total_tokens (long (:total_tokens row 0))+ :cache_read_tokens (long (:cache_read_tokens row 0)):last_message_at (:last_message_at row):profile_id (:profile_id row):search_count (:search_count row 0)diff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.cljindex 0a639ebde2b..7045f832a23 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj@@ -34,8 +34,24 @@created-at (assoc :created_at created-at)deleted-at (assoc :deleted_at deleted-at)))))+(defn- insert-usage!+ "Insert an `ai_usage_log` row for a conversation. `cache-read-tokens` may be+ omitted to seed a pre-cache-columns row (NULL in the table)."+ [{:keys [conversation-id cache-read-tokens]}]+ (t2/insert! :model/AiUsageLog+ (cond-> {:conversation_id conversation-id+ :source "agent"+ :model "anthropic/claude-sonnet-4-6"+ :prompt_tokens 1+ :completion_tokens 1+ :total_tokens 2}+ cache-read-tokens (assoc :cache_read_tokens cache-read-tokens))))+(defn- delete-conversations!+ "Delete seeded conversations and their `ai_usage_log` rows (no FK links the+ two tables, so usage rows must be removed explicitly)."[conversation-ids]+ (t2/delete! :model/AiUsageLog {:where [:in :conversation_id (vec conversation-ids)]})(t2/delete! :model/MetabotConversation {:where [:in :id (vec conversation-ids)]}))(defn- insert-feedback!@@ -123,6 +139,12 @@:total-tokens 99:data [{:role "assistant" :content "deleted"}]:deleted-at jan-5})+ ;; convo-1 sums across usage rows; convo-2 also has a NULL-cache row+ ;; (predates the cache-token columns); convo-3 has no usage rows at all.+ (insert-usage! {:conversation-id convo-1 :cache-read-tokens 100})+ (insert-usage! {:conversation-id convo-1 :cache-read-tokens 25})+ (insert-usage! {:conversation-id convo-2 :cache-read-tokens 40})+ (insert-usage! {:conversation-id convo-2})(thunk {:test-user-id test-user-id:response-path response-path:convo-1 convo-1@@ -153,12 +175,14 @@(is (= 0 (:message_count convo-3-response)))(is (= 0 (:assistant_message_count convo-3-response)))(is (= 0 (:total_tokens convo-3-response)))+ (is (= 0 (:cache_read_tokens convo-3-response)))(is (= {:conversation_id convo-1:summary "First conversation":message_count 2:user_message_count 1:assistant_message_count 1:total_tokens 10+ :cache_read_tokens 125:profile_id "nlq":user {:id test-user-id:email "[email protected]"@@ -166,16 +190,18 @@:last_name "Analytics"}}(-> (select-keys convo-1-response [:conversation_id :summary :message_count:user_message_count :assistant_message_count :total_tokens- :profile_id :user])+ :cache_read_tokens :profile_id :user])(update :user select-keys [:id :email :first_name :last_name]))))(is (= {:conversation_id convo-2:message_count 3:user_message_count 1:assistant_message_count 2:total_tokens 26+ :cache_read_tokens 40:profile_id "internal"}(select-keys convo-2-response [:conversation_id :message_count :user_message_count- :assistant_message_count :total_tokens :profile_id])))))))+ :assistant_message_count :total_tokens :cache_read_tokens+ :profile_id])))))))(deftest list-conversations-pagination-test(with-list-conversations-fixture!@@ -203,12 +229,15 @@and always returned `created_at` order would fail five of the six cases.Designed orderings (asc):- created_at: (a, m, z) jan-1 / jan-2 / jan-3- user: (m, a, z) first_name Alice / Ben / Chloe- profile_id: (z, a, m) 1-profile / 2-profile / 3-profile- ip_address: (z, m, a) 10.0.0.1 / 10.0.0.2 / 10.0.0.3- message_count: (m, z, a) counts 1, 2, 3- total_tokens: (a, z, m) tokens 10, 20, 30"+ created_at: (a, m, z) jan-1 / jan-2 / jan-3+ user: (m, a, z) first_name Alice / Ben / Chloe+ profile_id: (z, a, m) 1-profile / 2-profile / 3-profile+ ip_address: (z, m, a) 10.0.0.1 / 10.0.0.2 / 10.0.0.3+ message_count: (m, z, a) counts 1, 2, 3+ total_tokens: (a, z, m) tokens 10, 20, 30+ cache_read_tokens: (z, a, m) cache reads 5, 15 (7+8), 25+ (all six permutations of three handles are taken, so this one+ necessarily repeats profile_id's ordering)"[thunk](mt/with-premium-features #{:audit-app}(mt/with-temp [:model/User {user-a :id} {:email "[email protected]"@@ -249,6 +278,10 @@(seed-msg! convo-m "3-profile" 30)(seed-msg! convo-z "1-profile" 10)(seed-msg! convo-z "1-profile" 10)+ (insert-usage! {:conversation-id convo-a :cache-read-tokens 7})+ (insert-usage! {:conversation-id convo-a :cache-read-tokens 8})+ (insert-usage! {:conversation-id convo-m :cache-read-tokens 25})+ (insert-usage! {:conversation-id convo-z :cache-read-tokens 5})(thunk {:response-path response-path:convo-a convo-a:convo-m convo-m@@ -265,7 +298,8 @@["profile_id" [convo-z convo-a convo-m]]["ip_address" [convo-z convo-m convo-a]]["message_count" [convo-m convo-z convo-a]]- ["total_tokens" [convo-a convo-z convo-m]]]]+ ["total_tokens" [convo-a convo-z convo-m]]+ ["cache_read_tokens" [convo-z convo-a convo-m]]]](testing (str "sort_by=" sort-by " &sort_dir=asc")(let [response (mt/user-http-request :crowberto :get 200(str response-path "&sort_by=" sort-by "&sort_dir=asc"))]diff --git a/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationsPage/ConversationsTable.tsx b/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationsPage/ConversationsTable.tsxindex 5eec6864021..fe9fe264ef7 100644--- a/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationsPage/ConversationsTable.tsx+++ b/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationsPage/ConversationsTable.tsx@@ -9,7 +9,7 @@ import AdminS from "metabase/css/admin.module.css";import CS from "metabase/css/core/index.css";import { renderMetabotProfileLabel } from "metabase/metabot/constants";import { useDispatch } from "metabase/redux";-import { Badge, Ellipsified, Flex } from "metabase/ui";+import { Badge, Ellipsified, Flex, Tooltip } from "metabase/ui";import { EMPTY_CELL_PLACEHOLDER } from "metabase/utils/constants";import { formatNumber } from "metabase/utils/formatting";import { getUserName } from "metabase/utils/user";@@ -77,6 +77,17 @@ export function ConversationsTable({sortingOptions={sortingOptions}onSortingOptionsChange={onSortingOptionsChange}>{t`Tokens`}</SortableColumnHeader>+ <SortableColumnHeader+ name="cache_read_tokens"+ sortingOptions={sortingOptions}+ onSortingOptionsChange={onSortingOptionsChange}+ >+ <Tooltip+ label={t`Portion of tokens served from the provider cache. A subset of Tokens, not an additional count.`}+ >+ <span>{t`Cached tokens`}</span>+ </Tooltip>+ </SortableColumnHeader><th>{t`Queries`}</th><th>{t`Searches`}</th><SortableColumnHeader@@ -89,7 +100,7 @@ export function ConversationsTable({<tbody>{showLoadingAndError && (<tr>- <td colSpan={8}>+ <td colSpan={9}><LoadingAndErrorWrapper loading={isLoading} error={error} /></td></tr>@@ -99,7 +110,7 @@ export function ConversationsTable({<>{conversations.length === 0 && (<tr>- <td colSpan={8}>+ <td colSpan={9}><Flex c="text-disabled" justify="center">{t`No conversations found`}</Flex>@@ -128,6 +139,7 @@ export function ConversationsTable({</td><td>{formatNumber(convo.message_count)}</td><td>{formatNumber(convo.total_tokens)}</td>+ <td>{formatNumber(convo.cache_read_tokens)}</td><td>{formatNumber(convo.query_count)}</td><td>{formatNumber(convo.search_count)}</td><td>{convo.ip_address ?? EMPTY_CELL_PLACEHOLDER}</td>diff --git a/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/types.ts b/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/types.tsindex e48c04326a5..5df8fc82e0c 100644--- a/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/types.ts+++ b/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/types.ts@@ -23,6 +23,7 @@ export type ConversationSummary = {user_message_count: number;assistant_message_count: number;total_tokens: number;+ cache_read_tokens: number;last_message_at: string | null;profile_id: MetabotProfileId | null;search_count: number;@@ -39,6 +40,7 @@ export const CONVERSATION_SORT_COLUMNS = ["created_at","message_count","total_tokens",+ "cache_read_tokens","user","profile_id","ip_address",