The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[BOT-1351] Anthropic prompt caching (#72851)
Tyler Plude856f9836115a
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj b/enterprise/backend/src/metabase_enterprise/metabot/usage.cljindex caf6c14d154..28cdcbd3849 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/usage.clj@@ -42,7 +42,7 @@(defenterprise log-ai-usage!"Record an LLM API call in the ai_usage_log table.":feature :none- [{:keys [source model prompt-tokens completion-tokens+ [{:keys [source model prompt-tokens completion-tokens cache-creation-tokens cache-read-tokensuser-id tenant-id conversation-id profile-id request-id ai-proxied]}](when-not (contains? known-sources source)(throw (ex-info (str "Unknown ai_usage_log source " (pr-str source))@@ -53,19 +53,23 @@{:profile-id pid}))))(when-not (= "user-intent-classification" source)(try- (let [total-tokens (+ prompt-tokens completion-tokens)]+ (let [cache-creation (or cache-creation-tokens 0)+ cache-read (or cache-read-tokens 0)+ total-tokens (+ prompt-tokens completion-tokens)](t2/insert! :model/AiUsageLog- {:source source- :model model- :prompt_tokens prompt-tokens- :completion_tokens completion-tokens- :total_tokens total-tokens- :user_id (or user-id api/*current-user-id*)- :tenant_id (or tenant-id (some-> api/*current-user* deref :tenant_id))- :conversation_id conversation-id- :profile_id (some-> profile-id name)- :request_id request-id- :ai_proxied ai-proxied}))+ {:source source+ :model model+ :prompt_tokens prompt-tokens+ :completion_tokens completion-tokens+ :total_tokens total-tokens+ :cache_creation_tokens cache-creation+ :cache_read_tokens cache-read+ :user_id (or user-id api/*current-user-id*)+ :tenant_id (or tenant-id (some-> api/*current-user* deref :tenant_id))+ :conversation_id conversation-id+ :profile_id (some-> profile-id name)+ :request_id request-id+ :ai_proxied ai-proxied}))(catch Exception e(log/warn e "Failed to log LLM usage to ai_usage_log")))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.cljindex 5b039f64981..9421b2912f6 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj@@ -163,6 +163,50 @@:completion-tokens 1:profile-id :made-up-profile})))))))+(deftest log-ai-usage!-records-cache-tokens-test+ (mt/with-premium-features #{:ai-controls}+ (testing "log-ai-usage! persists cache_creation_tokens / cache_read_tokens as a breakdown of prompt-tokens"+ ;; Caller passes prompt-tokens already inclusive of cache_creation + cache_read+ ;; (the LLM adapter pre-sums to match OpenAI semantics).+ ;; 2100 = 100 fresh + 400 cache_creation + 1600 cache_read.+ (mt/with-test-user :rasta+ (let [model "anthropic/claude-cache-tokens-test"]+ (usage/log-ai-usage!+ {:source "metabot_agent"+ :model model+ :prompt-tokens 2100+ :completion-tokens 50+ :cache-creation-tokens 400+ :cache-read-tokens 1600})+ (try+ (let [row (t2/select-one :model/AiUsageLog :source "metabot_agent" :model model+ {:order-by [[:id :desc]]})]+ (is (= 2100 (:prompt_tokens row)))+ (is (= 400 (:cache_creation_tokens row)))+ (is (= 1600 (:cache_read_tokens row)))+ (testing "total_tokens = prompt + completion (prompt already includes cache breakdown)"+ (is (= 2150 (:total_tokens row)))))+ (finally+ (t2/delete! :model/AiUsageLog :source "metabot_agent" :model model))))))))++(deftest log-ai-usage!-cache-tokens-default-to-zero-test+ (mt/with-premium-features #{:ai-controls}+ (testing "log-ai-usage! defaults cache token columns to 0 when not provided"+ (mt/with-test-user :rasta+ (let [model "anthropic/claude-cache-tokens-default-test"]+ (usage/log-ai-usage!+ {:source "metabot_agent"+ :model model+ :prompt-tokens 1+ :completion-tokens 1})+ (try+ (let [row (t2/select-one :model/AiUsageLog :source "metabot_agent" :model model+ {:order-by [[:id :desc]]})]+ (is (= 0 (:cache_creation_tokens row)))+ (is (= 0 (:cache_read_tokens row))))+ (finally+ (t2/delete! :model/AiUsageLog :source "metabot_agent" :model model))))))))+;;; ------------------------------------------ check-usage-limits! ------------------------------------------(deftest no-limits-configured-returns-nil-testdiff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/v_ai_usage_log_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/v_ai_usage_log_test.cljindex 81d93160e9c..7fde6456cdb 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/v_ai_usage_log_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/v_ai_usage_log_test.clj@@ -170,3 +170,23 @@(let [rows (query-view [tenant-log no-tenant-log])](is (= tenant-id (:tenant_id (find-row rows tenant-log))))(is (nil? (:tenant_id (find-row rows no-tenant-log))))))))++(deftest cache-tokens-passthrough-test+ (testing "cache_creation_tokens and cache_read_tokens pass through; null when not set"+ (mt/with-temp [:model/AiUsageLog {with-cache :id} {:source "metabot_agent"+ :model default-model+ :prompt_tokens 950+ :completion_tokens 30+ :total_tokens 980+ :cache_creation_tokens 50+ :cache_read_tokens 800}+ :model/AiUsageLog {no-cache :id} {:source "metabot_agent"+ :model default-model+ :prompt_tokens 100+ :completion_tokens 30+ :total_tokens 130}]+ (let [rows (query-view [with-cache no-cache])]+ (is (= 50 (:cache_creation_tokens (find-row rows with-cache))))+ (is (= 800 (:cache_read_tokens (find-row rows with-cache))))+ (is (nil? (:cache_creation_tokens (find-row rows no-cache))))+ (is (nil? (:cache_read_tokens (find-row rows no-cache))))))))diff --git a/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationStatsPage/audit-views-fixture.ts b/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationStatsPage/audit-views-fixture.tsindex e47a6371e84..63aeb9929c4 100644--- a/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationStatsPage/audit-views-fixture.ts+++ b/enterprise/frontend/src/metabase-enterprise/audit_app/metabot-analytics/components/ConversationStatsPage/audit-views-fixture.ts@@ -85,6 +85,8 @@ const usageLogTable = buildTable(["text", "ip_address", null],["integer", "prompt_tokens", "type/Quantity"],["integer", "completion_tokens", "type/Quantity"],+ ["integer", "cache_creation_tokens", "type/Quantity"],+ ["integer", "cache_read_tokens", "type/Quantity"],],);diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_creation_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_creation_tokens.yamlnew file mode 100644index 00000000000..ec44066cc37--- /dev/null+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_creation_tokens.yaml@@ -0,0 +1,23 @@+name: cache_creation_tokens+display_name: Cache Creation Tokens+description: Input tokens written to the provider prompt cache (Anthropic cache_creation_input_tokens). Subset of Prompt Tokens.+created_at: '2026-04-24T19:39:29.861736Z'+visibility_type: normal+table_id: [Internal Metabase Database, public, v_ai_usage_log]+database_type: int4+base_type: type/Integer+effective_type: type/Integer+semantic_type: type/Quantity+dimensions: []+position: 17+custom_position: 0+database_position: 17+has_field_values: auto-list+serdes/meta:+- {id: Internal Metabase Database, model: Database}+- {id: public, model: Schema}+- {id: v_ai_usage_log, model: Table}+- {id: cache_creation_tokens, model: Field}+database_is_generated: false+database_is_nullable: true+database_is_pk: falsediff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_read_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_read_tokens.yamlnew file mode 100644index 00000000000..4ba8ddb0539--- /dev/null+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_read_tokens.yaml@@ -0,0 +1,23 @@+name: cache_read_tokens+display_name: Cache Read Tokens+description: Input tokens served from the provider prompt cache (Anthropic cache_read_input_tokens). Subset of Prompt Tokens.+created_at: '2026-04-24T19:39:29.861736Z'+visibility_type: normal+table_id: [Internal Metabase Database, public, v_ai_usage_log]+database_type: int4+base_type: type/Integer+effective_type: type/Integer+semantic_type: type/Quantity+dimensions: []+position: 18+custom_position: 0+database_position: 18+has_field_values: auto-list+serdes/meta:+- {id: Internal Metabase Database, model: Database}+- {id: public, model: Schema}+- {id: v_ai_usage_log, model: Table}+- {id: cache_read_tokens, model: Field}+database_is_generated: false+database_is_nullable: true+database_is_pk: falsediff --git a/resources/metabot/prompts/system/embedding-next.selmer b/resources/metabot/prompts/system/embedding-next.selmerindex a2b30c2700f..52cd32a60c2 100644--- a/resources/metabot/prompts/system/embedding-next.selmer+++ b/resources/metabot/prompts/system/embedding-next.selmer@@ -356,32 +356,6 @@ Key Rules:- Link entities naturally within conversation, not as separate lists. This allows users to go back in the conversation history and find linked entities easily.- When you create or find something, link it immediately in your response (so that users can find it later).-# Important Context--Current date and time: {{current_time}}--In the system, the first day of the week (index 1) is: {{ first_day_of_week }}-{% if current_user_info %}--<user_context>-Here is some information about the user:-{{ current_user_info|safe }}-{% if viewing_context %}-{{ viewing_context|safe }}-{% endif %}-</user_context>-{% elif viewing_context %}--<user_context>-{{ viewing_context|safe }}-</user_context>-{% endif %}-{% if recent_views %}--{{ recent_views|safe }}-{% endif %}--# Your ScopeYou help users analyze data through natural language queries. Your workflow is: search for relevant data sources, read their metadata to verify field names and values, then build accurate visualizations.@@ -415,4 +389,31 @@ You focus on answering analytical questions by building visualizations from metr{{ custom_instructions|safe }}{% endif %}++<<<METABOT_CACHE_BREAKPOINT>>>++# Important Context++Current date and time: {{current_time}}++In the system, the first day of the week (index 1) is: {{ first_day_of_week }}+{% if current_user_info %}++<user_context>+Here is some information about the user:+{{ current_user_info|safe }}+{% if viewing_context %}+{{ viewing_context|safe }}+{% endif %}+</user_context>+{% elif viewing_context %}++<user_context>+{{ viewing_context|safe }}+</user_context>+{% endif %}+{% if recent_views %}++{{ recent_views|safe }}+{% endif %}</output>diff --git a/resources/metabot/prompts/system/internal.selmer b/resources/metabot/prompts/system/internal.selmerindex 36aff7ff3a0..bf41cbd70af 100644--- a/resources/metabot/prompts/system/internal.selmer+++ b/resources/metabot/prompts/system/internal.selmer@@ -375,13 +375,6 @@ You work with different types of data sources, each with different reliability l----# Important Context--{% if recent_views %}--{{ recent_views|safe }}-{% endif %}-<entity_linking>CRITICAL: Link format MUST use the metabase:// protocol prefix@@ -443,3 +436,12 @@ Every interaction must:{{ custom_instructions|safe }}{% endif %}++<<<METABOT_CACHE_BREAKPOINT>>>++# Important Context++{% if recent_views %}++{{ recent_views|safe }}+{% endif %}diff --git a/resources/metabot/prompts/system/natural-language-querying-only.selmer b/resources/metabot/prompts/system/natural-language-querying-only.selmerindex ed6c2dfd30f..a24ef48974f 100644--- a/resources/metabot/prompts/system/natural-language-querying-only.selmer+++ b/resources/metabot/prompts/system/natural-language-querying-only.selmer@@ -90,31 +90,6 @@ You don't need to be silent between tool calls — short thinking-out-loud is fi{% include "metabot/prompts/shared/personality.selmer" %}-# Context--Current date and time: {{current_time}}--In the system, the first day of the week (index 1) is: {{ first_day_of_week }}-{% if current_user_info %}--<user_context>-Here is some information about the user:-{{ current_user_info|safe }}-{% if viewing_context %}-{{ viewing_context|safe }}-{% endif %}-</user_context>-{% elif viewing_context %}--<user_context>-{{ viewing_context|safe }}-</user_context>-{% endif %}-{% if recent_views %}--{{ recent_views|safe }}-{% endif %}-# ScopeYour job is to answer analytical questions by producing visualizations. For other needs:@@ -141,3 +116,30 @@ Your job is to answer analytical questions by producing visualizations. For othe{{ custom_instructions|safe }}{% endif %}++<<<METABOT_CACHE_BREAKPOINT>>>++# Important Context++Current date and time: {{current_time}}++In the system, the first day of the week (index 1) is: {{ first_day_of_week }}+{% if current_user_info %}++<user_context>+Here is some information about the user:+{{ current_user_info|safe }}+{% if viewing_context %}+{{ viewing_context|safe }}+{% endif %}+</user_context>+{% elif viewing_context %}++<user_context>+{{ viewing_context|safe }}+</user_context>+{% endif %}+{% if recent_views %}++{{ recent_views|safe }}+{% endif %}diff --git a/resources/metabot/prompts/system/slackbot.selmer b/resources/metabot/prompts/system/slackbot.selmerindex 44584d074de..843e89d2d2f 100644--- a/resources/metabot/prompts/system/slackbot.selmer+++ b/resources/metabot/prompts/system/slackbot.selmer@@ -339,8 +339,6 @@ The following tools have specific behavioral requirements that you must follow:<environment># Environment Integration-Current date and time: {{ current_time }}-<technical_integration>## Technical Integration@@ -433,3 +431,9 @@ Where:</csv_uploads>{% endif %}++<<<METABOT_CACHE_BREAKPOINT>>>++# Runtime Context++Current date and time: {{ current_time }}diff --git a/resources/metabot/prompts/system/sql-querying-only.selmer b/resources/metabot/prompts/system/sql-querying-only.selmerindex e638e8d6f3f..b9298084912 100644--- a/resources/metabot/prompts/system/sql-querying-only.selmer+++ b/resources/metabot/prompts/system/sql-querying-only.selmer@@ -271,31 +271,6 @@ When fixing errors, keep in mind that there might be other errors that are not rIf you are fixing SQL errors it is FORBIDDEN to replace referenced models and tables with different ones.-# Important Context--Current date and time: {{current_time}}--In the system, the first day of the week (index 1) is: {{ first_day_of_week }}-{% if current_user_info %}--<user_context>-Here is some information about the user:-{{ current_user_info|safe }}-{% if viewing_context %}-{{ viewing_context|safe }}-{% endif %}-</user_context>-{% elif viewing_context %}--<user_context>-{{ viewing_context|safe }}-</user_context>-{% endif %}-{% if recent_views %}--{{ recent_views|safe }}-{% endif %}-## Data Source Types- **Tables**: Raw data sources in the database@@ -362,3 +337,30 @@ The target database uses the **{{sql_dialect}}** SQL dialect. Follow these diale{{ custom_instructions|safe }}{% endif %}++<<<METABOT_CACHE_BREAKPOINT>>>++# Important Context++Current date and time: {{current_time}}++In the system, the first day of the week (index 1) is: {{ first_day_of_week }}+{% if current_user_info %}++<user_context>+Here is some information about the user:+{{ current_user_info|safe }}+{% if viewing_context %}+{{ viewing_context|safe }}+{% endif %}+</user_context>+{% elif viewing_context %}++<user_context>+{{ viewing_context|safe }}+</user_context>+{% endif %}+{% if recent_views %}++{{ recent_views|safe }}+{% endif %}diff --git a/resources/metabot/prompts/system/transform-codegen.selmer b/resources/metabot/prompts/system/transform-codegen.selmerindex 3e0e2587f78..5f1a6c1b70b 100644--- a/resources/metabot/prompts/system/transform-codegen.selmer+++ b/resources/metabot/prompts/system/transform-codegen.selmer@@ -387,6 +387,8 @@ The following tools have specific behavioral requirements that you must follow:</tools>{% endif %}+<<<METABOT_CACHE_BREAKPOINT>>>+# Important ContextCurrent date and time: {{current_time}}diff --git a/resources/migrations/061/20260419_ai_usage_log_cache_tokens.yaml b/resources/migrations/061/20260419_ai_usage_log_cache_tokens.yamlnew file mode 100644index 00000000000..4b2dfe7ffd3--- /dev/null+++ b/resources/migrations/061/20260419_ai_usage_log_cache_tokens.yaml@@ -0,0 +1,37 @@+## Track Anthropic prompt-caching token counts on ai_usage_log+##+## Note: the v3 view update that exposes these columns lives in a separate+## file (20260511_ai_usage_log_view_v3.yaml) so it runs after the BOT-1406+## PII compliance migration (20260504), which is what adds embedding_hostname+## and the other PII columns to metabot_conversation that v3 projects.++databaseChangeLog:+ - changeSet:+ id: v61.2026-04-19T00:00:01+ author: tsplude+ comment: Add cache_creation_tokens to ai_usage_log for Anthropic prompt caching+ changes:+ - addColumn:+ tableName: ai_usage_log+ columns:+ - column:+ name: cache_creation_tokens+ type: int+ remarks: Number of input tokens written to the provider prompt cache (Anthropic cache_creation_input_tokens).+ constraints:+ nullable: true++ - changeSet:+ id: v61.2026-04-19T00:00:02+ author: tsplude+ comment: Add cache_read_tokens to ai_usage_log for Anthropic prompt caching+ changes:+ - addColumn:+ tableName: ai_usage_log+ columns:+ - column:+ name: cache_read_tokens+ type: int+ remarks: Number of input tokens served from the provider prompt cache (Anthropic cache_read_input_tokens).+ constraints:+ nullable: truediff --git a/resources/migrations/061/20260511_ai_usage_log_view_v3.yaml b/resources/migrations/061/20260511_ai_usage_log_view_v3.yamlnew file mode 100644index 00000000000..0621e5e3f16--- /dev/null+++ b/resources/migrations/061/20260511_ai_usage_log_view_v3.yaml@@ -0,0 +1,40 @@+## Update v_ai_usage_log to v3 — expose cache_creation_tokens and cache_read_tokens.+##+## Must run after 20260504_metabot_conversation_pii_compliance.yaml (BOT-1406),+## because v3 of the view projects embedding_hostname, embedding_path, user_agent,+## and sanitized_user_agent from metabot_conversation — columns that migration+## introduces. Liquibase processes change-log files alphabetically, so the+## filename date is what enforces the ordering.++databaseChangeLog:+ - changeSet:+ id: v61.2026-05-11T00:00:01+ author: tsplude+ runOnChange: true+ comment: Update v_ai_usage_log to v3 - expose cache_creation_tokens, cache_read_tokens+ changes:+ - sqlFile:+ dbms: postgresql+ path: ../instance_analytics_views/ai_usage_log/v3/postgres-ai_usage_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: mysql,mariadb+ path: ../instance_analytics_views/ai_usage_log/v3/mysql-ai_usage_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: h2+ path: ../instance_analytics_views/ai_usage_log/v3/h2-ai_usage_log.sql+ relativeToChangelogFile: true+ rollback:+ - sqlFile:+ dbms: postgresql+ path: ../instance_analytics_views/ai_usage_log/v2/postgres-ai_usage_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: mysql,mariadb+ path: ../instance_analytics_views/ai_usage_log/v2/mysql-ai_usage_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: h2+ path: ../instance_analytics_views/ai_usage_log/v2/h2-ai_usage_log.sql+ relativeToChangelogFile: truediff --git a/resources/migrations/instance_analytics_views/ai_usage_log/v3/h2-ai_usage_log.sql b/resources/migrations/instance_analytics_views/ai_usage_log/v3/h2-ai_usage_log.sqlnew file mode 100644index 00000000000..5f0773e8312--- /dev/null+++ b/resources/migrations/instance_analytics_views/ai_usage_log/v3/h2-ai_usage_log.sql@@ -0,0 +1,49 @@+DROP VIEW IF EXISTS v_ai_usage_log;++CREATE OR REPLACE VIEW v_ai_usage_log AS+SELECT+ a.id AS usage_log_id,+ a.created_at,+ a.source,+ CASE a.source+ WHEN 'metabot_agent' THEN 'Metabot'+ WHEN 'agent' THEN 'Metabot'+ WHEN 'document_generate_content' THEN 'Documents'+ WHEN 'example_question_generation_batch' THEN 'Suggested Prompts'+ WHEN 'slack' THEN 'Slackbot'+ WHEN 'slackbot' THEN 'Slackbot'+ WHEN 'oss-sql-gen' THEN 'SQL'+ WHEN 'sql-gen' THEN 'SQL'+ WHEN 'unknown' THEN 'Unknown'+ ELSE a.source+ END AS source_name,+ a.model,+ a.profile_id,+ a.prompt_tokens,+ a.completion_tokens,+ a.total_tokens,+ a.conversation_id,+ a.user_id,+ 'user_' || a.user_id AS user_qualified_id,+ COALESCE(u.first_name || ' ' || u.last_name, u.email) AS user_display_name,+ (SELECT pg.name+ FROM permissions_group_membership pgm+ JOIN permissions_group pg ON pg.id = pgm.group_id+ WHERE pgm.user_id = a.user_id+ AND pg.id != 1+ ORDER BY pg.name+ LIMIT 1) AS group_name,+ c.ip_address AS ip_address,+ c.embedding_hostname AS embedding_hostname,+ c.embedding_path AS embedding_path,+ c.user_agent AS user_agent,+ c.sanitized_user_agent AS sanitized_user_agent,+ a.tenant_id,+ a.request_id,+ a.cache_creation_tokens,+ a.cache_read_tokens+FROM ai_usage_log a+LEFT JOIN core_user u+ ON u.id = a.user_id+LEFT JOIN metabot_conversation c+ ON c.id = a.conversation_id;diff --git a/resources/migrations/instance_analytics_views/ai_usage_log/v3/mysql-ai_usage_log.sql b/resources/migrations/instance_analytics_views/ai_usage_log/v3/mysql-ai_usage_log.sqlnew file mode 100644index 00000000000..3653fbd3149--- /dev/null+++ b/resources/migrations/instance_analytics_views/ai_usage_log/v3/mysql-ai_usage_log.sql@@ -0,0 +1,49 @@+DROP VIEW IF EXISTS v_ai_usage_log;++CREATE OR REPLACE VIEW v_ai_usage_log AS+SELECT+ a.id AS usage_log_id,+ a.created_at,+ a.source,+ CASE a.source+ WHEN 'metabot_agent' THEN 'Metabot'+ WHEN 'agent' THEN 'Metabot'+ WHEN 'document_generate_content' THEN 'Documents'+ WHEN 'example_question_generation_batch' THEN 'Suggested Prompts'+ WHEN 'slack' THEN 'Slackbot'+ WHEN 'slackbot' THEN 'Slackbot'+ WHEN 'oss-sql-gen' THEN 'SQL'+ WHEN 'sql-gen' THEN 'SQL'+ WHEN 'unknown' THEN 'Unknown'+ ELSE a.source+ END AS source_name,+ a.model,+ a.profile_id,+ a.prompt_tokens,+ a.completion_tokens,+ a.total_tokens,+ a.conversation_id,+ a.user_id,+ CONCAT('user_', a.user_id) AS user_qualified_id,+ COALESCE(CONCAT(u.first_name, ' ', u.last_name), u.email) AS user_display_name,+ (SELECT pg.name+ FROM permissions_group_membership pgm+ JOIN permissions_group pg ON pg.id = pgm.group_id+ WHERE pgm.user_id = a.user_id+ AND pg.id != 1+ ORDER BY pg.name+ LIMIT 1) AS group_name,+ c.ip_address AS ip_address,+ c.embedding_hostname AS embedding_hostname,+ c.embedding_path AS embedding_path,+ c.user_agent AS user_agent,+ c.sanitized_user_agent AS sanitized_user_agent,+ a.tenant_id,+ a.request_id,+ a.cache_creation_tokens,+ a.cache_read_tokens+FROM ai_usage_log a+LEFT JOIN core_user u+ ON u.id = a.user_id+LEFT JOIN metabot_conversation c+ ON c.id = a.conversation_id;diff --git a/resources/migrations/instance_analytics_views/ai_usage_log/v3/postgres-ai_usage_log.sql b/resources/migrations/instance_analytics_views/ai_usage_log/v3/postgres-ai_usage_log.sqlnew file mode 100644index 00000000000..5f0773e8312--- /dev/null+++ b/resources/migrations/instance_analytics_views/ai_usage_log/v3/postgres-ai_usage_log.sql@@ -0,0 +1,49 @@+DROP VIEW IF EXISTS v_ai_usage_log;++CREATE OR REPLACE VIEW v_ai_usage_log AS+SELECT+ a.id AS usage_log_id,+ a.created_at,+ a.source,+ CASE a.source+ WHEN 'metabot_agent' THEN 'Metabot'+ WHEN 'agent' THEN 'Metabot'+ WHEN 'document_generate_content' THEN 'Documents'+ WHEN 'example_question_generation_batch' THEN 'Suggested Prompts'+ WHEN 'slack' THEN 'Slackbot'+ WHEN 'slackbot' THEN 'Slackbot'+ WHEN 'oss-sql-gen' THEN 'SQL'+ WHEN 'sql-gen' THEN 'SQL'+ WHEN 'unknown' THEN 'Unknown'+ ELSE a.source+ END AS source_name,+ a.model,+ a.profile_id,+ a.prompt_tokens,+ a.completion_tokens,+ a.total_tokens,+ a.conversation_id,+ a.user_id,+ 'user_' || a.user_id AS user_qualified_id,+ COALESCE(u.first_name || ' ' || u.last_name, u.email) AS user_display_name,+ (SELECT pg.name+ FROM permissions_group_membership pgm+ JOIN permissions_group pg ON pg.id = pgm.group_id+ WHERE pgm.user_id = a.user_id+ AND pg.id != 1+ ORDER BY pg.name+ LIMIT 1) AS group_name,+ c.ip_address AS ip_address,+ c.embedding_hostname AS embedding_hostname,+ c.embedding_path AS embedding_path,+ c.user_agent AS user_agent,+ c.sanitized_user_agent AS sanitized_user_agent,+ a.tenant_id,+ a.request_id,+ a.cache_creation_tokens,+ a.cache_read_tokens+FROM ai_usage_log a+LEFT JOIN core_user u+ ON u.id = a.user_id+LEFT JOIN metabot_conversation c+ ON c.id = a.conversation_id;diff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/token_usage/jsonschema/1-0-5 b/snowplow/iglu-client-embedded/schemas/com.metabase/token_usage/jsonschema/1-0-5new file mode 100644index 00000000000..2b83acef4d9--- /dev/null+++ b/snowplow/iglu-client-embedded/schemas/com.metabase/token_usage/jsonschema/1-0-5@@ -0,0 +1,130 @@+{+ "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",+ "self": {+ "vendor": "com.metabase",+ "name": "token_usage",+ "format": "jsonschema",+ "version": "1-0-5"+ },+ "type": "object",+ "description": "Event to capture the token usage of individual LLM invocations. Created for billing purposes.",+ "properties": {+ "hashed_metabase_license_token": {+ "type": "string",+ "description": "The hashed value of the Metabase license token for instances with a token. For Open Source instances, this will use the snowplow anonymous uuid in the form `oss_<anonymous_uuid>` to allow unique counts of instances.",+ "maxLength": 255+ },+ "user_id": {+ "type": [+ "integer",+ "null"+ ],+ "description": "The ID of the user that send the request",+ "minimum": 0,+ "maximum": 2147483647+ },+ "request_id": {+ "type": "string",+ "description": "Unique ID that is assigned to each incoming request. Can be used to aggregate usage for a single request.",+ "maxLength": 255+ },+ "session_id": {+ "type": [+ "string",+ "null"+ ],+ "description": "An identifier for the conversation or session in which the tokens were used. This can help group related requests together. Can be null for non-conversational use cases.",+ "maxLength": 255+ },+ "model_id": {+ "type": "string",+ "description": "The ID of the model that was used to generate the tokens",+ "maxLength": 255+ },+ "total_tokens": {+ "type": "integer",+ "description": "The total number of tokens",+ "minimum": 0,+ "maximum": 2147483647+ },+ "prompt_tokens": {+ "type": "integer",+ "description": "The number of prompt tokens",+ "minimum": 0,+ "maximum": 2147483647+ },+ "completion_tokens": {+ "type": "integer",+ "description": "The number of completion tokens",+ "minimum": 0,+ "maximum": 2147483647+ },+ "cache_creation_tokens": {+ "type": [+ "integer",+ "null"+ ],+ "description": "Number of input tokens written to the provider prompt cache (Anthropic cache_creation_input_tokens). Null or absent for providers that do not report cache usage.",+ "minimum": 0,+ "maximum": 2147483647+ },+ "cache_read_tokens": {+ "type": [+ "integer",+ "null"+ ],+ "description": "Number of input tokens served from the provider prompt cache (Anthropic cache_read_input_tokens). Null or absent for providers that do not report cache usage.",+ "minimum": 0,+ "maximum": 2147483647+ },+ "estimated_costs_usd": {+ "type": "number",+ "description": "The estimated cost of for the tokens in USD",+ "minimum": 0,+ "maximum": 2147483647+ },+ "duration_ms": {+ "type": [+ "integer",+ "null"+ ],+ "description": "The time it took to (in milliseconds) to receive an answer from the LLM.",+ "minimum": 0,+ "maximum": 2147483647+ },+ "tag": {+ "type": [+ "string",+ "null"+ ],+ "description": "A classifier indicating the specific purpose or functionality for which the tokens were used.",+ "maxLength": 255+ },+ "profile": {+ "type": [+ "string",+ "null"+ ],+ "description": "The profile/configuration variant used for the request. Allows distinguishing different configuration variants and usage patterns.",+ "maxLength": 255+ },+ "source": {+ "type": [+ "string",+ "null"+ ],+ "description": "The source/trigger of the request (e.g., 'metabot_agent', 'sql_generation_workflow', 'analyze_chart'). Indicates which API endpoint or workflow initiated the LLM call.",+ "maxLength": 255+ }+ },+ "required": [+ "hashed_metabase_license_token",+ "request_id",+ "model_id",+ "total_tokens",+ "prompt_tokens",+ "completion_tokens",+ "estimated_costs_usd"+ ],+ "additionalProperties": false+}diff --git a/src/metabase/analytics/llm_token_usage.clj b/src/metabase/analytics/llm_token_usage.cljindex eeab38e6705..d062fb169ac 100644--- a/src/metabase/analytics/llm_token_usage.clj+++ b/src/metabase/analytics/llm_token_usage.clj@@ -17,6 +17,8 @@[:prompt-tokens ms/IntGreaterThanOrEqualToZero][:completion-tokens ms/IntGreaterThanOrEqualToZero][:estimated-costs-usd number?]+ [:cache-creation-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:cache-read-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]][:user-id {:optional true} [:maybe :int]][:duration-ms {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]][:source {:optional true} [:maybe :string]]@@ -28,6 +30,7 @@(mu/defn track-snowplow!"Track snowplow token_usage event."[{:keys [request-id model-id total-tokens prompt-tokens completion-tokens+ cache-creation-tokens cache-read-tokensestimated-costs-usd user-id duration-ms source tag session-id profilehashed-metabase-license-token]}:- SnowplowArgs]@@ -39,6 +42,8 @@:total-tokens total-tokens:prompt-tokens prompt-tokens:completion-tokens completion-tokens+ :cache-creation-tokens cache-creation-tokens+ :cache-read-tokens cache-read-tokens:estimated-costs-usd estimated-costs-usd:user-id user-id:duration-ms (some-> duration-ms long)@@ -50,18 +55,25 @@(def ^:private PrometheusArgs[:map- [:model-id :string]- [:tag :string]- [:prompt-tokens ms/IntGreaterThanOrEqualToZero]- [:completion-tokens ms/IntGreaterThanOrEqualToZero]])+ [:model-id :string]+ [:tag :string]+ [:prompt-tokens ms/IntGreaterThanOrEqualToZero]+ [:completion-tokens ms/IntGreaterThanOrEqualToZero]+ [:cache-creation-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:cache-read-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]])(mu/defn track-prometheus!"Track Prometheus LLM token usage metrics."- [{:keys [model-id tag prompt-tokens completion-tokens]}+ [{:keys [model-id tag prompt-tokens completion-tokens+ cache-creation-tokens cache-read-tokens]}:- PrometheusArgs](let [labels {:model model-id :source tag}](analytics/inc! :metabase-metabot/llm-input-tokens labels prompt-tokens)(analytics/inc! :metabase-metabot/llm-output-tokens labels completion-tokens)+ (when (and cache-creation-tokens (pos? cache-creation-tokens))+ (analytics/inc! :metabase-metabot/llm-cache-creation-tokens labels cache-creation-tokens))+ (when (and cache-read-tokens (pos? cache-read-tokens))+ (analytics/inc! :metabase-metabot/llm-cache-read-tokens labels cache-read-tokens))(analytics/observe! :metabase-metabot/llm-tokens-per-call labels (+ prompt-tokens completion-tokens))))(mu/defn track-token-usage!diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex d704951f324..dee4b69e1fb 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -614,6 +614,12 @@(prometheus/counter :metabase-metabot/llm-output-tokens{:description "LLM output tokens":labels [:model :source]})+ (prometheus/counter :metabase-metabot/llm-cache-creation-tokens+ {:description "LLM cache creation input tokens (Anthropic prompt caching)"+ :labels [:model :source]})+ (prometheus/counter :metabase-metabot/llm-cache-read-tokens+ {:description "LLM cache read input tokens (Anthropic prompt caching)"+ :labels [:model :source]})(prometheus/histogram :metabase-metabot/llm-tokens-per-call{:description "Tokens per LLM call":labels [:model :source]diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.cljindex d4152c7c881..394c79d244d 100644--- a/src/metabase/analytics/snowplow.clj+++ b/src/metabase/analytics/snowplow.clj@@ -50,7 +50,7 @@:snowplow/action "1-0-0":snowplow/embed_share "1-0-2":snowplow/llm_usage "1-0-0"- :snowplow/token_usage "1-0-4"+ :snowplow/token_usage "1-0-5":snowplow/serialization "1-0-1":snowplow/simple_event "1-0-0":snowplow/cleanup "1-0-0"diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex bdbb404bba9..a1f801950d7 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -160,35 +160,41 @@(let [start-ms (u/start-timer)](map (fn [part](when (= (:type part) :usage)- (let [usage (:usage part)- model (or model (:model part) "unknown")- prompt (:promptTokens usage 0)- completion (:completionTokens usage 0)]+ (let [usage (:usage part)+ model (or model (:model part) "unknown")+ prompt (:promptTokens usage 0)+ completion (:completionTokens usage 0)+ cache-creation (:cacheCreationTokens usage 0)+ cache-read (:cacheReadTokens usage 0)](analytics.core/track-token-usage!;; The caller can omit request-id (and other snowplow opts) to skip snowplow tracking.- {:prometheus true- :snowplow (some? request-id)- :profile (some-> profile-id name)- :model-id model- :prompt-tokens prompt- :completion-tokens completion- :total-tokens (+ prompt completion)- :estimated-costs-usd 0.0- :duration-ms (long (u/since-ms start-ms))- :user-id api/*current-user-id*- :request-id (some-> request-id analytics.core/uuid->ai-service-hex-uuid)- :session-id session-id- :source source- :tag tag})+ {:prometheus true+ :snowplow (some? request-id)+ :profile (some-> profile-id name)+ :model-id model+ :prompt-tokens prompt+ :completion-tokens completion+ :cache-creation-tokens cache-creation+ :cache-read-tokens cache-read+ :total-tokens (+ prompt completion)+ :estimated-costs-usd 0.0+ :duration-ms (long (u/since-ms start-ms))+ :user-id api/*current-user-id*+ :request-id (some-> request-id analytics.core/uuid->ai-service-hex-uuid)+ :session-id session-id+ :source source+ :tag tag})(usage/log-ai-usage!- {:source (or source tag "unknown")- :model model- :prompt-tokens prompt- :completion-tokens completion- :conversation-id session-id- :profile-id profile-id- :request-id request-id- :ai-proxied (boolean ai-proxy?)})))+ {:source (or source tag "unknown")+ :model model+ :prompt-tokens prompt+ :completion-tokens completion+ :cache-creation-tokens cache-creation+ :cache-read-tokens cache-read+ :conversation-id session-id+ :profile-id profile-id+ :request-id request-id+ :ai-proxied (boolean ai-proxy?)})))part))))(defn- report-tool-usage-xfdiff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.cljindex a3220e9c953..4137c61fa83 100644--- a/src/metabase/metabot/self/claude.clj+++ b/src/metabase/metabot/self/claude.clj@@ -14,6 +14,43 @@(set! *warn-on-reflection* true)+(defn- claude-usage->aisdk-usage+ "Convert an Anthropic `usage` block into the AISDK `:usage` shape.++ Anthropic reports three disjoint input-token buckets; the total input sent to+ the model is the sum of all three:++ input_tokens — fresh (non-cached) input+ cache_creation_input_tokens — input written to the provider cache+ cache_read_input_tokens — input served from the provider cache++ We pre-sum these into :promptTokens so downstream analytics and ai_usage_log+ see a provider-neutral total-input count, matching OpenAI's prompt_tokens+ semantic (where cache counts are a subset breakdown of the total).++ ai_usage_log column mapping:++ without Anthropic prompt caching:+ prompt_tokens := input_tokens+ completion_tokens := output_tokens+ total_tokens := input_tokens + output_tokens++ with Anthropic prompt caching:+ prompt_tokens := input_tokens + cache_creation_input_tokens + cache_read_input_tokens+ completion_tokens := output_tokens+ total_tokens := prompt_tokens + completion_tokens++ The two are equivalent when caching is inactive (both cache buckets are 0),+ so one unified formula is used in code; the split above is purely for reader+ clarity."+ [u]+ {:promptTokens (+ (:input_tokens u 0)+ (:cache_creation_input_tokens u 0)+ (:cache_read_input_tokens u 0))+ :completionTokens (:output_tokens u 0)+ :cacheCreationTokens (:cache_creation_input_tokens u 0)+ :cacheReadTokens (:cache_read_input_tokens u 0)})+(defn claude->aisdk-chunks-xf"Translates Claude /v1/messages streaming events into AI SDK v5 protocol chunks.@@ -57,10 +94,9 @@(cond-> result;; close up latest type if incomplete@current-type (close!)- ;; flush last-known usage if stream ended before message_delta+ ;; flush last-known usage if stream ended before message_delta.@last-usage (rf {:type :usage- :usage {:promptTokens (:input_tokens @last-usage 0)- :completionTokens (:output_tokens @last-usage 0)}+ :usage (claude-usage->aisdk-usage @last-usage):id @message-id:model @model-name})true (rf)))@@ -105,6 +141,7 @@;; but message_delta values are cumulative and include the earlier;; counts.;; https://platform.claude.com/docs/en/build-with-claude/streaming#event-types+ ;; https://platform.claude.com/docs/en/api/cli/messages#message_delta_usage(= t "message_delta") (u/prog1(vreset! last-usage (:usage chunk)));; end of message@@ -184,6 +221,45 @@:description doc:input_schema (mjs/transform params {:additionalProperties false})}))+(defn- add-tools-cache-breakpoint+ "Attach an ephemeral cache_control marker to the last tool in `tools`.+ Anthropic caches everything in the request up to and including the block with+ `cache_control`, so a single breakpoint on the final tool covers the whole+ tool list."+ [tools]+ (if (seq tools)+ (update tools (dec (count tools)) assoc :cache_control {:type "ephemeral"})+ tools))++(def ^:private system-cache-breakpoint-sentinel+ "Literal marker placed in selmer templates to indicate where the static cacheable+ prefix ends and the dynamic per-request suffix begins. Anthropic-only; ignored+ by other provider adapters."+ "<<<METABOT_CACHE_BREAKPOINT>>>")++(defn- system->cached-content-blocks+ "Wrap a rendered system prompt for Anthropic, applying ephemeral cache_control.++ If `system` contains the cache breakpoint sentinel, split it into two content+ blocks: a cached static prefix and an uncached dynamic suffix. The model sees+ the concatenation; the split is purely a wire-protocol device for caching.++ If the sentinel is absent, fall back to a single cached content block covering+ the whole prompt."+ [system]+ (let [idx (.indexOf ^String system ^String system-cache-breakpoint-sentinel)]+ (if (neg? idx)+ [{:type "text"+ :text system+ :cache_control {:type "ephemeral"}}]+ (let [prefix (str/trimr (subs system 0 idx))+ suffix (str/triml (subs system (+ idx (count system-cache-breakpoint-sentinel))))]+ [{:type "text"+ :text prefix+ :cache_control {:type "ephemeral"}}+ {:type "text"+ :text suffix}]))))+(defn- anthropic-errors [res](let [status (long (:status res 0))error-msg (get-in res [:body :error :message])]@@ -227,11 +303,15 @@:or {model "claude-haiku-4-5"}} :- core/LLMRequestOpts](let [messages (parts->claude-messages input)all-tools (when (seq tools) (mapv tool->claude tools))+ all-tools (if (and all-tools (not schema))+ (add-tools-cache-breakpoint all-tools)+ all-tools)req (cond-> {:model model:max_tokens (or max-tokens 4096):stream true+ :cache_control {:type "ephemeral"}:messages messages}- system (assoc :system system)+ system (assoc :system (system->cached-content-blocks system))all-tools (assoc :tools all-tools)(and all-toolstool_choice) (assoc :tool_choice (case (name tool_choice)diff --git a/src/metabase/metabot/usage.clj b/src/metabase/metabot/usage.cljindex 90979b3fb81..b9f9f9797be 100644--- a/src/metabase/metabot/usage.clj+++ b/src/metabase/metabot/usage.clj@@ -17,16 +17,18 @@(def ^:private usage-map-schema[:map- [:source :string]- [:model :string]- [:prompt-tokens [:int {:min 0}]]- [:completion-tokens [:int {:min 0}]]- [:user-id {:optional true} [:maybe ms/PositiveInt]]- [:tenant-id {:optional true} [:maybe ms/PositiveInt]]- [:conversation-id {:optional true} [:maybe :string]]- [:profile-id {:optional true} [:maybe :keyword]]- [:request-id {:optional true} [:maybe :string]]- [:ai-proxied {:optional true} [:maybe :boolean]]])+ [:source :string]+ [:model :string]+ [:prompt-tokens [:int {:min 0}]]+ [:completion-tokens [:int {:min 0}]]+ [:cache-creation-tokens {:optional true} [:maybe [:int {:min 0}]]]+ [:cache-read-tokens {:optional true} [:maybe [:int {:min 0}]]]+ [:user-id {:optional true} [:maybe ms/PositiveInt]]+ [:tenant-id {:optional true} [:maybe ms/PositiveInt]]+ [:conversation-id {:optional true} [:maybe :string]]+ [:profile-id {:optional true} [:maybe :keyword]]+ [:request-id {:optional true} [:maybe :string]]+ [:ai-proxied {:optional true} [:maybe :boolean]]])(defenterprise-schema log-ai-usage! :- :any"Record an LLM API call in the ai_usage_log table.diff --git a/test/metabase/analytics/llm_token_usage_test.clj b/test/metabase/analytics/llm_token_usage_test.cljindex f40df5a895b..084da85dfba 100644--- a/test/metabase/analytics/llm_token_usage_test.clj+++ b/test/metabase/analytics/llm_token_usage_test.clj@@ -50,18 +50,20 @@(mt/with-temporary-setting-values [premium-embedding-token nilanalytics-uuid "uuid-for-test"](snowplow-test/with-fake-snowplow-collector- (llm-token-usage/track-snowplow! {:request-id "deadbeef00"- :model-id "openai/gpt-4"- :total-tokens 300- :prompt-tokens 200- :completion-tokens 100- :estimated-costs-usd 0.0- :user-id 42- :duration-ms 1234- :source "oss_metabot"- :tag "oss-sqlgen"- :session-id "session-abc"- :profile "internal"})+ (llm-token-usage/track-snowplow! {:request-id "deadbeef00"+ :model-id "openai/gpt-4"+ :total-tokens 300+ :prompt-tokens 200+ :completion-tokens 100+ :cache-creation-tokens 250+ :cache-read-tokens 900+ :estimated-costs-usd 0.0+ :user-id 42+ :duration-ms 1234+ :source "oss_metabot"+ :tag "oss-sqlgen"+ :session-id "session-abc"+ :profile "internal"})(is (=? [{:user-id "42":data {"hashed_metabase_license_token" "oss__uuid-for-test""request_id" "deadbeef00"@@ -69,6 +71,8 @@"total_tokens" 300"prompt_tokens" 200"completion_tokens" 100+ "cache_creation_tokens" 250+ "cache_read_tokens" 900"estimated_costs_usd" 0.0"duration_ms" 1234"source" "oss_metabot"@@ -83,19 +87,48 @@;; mt/with-prometheus-system! is slow, so prefer to clear metrics between test cases(analytics/clear! :metabase-metabot/llm-input-tokens)(analytics/clear! :metabase-metabot/llm-output-tokens)+ (analytics/clear! :metabase-metabot/llm-cache-creation-tokens)+ (analytics/clear! :metabase-metabot/llm-cache-read-tokens)(analytics/clear! :metabase-metabot/llm-tokens-per-call))(deftest track-prometheus!-test(mt/with-prometheus-system! [_ system]- (testing "increments prometheus metrics with correct labels and values"- (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"- :tag "test-tag"- :prompt-tokens 100- :completion-tokens 50})- (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag"}]+ (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag"}]+ (testing "increments prometheus metrics with correct labels and values"+ (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :tag "test-tag"+ :prompt-tokens 100+ :completion-tokens 50})(is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens labels)))(is (= 50.0 (mt/metric-value system :metabase-metabot/llm-output-tokens labels)))- (is (= 150.0 (:sum (mt/metric-value system :metabase-metabot/llm-tokens-per-call labels))))))))+ (is (= 150.0 (:sum (mt/metric-value system :metabase-metabot/llm-tokens-per-call labels))))+ (testing "cache counters are untouched when cache fields are omitted"+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels)))))++ (clear-llm-metrics!)++ (testing "positive cache token fields increment their counters"+ (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :tag "test-tag"+ :prompt-tokens 100+ :completion-tokens 50+ :cache-creation-tokens 400+ :cache-read-tokens 1600})+ (is (= 400.0 (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))+ (is (= 1600.0 (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels))))++ (clear-llm-metrics!)++ (testing "zero / nil cache token fields do not increment their counters"+ (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :tag "test-tag"+ :prompt-tokens 100+ :completion-tokens 50+ :cache-creation-tokens 0+ :cache-read-tokens nil})+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels)))))));;; ------------------------------------------- track-token-usage! -------------------------------------------diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.cljindex 5a02fc5ae32..4d877d5f0f4 100644--- a/test/metabase/metabot/self/claude_test.clj+++ b/test/metabase/metabot/self/claude_test.clj@@ -1,14 +1,17 @@(ns metabase.metabot.self.claude-test(:require[clj-http.client :as http]+ [clojure.java.io :as io][clojure.test :refer :all][medley.core :as m][metabase.llm.settings :as llm.settings][metabase.metabot.self.claude :as claude][metabase.metabot.self.core :as self.core]+ [metabase.metabot.self.debug :as debug][metabase.metabot.test-util :as metabot.tu][metabase.premium-features.core :as premium-features]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.util.json :as json]))(set! *warn-on-reflection* true)@@ -30,7 +33,11 @@(testing "through full pipeline produces text + usage"(is (=? [{:type :start :id string?}{:type :text :id string? :text string?}- {:type :usage :id string? :model string? :usage {:promptTokens pos-int?}}]+ {:type :usage :id string? :model string?+ :usage {:promptTokens pos-int?+ :completionTokens nat-int?+ :cacheCreationTokens nat-int?+ :cacheReadTokens nat-int?}}](into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))(deftest ^:parallel claude-tool-input-conv-test@@ -43,7 +50,11 @@(testing "through full pipeline produces tool-input + usage"(is (=? [{:type :start}{:type :tool-input :arguments map?}- {:type :usage :model string? :usage {:promptTokens pos-int?}}]+ {:type :usage :model string?+ :usage {:promptTokens pos-int?+ :completionTokens nat-int?+ :cacheCreationTokens nat-int?+ :cacheReadTokens nat-int?}}](into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))(deftest ^:parallel claude-text-and-tool-input-conv-test@@ -60,7 +71,11 @@(is (=? [{:type :start}{:type :text :text string?}{:type :tool-input :function "get-time" :arguments {:tz string?}}- {:type :usage :model string? :usage {:promptTokens pos-int?}}]+ {:type :usage :model string?+ :usage {:promptTokens pos-int?+ :completionTokens nat-int?+ :cacheCreationTokens nat-int?+ :cacheReadTokens nat-int?}}](into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))(deftest ^:parallel claude-lite-aisdk-xf-test@@ -71,11 +86,61 @@(testing "lite-aisdk-xf collects tool inputs"(is (=? [{:type :start}{:type :tool-input :function "get-time" :arguments {:tz string?}}- {:type :usage :model string? :usage {:promptTokens pos-int?}}]+ {:type :usage :model string?+ :usage {:promptTokens pos-int?+ :completionTokens nat-int?+ :cacheCreationTokens nat-int?+ :cacheReadTokens nat-int?}}](remove #(= :text (:type %)) res))))(testing "lite-aisdk-xf streams text deltas"(is (< 10 (count (filter #(= :text (:type %)) res)))))))+(deftest ^:parallel claude-usage-chunk-cache-fields-test+ (testing "cache_creation_input_tokens / cache_read_input_tokens from Claude are surfaced on the :usage chunk;+ :promptTokens is the total input (fresh + cache_creation + cache_read)"+ (let [events [{:type "message_start"+ :message {:id "msg-1"+ :model "claude-haiku-4-5"+ :usage {:input_tokens 100+ :output_tokens 0+ :cache_creation_input_tokens 250+ :cache_read_input_tokens 4200}}}+ {:type "content_block_start" :index 0 :content_block {:type "text" :id "text-1"}}+ {:type "content_block_delta" :index 0 :delta {:type "text_delta" :text "ok"}}+ {:type "content_block_stop" :index 0}+ {:type "message_delta"+ :delta {:stop_reason "end_turn"}+ :usage {:input_tokens 100+ :output_tokens 7+ :cache_creation_input_tokens 250+ :cache_read_input_tokens 4200}}+ {:type "message_stop"}]+ chunks (into [] (claude/claude->aisdk-chunks-xf) events)+ usage (first (filter #(= :usage (:type %)) chunks))]+ ;; promptTokens = 100 fresh + 250 cache_creation + 4200 cache_read = 4550+ (is (=? {:type :usage+ :id "msg-1"+ :model "claude-haiku-4-5"+ :usage {:promptTokens 4550+ :completionTokens 7+ :cacheCreationTokens 250+ :cacheReadTokens 4200}}+ usage))))++ (testing "missing cache fields default to 0"+ (let [events [{:type "message_start"+ :message {:id "msg-2"+ :model "claude-haiku-4-5"+ :usage {:input_tokens 10 :output_tokens 0}}}+ {:type "message_delta"+ :delta {:stop_reason "end_turn"}+ :usage {:input_tokens 10 :output_tokens 3}}+ {:type "message_stop"}]+ chunks (into [] (claude/claude->aisdk-chunks-xf) events)+ usage (first (filter #(= :usage (:type %)) chunks))]+ (is (= {:cacheCreationTokens 0 :cacheReadTokens 0}+ (select-keys (:usage usage) [:cacheCreationTokens :cacheReadTokens]))))))+;;; ──────────────────────────────────────────────────────────────────;;; parts->claude-messages tests;;; ──────────────────────────────────────────────────────────────────@@ -170,6 +235,7 @@llm.settings/llm-proxy-base-url "https://proxy.example"](testing "Prefers BYOK over ai proxy"(with-redefs [self.core/sse-reducible identity+ debug/capture-stream (fn [r _] r)http/request (fn [req] {:body req})](is (=? {:method :post:url "https://api.anthropic.com/v1/messages"@@ -180,6 +246,7 @@(testing "Uses ai proxy when explicitly requested"(with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)self.core/sse-reducible identity+ debug/capture-stream (fn [r _] r)http/request (fn [req] {:body req})](is (=? {:method :post:url "https://proxy.example/anthropic/v1/messages"@@ -203,6 +270,105 @@#"No Anthropic API key is set"(claude/claude-raw {:input [{:role :user :content "hi"}]}))))))))))+(defn- capture-claude-request-body!+ "Invoke `claude-raw` with stubbed HTTP, returning the decoded request body map."+ [opts]+ (let [captured (atom nil)]+ (with-redefs [self.core/sse-reducible identity+ http/request (fn [req]+ (reset! captured (json/decode+kw (:body req)))+ {:body req})]+ (claude/claude-raw opts))+ @captured))++(deftest claude-tools-cache-breakpoint-test+ (mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-test"]+ (let [tools [(metabot.tu/get-time-tool) (metabot.tu/convert-currency-tool)]+ input [{:role :user :content "hi"}]]+ (testing "cache_control is attached to the last tool only"+ (let [body (capture-claude-request-body! {:input input :tools tools})+ [t1 t2] (:tools body)]+ (is (= 2 (count (:tools body))))+ (is (not (contains? t1 :cache_control)))+ (is (= {:type "ephemeral"} (:cache_control t2)))))++ (testing "no :tools key in request when no tools passed"+ (let [body (capture-claude-request-body! {:input input})]+ (is (not (contains? body :tools)))))++ (testing "no cache_control on structured-output path (schema set)"+ (let [body (capture-claude-request-body!+ {:input input+ :schema {:type "object" :properties {:answer {:type "string"}}}})]+ (is (= 1 (count (:tools body))))+ (is (= "structured_output" (-> body :tools first :name)))+ (is (not (contains? (-> body :tools first) :cache_control))))))))++(deftest claude-auto-cache-breakpoint-test+ (mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-test"]+ (let [input [{:role :user :content "hi"}]]+ (testing "top-level cache_control is set on every request (enables automatic caching of message history)"+ (is (= {:type "ephemeral"}+ (:cache_control (capture-claude-request-body! {:input input}))))+ (is (= {:type "ephemeral"}+ (:cache_control (capture-claude-request-body!+ {:input input+ :system "You are a helpful assistant."+ :tools [(metabot.tu/get-time-tool)]})))))++ (testing "top-level cache_control is set on the structured-output path too"+ (is (= {:type "ephemeral"}+ (:cache_control (capture-claude-request-body!+ {:input input+ :schema {:type "object" :properties {:answer {:type "string"}}}}))))))))++(deftest claude-system-cache-breakpoint-test+ (mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-test"]+ (let [input [{:role :user :content "hi"}]]+ (testing "system prompt without sentinel is wrapped as a single cached content block"+ (let [body (capture-claude-request-body!+ {:input input+ :system "You are a helpful assistant."})]+ (is (= [{:type "text"+ :text "You are a helpful assistant."+ :cache_control {:type "ephemeral"}}]+ (:system body)))))++ (testing "system prompt with sentinel is split into cached prefix + uncached suffix"+ (let [body (capture-claude-request-body!+ {:input input+ :system "Stable prefix content.\n\n<<<METABOT_CACHE_BREAKPOINT>>>\n\nDynamic suffix content."})]+ (is (= [{:type "text"+ :text "Stable prefix content."+ :cache_control {:type "ephemeral"}}+ {:type "text"+ :text "Dynamic suffix content."}]+ (:system body)))))++ (testing "no :system key when system is not provided"+ (let [body (capture-claude-request-body! {:input input})]+ (is (not (contains? body :system))))))))++(deftest system-templates-cache-breakpoint-presence-test+ (testing "every selmer template that contains per-request volatile content carries exactly one cache breakpoint sentinel"+ (let [system-dir (io/file (io/resource "metabot/prompts/system"))+ templates (->> (.listFiles system-dir)+ (filter #(re-find #"\.selmer$" (.getName ^java.io.File %))))]+ (doseq [^java.io.File f templates]+ (let [body (slurp f)+ n (count (re-seq #"<<<METABOT_CACHE_BREAKPOINT>>>" body))+ has-volatile? (some #(re-find % body)+ [#"\{\{\s*current_time\s*\}\}"+ #"\{%\s*if\s+recent_views\s*%\}"+ #"\{%\s*if\s+current_user_info\s*%\}"+ #"\{%\s*if\s+viewing_context\s*%\}"+ #"\{\{\s*viewing_context"+ #"\{\{\s*first_day_of_week\s*\}\}"])]+ (testing (.getName f)+ (if has-volatile?+ (is (= 1 n) "exactly one sentinel expected when template references volatile context vars")+ (is (zero? n) "no sentinel expected when template has no volatile context vars"))))))))+(deftest claude-list-models-auth-preferences-test(mt/with-premium-features #{:metabase-ai-managed}(mt/with-dynamic-fn-redefs [premium-features/premium-embedding-token (constantly "proxy-token")]diff --git a/test/metabase/metabot/self/openai_test.clj b/test/metabase/metabot/self/openai_test.cljindex 3411212829b..cd9b1233737 100644--- a/test/metabase/metabot/self/openai_test.clj+++ b/test/metabase/metabot/self/openai_test.clj@@ -5,6 +5,7 @@[medley.core :as m][metabase.llm.settings :as llm.settings][metabase.metabot.self.core :as self.core]+ [metabase.metabot.self.debug :as debug][metabase.metabot.self.openai :as openai][metabase.metabot.test-util :as metabot.tu][metabase.premium-features.core :as premium-features]@@ -171,6 +172,7 @@llm.settings/llm-proxy-base-url "https://proxy.example"](testing "Prefers BYOK over ai proxy"(with-redefs [self.core/sse-reducible identity+ debug/capture-stream (fn [r _] r)http/request (fn [req] {:body req})](is (=? {:method :post:url "https://api.openai.com/v1/responses"@@ -181,6 +183,7 @@(testing "Uses ai proxy when explicitly requested"(mt/with-temporary-setting-values [llm.settings/llm-openai-api-key nil](with-redefs [self.core/sse-reducible identity+ debug/capture-stream (fn [r _] r)http/request (fn [req] {:body req})](is (=? {:method :post:url "https://proxy.example/openai/v1/responses"diff --git a/test/metabase/metabot/self/openrouter_test.clj b/test/metabase/metabot/self/openrouter_test.cljindex 114700c3657..59b16e0dbdc 100644--- a/test/metabase/metabot/self/openrouter_test.clj+++ b/test/metabase/metabot/self/openrouter_test.clj@@ -5,6 +5,7 @@[medley.core :as m][metabase.llm.settings :as llm.settings][metabase.metabot.self.core :as self.core]+ [metabase.metabot.self.debug :as debug][metabase.metabot.self.openrouter :as openrouter][metabase.metabot.test-util :as metabot.tu][metabase.premium-features.core :as premium-features]@@ -191,6 +192,7 @@llm.settings/llm-proxy-base-url "https://proxy.example"](testing "Prefers BYOK over ai proxy"(with-redefs [self.core/sse-reducible identity+ debug/capture-stream (fn [r _] r)http/request (fn [req] {:body req})](is (=? {:method :post:url "https://openrouter.ai/api/v1/chat/completions"@@ -201,6 +203,7 @@(testing "Uses ai proxy when explicitly requested"(mt/with-temporary-setting-values [llm.settings/llm-openrouter-api-key nil](with-redefs [self.core/sse-reducible identity+ debug/capture-stream (fn [r _] r)http/request (fn [req] {:body req})](is (=? {:method :post:url "https://proxy.example/openrouter/v1/chat/completions"diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 3cdd4416e38..8fe1e79ea65 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -677,7 +677,43 @@(run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"})))(is (== 100 (mt/metric-value system :metabase-metabot/llm-input-tokens labels)))(is (== 25 (mt/metric-value system :metabase-metabot/llm-output-tokens labels)))- (is (== 125 (:sum (mt/metric-value system :metabase-metabot/llm-tokens-per-call labels)))))))))+ (is (== 125 (:sum (mt/metric-value system :metabase-metabot/llm-tokens-per-call labels)))))++ (analytics/clear! :metabase-metabot/llm-input-tokens)+ (analytics/clear! :metabase-metabot/llm-output-tokens)+ (analytics/clear! :metabase-metabot/llm-cache-creation-tokens)+ (analytics/clear! :metabase-metabot/llm-cache-read-tokens)++ (testing "increments cache token counters when the :usage part carries cache fields"+ ;; :promptTokens is the pre-summed total input (40 fresh + 300 cache_creation + 1200 cache_read = 1540).+ (with-redefs [openrouter/openrouter+ (constantly (test-util/mock-llm-response+ [{:type :start :id "m1"}+ {:type :usage+ :usage {:promptTokens 1540+ :completionTokens 10+ :cacheCreationTokens 300+ :cacheReadTokens 1200}+ :model "test-model"}]))]+ (run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"})))+ (is (== 300 (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))+ (is (== 1200 (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels))))++ (analytics/clear! :metabase-metabot/llm-input-tokens)+ (analytics/clear! :metabase-metabot/llm-output-tokens)+ (analytics/clear! :metabase-metabot/llm-cache-creation-tokens)+ (analytics/clear! :metabase-metabot/llm-cache-read-tokens)++ (testing "does not increment cache counters when cache fields are absent or zero"+ (with-redefs [openrouter/openrouter+ (constantly (test-util/mock-llm-response+ [{:type :start :id "m1"}+ {:type :usage+ :usage {:promptTokens 10 :completionTokens 5}+ :model "test-model"}]))]+ (run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"})))+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels))))))))(deftest call-llm-structured-prometheus-test(mt/with-prometheus-system! [_ system]@@ -770,12 +806,18 @@(deftest call-llm-snowplow-test(testing "fires :snowplow/token_usage and :snowplow/ai_service_event for call-llm with a tool call"(let [rasta-id (mt/user->id :rasta)]+ ;; The adapter pre-sums input + cache_creation + cache_read into :promptTokens,+ ;; so the mock supplies the already-summed value (950 = 100 fresh + 50 cache_creation + 800 cache_read).+ ;; total_tokens reverts to prompt + completion = 950 + 20 = 970.(with-redefs [openrouter/openrouter(constantly (test-util/mock-llm-response[{:type :start :id "msg-1"}{:type :tool-input :id "call-1" :function "get-time":arguments {:tz "UTC"}}- {:type :usage :usage {:promptTokens 100 :completionTokens 20}+ {:type :usage :usage {:promptTokens 950+ :completionTokens 20+ :cacheCreationTokens 50+ :cacheReadTokens 800}:model "test-model" :id "msg-1"}]))](mt/with-current-user rasta-id(snowplow-test/with-fake-snowplow-collector@@ -784,15 +826,17 @@token-events (filter #(contains? (:data %) "total_tokens") events)tool-events (filter #(= "agent_used_tool" (get-in % [:data "event"])) events)](is (=? [{:user-id (str rasta-id)- :data {"model_id" "openrouter/test-model"- "total_tokens" 120- "prompt_tokens" 100- "completion_tokens" 20- "estimated_costs_usd" 0.0- "duration_ms" nat-int?- "source" "metabot_agent"- "tag" "test-tag"- "session_id" "00000000-0000-0000-0000-000000000002"}}]+ :data {"model_id" "openrouter/test-model"+ "total_tokens" 970+ "prompt_tokens" 950+ "completion_tokens" 20+ "cache_creation_tokens" 50+ "cache_read_tokens" 800+ "estimated_costs_usd" 0.0+ "duration_ms" nat-int?+ "source" "metabot_agent"+ "tag" "test-tag"+ "session_id" "00000000-0000-0000-0000-000000000002"}}]token-events))(is (=? [{:user-id (str rasta-id):data {"event" "agent_used_tool"