A platform for community discussion. Free, open, simple.
FEATURE: show conversation spending in debug modal (#39364)
Samd3d90aeb5048
diff --git a/plugins/discourse-ai/app/models/ai_api_audit_log.rb b/plugins/discourse-ai/app/models/ai_api_audit_log.rbindex b5048b7b1cc..612c66b661e 100644--- a/plugins/discourse-ai/app/models/ai_api_audit_log.rb+++ b/plugins/discourse-ai/app/models/ai_api_audit_log.rb@@ -21,6 +21,27 @@ class AiApiAuditLog < ActiveRecord::BaseBedrockConverse = 11end+ def self.token_and_spending_stats(scope)+ spending_expr = LlmModel.spending_sql(table_name)+ request_tokens, response_tokens, cache_read_tokens, cache_write_tokens, spending, model_count =+ scope.joins("LEFT JOIN llm_models ON llm_models.id = #{table_name}.llm_id").pick(+ Arel.sql("COALESCE(SUM(COALESCE(#{table_name}.request_tokens, 0)), 0)"),+ Arel.sql("COALESCE(SUM(COALESCE(#{table_name}.response_tokens, 0)), 0)"),+ Arel.sql("COALESCE(SUM(COALESCE(#{table_name}.cache_read_tokens, 0)), 0)"),+ Arel.sql("COALESCE(SUM(COALESCE(#{table_name}.cache_write_tokens, 0)), 0)"),+ Arel.sql("COALESCE(SUM(#{spending_expr}), 0) / 1000000.0"),+ Arel.sql("COUNT(llm_models.id)"),+ )++ {+ request_tokens: request_tokens.to_i,+ response_tokens: response_tokens.to_i,+ cache_read_tokens: cache_read_tokens.to_i,+ cache_write_tokens: cache_write_tokens.to_i,+ spending: model_count.to_i.positive? ? spending.to_f.round(6) : nil,+ }+ end+def next_log_idself.class.where("id > ?", id).where(topic_id: topic_id).order(id: :asc).pluck(:id).firstenddiff --git a/plugins/discourse-ai/app/models/llm_model.rb b/plugins/discourse-ai/app/models/llm_model.rbindex d71940434b6..5c8d39dc0b7 100644--- a/plugins/discourse-ai/app/models/llm_model.rb+++ b/plugins/discourse-ai/app/models/llm_model.rb@@ -16,6 +16,43 @@ class LlmModel < ActiveRecord::Base"text" => "txt",}.freeze+ COST_COMPONENTS = {+ input: {+ tokens: :request_tokens,+ cost: :input_cost,+ },+ output: {+ tokens: :response_tokens,+ cost: :output_cost,+ },+ cache_read: {+ tokens: :cache_read_tokens,+ cost: :cached_input_cost,+ },+ cache_write: {+ tokens: :cache_write_tokens,+ cost: :cache_write_cost,+ },+ }.freeze++ def self.spending_component_sql(component, table)+ info = COST_COMPONENTS.fetch(component)+ qt = connection.quote_table_name(table.to_s)+ "COALESCE(#{qt}.#{info[:tokens]}, 0) * COALESCE(llm_models.#{info[:cost]}, 0)"+ end++ def self.spending_sql(table)+ COST_COMPONENTS.keys.map { |k| spending_component_sql(k, table) }.join(" + ")+ end++ def spending_for(record)+ total =+ COST_COMPONENTS.values.sum do |info|+ record.public_send(info[:tokens]).to_i * public_send(info[:cost]).to_f+ end+ (total / 1_000_000.0).round(6)+ end+has_many :llm_quotas, dependent: :destroyhas_one :llm_credit_allocation, dependent: :destroyhas_many :llm_feature_credit_costs, dependent: :destroydiff --git a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rbindex 923bbc01f6a..a4a1c74af8c 100644--- a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb+++ b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb@@ -17,5 +17,48 @@ class AiApiAuditLogSerializer < ApplicationSerializer:language_model,:created_at,:prev_log_id,- :next_log_id+ :next_log_id,+ :spending,+ :conversation_request_tokens,+ :conversation_response_tokens,+ :conversation_cache_read_tokens,+ :conversation_cache_write_tokens,+ :conversation_spending++ def spending+ object.llm_model&.spending_for(object)+ end++ def conversation_request_tokens+ conversation_stats && conversation_stats[:request_tokens]+ end++ def conversation_response_tokens+ conversation_stats && conversation_stats[:response_tokens]+ end++ def conversation_cache_read_tokens+ conversation_stats && conversation_stats[:cache_read_tokens]+ end++ def conversation_cache_write_tokens+ conversation_stats && conversation_stats[:cache_write_tokens]+ end++ def conversation_spending+ conversation_stats && conversation_stats[:spending]+ end++ private++ def conversation_stats+ return @conversation_stats if defined?(@conversation_stats)++ @conversation_stats =+ if object.topic_id.blank?+ nil+ else+ AiApiAuditLog.token_and_spending_stats(AiApiAuditLog.where(topic_id: object.topic_id))+ end+ endenddiff --git a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjs b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjsindex ee0a7a233d7..c8f27088b2d 100644--- a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjs+++ b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjs@@ -124,6 +124,73 @@ export default class DebugAiModal extends Component {e.preventDefault();}+ get formattedSpending() {+ return this.formatCost(this.info?.spending);+ }++ get formattedConversationSpending() {+ return this.formatCost(this.info?.conversation_spending);+ }++ get turnCacheLabel() {+ return this.cacheLabel(+ this.info?.cache_read_tokens,+ this.info?.cache_write_tokens+ );+ }++ get conversationCacheLabel() {+ return this.cacheLabel(+ this.info?.conversation_cache_read_tokens,+ this.info?.conversation_cache_write_tokens+ );+ }++ get showConversationLine() {+ if (!this.info) {+ return false;+ }++ return (+ this.info.conversation_spending != null ||+ this.info.conversation_request_tokens > 0 ||+ this.info.conversation_response_tokens > 0 ||+ this.info.conversation_cache_read_tokens > 0 ||+ this.info.conversation_cache_write_tokens > 0+ );+ }++ cacheLabel(read, write) {+ const hasRead = read && read > 0;+ const hasWrite = write && write > 0;++ if (hasRead && hasWrite) {+ return i18n("discourse_ai.ai_bot.debug_ai_modal.cache_both", {+ read,+ write,+ });+ }+ if (hasRead) {+ return i18n("discourse_ai.ai_bot.debug_ai_modal.cache_read_only", {+ read,+ });+ }+ if (hasWrite) {+ return i18n("discourse_ai.ai_bot.debug_ai_modal.cache_write_only", {+ write,+ });+ }+ return null;+ }++ formatCost(value) {+ if (value == null || Number(value) === 0) {+ return null;+ }++ return `$${Number(value).toFixed(4)}`;+ }+<template><DModalclass="ai-debug-modal"@@ -143,29 +210,49 @@ export default class DebugAiModal extends Component {{{on "click" this.responseClicked}}>{{i18n "discourse_ai.ai_bot.debug_ai_modal.response"}}</a></li></ul>- <div class="ai-debug-modal__tokens">- <span class="ai-debug-modal__tokens__count">- {{i18n "discourse_ai.ai_bot.debug_ai_modal.request_tokens"}}- {{this.info.request_tokens}}- </span>- <span class="ai-debug-modal__tokens__count">- {{i18n "discourse_ai.ai_bot.debug_ai_modal.response_tokens"}}- {{this.info.response_tokens}}- </span>- {{#if this.info.cache_read_tokens}}- <span class="ai-debug-modal__tokens__count">- {{i18n "discourse_ai.ai_bot.debug_ai_modal.cache_read_tokens"}}- {{this.info.cache_read_tokens}}- </span>- {{/if}}- {{#if this.info.cache_write_tokens}}- <span class="ai-debug-modal__tokens__count">- {{i18n "discourse_ai.ai_bot.debug_ai_modal.cache_write_tokens"}}- {{this.info.cache_write_tokens}}- </span>+ <div class="ai-debug-modal__stats">+ <p class="ai-debug-modal__stats-line">+ <strong class="ai-debug-modal__stats-line__label">+ {{i18n "discourse_ai.ai_bot.debug_ai_modal.this_turn"}}+ </strong>+ {{i18n+ "discourse_ai.ai_bot.debug_ai_modal.tokens_summary"+ request_tokens=this.info.request_tokens+ response_tokens=this.info.response_tokens+ }}+ {{#if this.turnCacheLabel}}+ <span+ class="ai-debug-modal__stats-line__cache"+ >{{this.turnCacheLabel}}</span>+ {{/if}}+ {{#if this.formattedSpending}}+ <span class="ai-debug-modal__stats-line__cost">:+ {{this.formattedSpending}}</span>+ {{/if}}+ </p>+ {{#if this.showConversationLine}}+ <p class="ai-debug-modal__stats-line">+ <strong class="ai-debug-modal__stats-line__label">+ {{i18n "discourse_ai.ai_bot.debug_ai_modal.whole_conversation"}}+ </strong>+ {{i18n+ "discourse_ai.ai_bot.debug_ai_modal.tokens_summary"+ request_tokens=this.info.conversation_request_tokens+ response_tokens=this.info.conversation_response_tokens+ }}+ {{#if this.conversationCacheLabel}}+ <span+ class="ai-debug-modal__stats-line__cache"+ >{{this.conversationCacheLabel}}</span>+ {{/if}}+ {{#if this.formattedConversationSpending}}+ <span class="ai-debug-modal__stats-line__cost">:+ {{this.formattedConversationSpending}}</span>+ {{/if}}+ </p>{{/if}}</div>- <div class="debug-ai-modal__preview">+ <div class="ai-debug-modal__preview">{{this.htmlContext}}</div></:body>@@ -199,7 +286,7 @@ export default class DebugAiModal extends Component {@label="discourse_ai.ai_bot.debug_ai_modal.next_log"/>{{/if}}- <span class="ai-debut-modal__just-copied">{{this.justCopiedText}}</span>+ <span class="ai-debug-modal__just-copied">{{this.justCopiedText}}</span></:footer></DModal></template>diff --git a/plugins/discourse-ai/assets/stylesheets/modules/ai-bot/common/bot-replies.scss b/plugins/discourse-ai/assets/stylesheets/modules/ai-bot/common/bot-replies.scssindex c6280cd6105..00ff9a90cf9 100644--- a/plugins/discourse-ai/assets/stylesheets/modules/ai-bot/common/bot-replies.scss+++ b/plugins/discourse-ai/assets/stylesheets/modules/ai-bot/common/bot-replies.scss@@ -155,8 +155,43 @@ span.onebox-ai-llm-title {}}-.ai-debug-modal__tokens__count {- display: block;+.ai-debug-modal__stats {+ margin-bottom: 1em;+ font-size: var(--font-down-1);+ font-variant-numeric: tabular-nums;+}++.ai-debug-modal__stats-line {+ margin: 0 0 0.25em;++ &__label {+ color: var(--primary-high);+ margin-right: 0.25em;+ }++ &__cache {+ color: var(--primary-high);+ margin-left: 0.25em;+ }+}++.ai-debug-modal__just-copied {+ font-size: var(--font-down-1);+ color: var(--success);+ margin-left: auto;+ align-self: center;+}++.ai-debug-modal__preview {+ background-color: var(--primary-very-low);+ border: 1px solid var(--primary-low);+ border-radius: var(--d-border-radius);+ padding: 0.75em 1em;+ max-height: 60vh;+ overflow: auto;+ font-family: var(--d-font-family--monospace);+ font-size: var(--font-down-1);+ line-height: 1.45;}.d-modal ul.ai-debug-modal__nav {diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.ymlindex c703237f7cc..815aa2bede9 100644--- a/plugins/discourse-ai/config/locales/client.en.yml+++ b/plugins/discourse-ai/config/locales/client.en.yml@@ -1177,10 +1177,12 @@ en:title: "View AI interaction"copy_request: "Copy request"copy_response: "Copy response"- request_tokens: "Request tokens:"- response_tokens: "Response tokens:"- cache_read_tokens: "Cache read tokens:"- cache_write_tokens: "Cache write tokens:"+ this_turn: "This turn:"+ whole_conversation: "Conversation:"+ tokens_summary: "%{request_tokens} in → %{response_tokens} out"+ cache_both: "(cache %{read} read, %{write} write)"+ cache_read_only: "(cache %{read} read)"+ cache_write_only: "(cache %{write} write)"request: "Request"response: "Response"next_log: "Next"diff --git a/plugins/discourse-ai/lib/completions/report.rb b/plugins/discourse-ai/lib/completions/report.rbindex 5006700164b..57ac40bd8d1 100644--- a/plugins/discourse-ai/lib/completions/report.rb+++ b/plugins/discourse-ai/lib/completions/report.rb@@ -52,24 +52,28 @@ module DiscourseAienddef total_input_spending- model_costs.sum { |row| row.input_cost.to_f * row.total_request_tokens.to_i / 1_000_000.0 }+ spending_component_total(:input)enddef total_output_spending- model_costs.sum do |row|- row.output_cost.to_f * row.total_response_tokens.to_i / 1_000_000.0- end+ spending_component_total(:output)enddef total_cache_read_spending- model_costs.sum do |row|- row.cached_input_cost.to_f * row.total_cache_read_tokens.to_i / 1_000_000.0- end+ spending_component_total(:cache_read)enddef total_cache_write_spending+ spending_component_total(:cache_write)+ end++ def spending_component_total(component)+ info = LlmModel::COST_COMPONENTS.fetch(component)+ cost_attr = info[:cost]+ tokens_attr = "total_#{info[:tokens]}"+model_costs.sum do |row|- row.cache_write_cost.to_f * row.total_cache_write_tokens.to_i / 1_000_000.0+ row.public_send(cost_attr).to_f * row.public_send(tokens_attr).to_i / 1_000_000.0endend@@ -242,13 +246,12 @@ module DiscourseAienddef token_count_and_total_columns- [- *token_total_columns,- "SUM(COALESCE(request_tokens, 0) * COALESCE(llm_models.input_cost, 0)) / 1000000.0 as input_spending",- "SUM(COALESCE(response_tokens, 0) * COALESCE(llm_models.output_cost, 0)) / 1000000.0 as output_spending",- "SUM(COALESCE(cache_read_tokens, 0) * COALESCE(llm_models.cached_input_cost, 0)) / 1000000.0 as cache_read_spending",- "SUM(COALESCE(cache_write_tokens, 0) * COALESCE(llm_models.cache_write_cost, 0)) / 1000000.0 as cache_write_spending",- ]+ spending_columns =+ LlmModel::COST_COMPONENTS.keys.map do |component|+ expr = LlmModel.spending_component_sql(component, :ai_api_request_stats)+ "SUM(#{expr}) / 1000000.0 as #{component}_spending"+ end+ [*token_total_columns, *spending_columns]endendenddiff --git a/plugins/discourse-ai/spec/requests/ai_bot/bot_controller_spec.rb b/plugins/discourse-ai/spec/requests/ai_bot/bot_controller_spec.rbindex 75958c63fbe..e16728ac408 100644--- a/plugins/discourse-ai/spec/requests/ai_bot/bot_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/ai_bot/bot_controller_spec.rb@@ -83,6 +83,128 @@ RSpec.describe DiscourseAi::AiBot::BotController doexpect(response.status).to eq(200)expect(response.parsed_body["id"]).to eq(log1.id)end++ context "with conversation totals and spending" do+ fab!(:llm_model) do+ Fabricate(+ :llm_model,+ input_cost: 3.0,+ output_cost: 6.0,+ cached_input_cost: 1.0,+ cache_write_cost: 2.0,+ )+ end++ fab!(:other_llm_model) do+ Fabricate(+ :llm_model,+ name: "other-model",+ input_cost: 10.0,+ output_cost: 20.0,+ cached_input_cost: 5.0,+ cache_write_cost: 8.0,+ )+ end++ let(:allowed_user) { pm_topic.topic_allowed_users.first.user }++ before do+ sign_in(allowed_user)+ Group.refresh_automatic_groups!+ SiteSetting.ai_bot_debugging_allowed_groups = allowed_user.groups.first.id.to_s+ end++ it "computes per-turn spending and conversation totals across multiple models" do+ log_a =+ AiApiAuditLog.create!(+ post_id: pm_post.id,+ provider_id: 1,+ topic_id: pm_topic.id,+ llm_id: llm_model.id,+ raw_request_payload: "req",+ raw_response_payload: "res",+ request_tokens: 1_000_000,+ response_tokens: 500_000,+ cache_read_tokens: 200_000,+ cache_write_tokens: 100_000,+ )++ AiApiAuditLog.create!(+ post_id: pm_post2.id,+ provider_id: 1,+ topic_id: pm_topic.id,+ llm_id: other_llm_model.id,+ raw_request_payload: "req",+ raw_response_payload: "res",+ request_tokens: 2_000_000,+ response_tokens: 1_000_000,+ )++ get "/discourse-ai/ai-bot/show-debug-info/#{log_a.id}"+ expect(response.status).to eq(200)++ body = response.parsed_body+ expected_turn_spending =+ (1_000_000 * 3.0 + 500_000 * 6.0 + 200_000 * 1.0 + 100_000 * 2.0) / 1_000_000.0+ expect(body["spending"]).to be_within(0.000001).of(expected_turn_spending)++ expect(body["conversation_request_tokens"]).to eq(3_000_000)+ expect(body["conversation_response_tokens"]).to eq(1_500_000)+ expect(body["conversation_cache_read_tokens"]).to eq(200_000)+ expect(body["conversation_cache_write_tokens"]).to eq(100_000)++ expected_conversation_spending =+ expected_turn_spending + (2_000_000 * 10.0 + 1_000_000 * 20.0) / 1_000_000.0+ expect(body["conversation_spending"]).to be_within(0.000001).of(+ expected_conversation_spending,+ )+ end++ it "returns nil spending when the log has no llm_model" do+ log_a =+ AiApiAuditLog.create!(+ post_id: pm_post.id,+ provider_id: 1,+ topic_id: pm_topic.id,+ raw_request_payload: "req",+ raw_response_payload: "res",+ request_tokens: 100,+ response_tokens: 200,+ )++ get "/discourse-ai/ai-bot/show-debug-info/#{log_a.id}"+ expect(response.status).to eq(200)++ body = response.parsed_body+ expect(body["spending"]).to be_nil+ expect(body["conversation_request_tokens"]).to eq(100)+ expect(body["conversation_response_tokens"]).to eq(200)+ expect(body["conversation_spending"]).to be_nil+ end++ it "returns nil conversation fields when topic_id is absent" do+ log_a =+ AiApiAuditLog.create!(+ provider_id: 1,+ llm_id: llm_model.id,+ raw_request_payload: "req",+ raw_response_payload: "res",+ request_tokens: 100,+ response_tokens: 200,+ )+ SiteSetting.ai_bot_debugging_allowed_groups = Group::AUTO_GROUPS[:admins].to_s+ sign_in(Fabricate(:admin))++ get "/discourse-ai/ai-bot/show-debug-info/#{log_a.id}"+ # no topic means the endpoint 404s, so assert via serializer directly+ serialized = AiApiAuditLogSerializer.new(log_a, root: false).as_json+ expect(serialized[:conversation_request_tokens]).to be_nil+ expect(serialized[:conversation_response_tokens]).to be_nil+ expect(serialized[:conversation_cache_read_tokens]).to be_nil+ expect(serialized[:conversation_cache_write_tokens]).to be_nil+ expect(serialized[:conversation_spending]).to be_nil+ end+ endenddescribe "#stop_streaming_response" do