Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Track AI time to first token (#42643)

Sam8799a36cbda3
diff --git a/db/structure.sql b/db/structure.sql
index 6d1d73db5bb..1eb5810955e 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -520,7 +520,8 @@ CREATE TABLE public.ai_api_audit_logs (
llm_id bigint,
response_status integer,
request_attempts jsonb,
- estimated_cost numeric(20,10)
+ estimated_cost numeric(20,10),
+ time_to_first_token_msecs integer
);
@@ -23172,6 +23173,7 @@ SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20260817054353'),
+('20260817054044'),
('20260814083721'),
('20260814060100'),
('20260814060000'),
diff --git a/plugins/discourse-ai/app/models/ai_api_audit_log.rb b/plugins/discourse-ai/app/models/ai_api_audit_log.rb
index 4dbea109e5e..6f293cce9b9 100644
--- a/plugins/discourse-ai/app/models/ai_api_audit_log.rb
+++ b/plugins/discourse-ai/app/models/ai_api_audit_log.rb
@@ -62,27 +62,28 @@ end
#
# Table name: ai_api_audit_logs
#
-# id :bigint not null, primary key
-# cache_read_tokens :integer
-# cache_write_tokens :integer
-# duration_msecs :integer
-# estimated_cost :decimal(20, 10)
-# feature_context :jsonb
-# feature_name :string(255)
-# language_model :string(255)
-# raw_request_payload :string
-# raw_response_payload :string
-# request_attempts :jsonb
-# request_tokens :integer
-# response_status :integer
-# response_tokens :integer
-# created_at :datetime not null
-# updated_at :datetime not null
-# llm_id :bigint
-# post_id :integer
-# provider_id :integer not null
-# topic_id :integer
-# user_id :integer
+# id :bigint not null, primary key
+# cache_read_tokens :integer
+# cache_write_tokens :integer
+# duration_msecs :integer
+# estimated_cost :decimal(20, 10)
+# feature_context :jsonb
+# feature_name :string(255)
+# language_model :string(255)
+# raw_request_payload :string
+# raw_response_payload :string
+# request_attempts :jsonb
+# request_tokens :integer
+# response_status :integer
+# response_tokens :integer
+# time_to_first_token_msecs :integer
+# created_at :datetime not null
+# updated_at :datetime not null
+# llm_id :bigint
+# post_id :integer
+# provider_id :integer not null
+# topic_id :integer
+# user_id :integer
#
# Indexes
#
diff --git a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb
index 07224f3a991..3722469b1e2 100644
--- a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb
+++ b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb
@@ -17,6 +17,8 @@ class AiApiAuditLogSerializer < ApplicationSerializer
:language_model,
:response_status,
:request_attempts,
+ :duration_msecs,
+ :time_to_first_token_msecs,
:created_at,
:prev_log_id,
:next_log_id,
diff --git a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjs b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjs
index e5f3dc64da7..fb018024861 100644
--- a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjs
+++ b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/debug-ai-modal.gjs
@@ -124,6 +124,33 @@ export default class DebugAiModal extends Component {
e.preventDefault();
}
+ get formattedDurationSummary() {
+ const durationSeconds = this.seconds(this.info?.duration_msecs);
+ const firstTokenSeconds = this.seconds(
+ this.info?.time_to_first_token_msecs
+ );
+
+ if (durationSeconds == null) {
+ return i18n("discourse_ai.ai_bot.debug_ai_modal.duration_unavailable");
+ }
+
+ if (firstTokenSeconds == null) {
+ return i18n(
+ "discourse_ai.ai_bot.debug_ai_modal.duration_without_first_token",
+ { duration_seconds: durationSeconds }
+ );
+ }
+
+ return i18n("discourse_ai.ai_bot.debug_ai_modal.duration", {
+ duration_seconds: durationSeconds,
+ first_token_seconds: firstTokenSeconds,
+ });
+ }
+
+ seconds(milliseconds) {
+ return milliseconds == null ? null : (milliseconds / 1000).toFixed(1);
+ }
+
get formattedSpending() {
return this.formatCost(this.info?.spending);
}
@@ -210,83 +237,97 @@ export default class DebugAiModal extends Component {
{{on "click" this.responseClicked}}
>{{i18n "discourse_ai.ai_bot.debug_ai_modal.response"}}</a></li>
</ul>
- <div class="ai-debug-modal__stats">
- <p class="ai-debug-modal__stats-line">
- <strong class="ai-debug-modal__stats-line__label">
- {{i18n "discourse_ai.ai_bot.debug_ai_modal.this_turn"}}
- </strong>
- {{i18n
- "discourse_ai.ai_bot.debug_ai_modal.tokens_summary"
- request_tokens=this.info.request_tokens
- response_tokens=this.info.response_tokens
- }}
- {{#if this.turnCacheLabel}}
- <span
- class="ai-debug-modal__stats-line__cache"
- >{{this.turnCacheLabel}}</span>
- {{/if}}
- {{#if this.formattedSpending}}
- <span class="ai-debug-modal__stats-line__cost">:
- {{this.formattedSpending}}</span>
- {{/if}}
- </p>
- {{#if this.showConversationLine}}
+ {{#if this.info}}
+ <div class="ai-debug-modal__stats">
<p class="ai-debug-modal__stats-line">
<strong class="ai-debug-modal__stats-line__label">
- {{i18n "discourse_ai.ai_bot.debug_ai_modal.whole_conversation"}}
+ {{i18n "discourse_ai.ai_bot.debug_ai_modal.this_turn"}}
</strong>
{{i18n
"discourse_ai.ai_bot.debug_ai_modal.tokens_summary"
- request_tokens=this.info.conversation_request_tokens
- response_tokens=this.info.conversation_response_tokens
+ request_tokens=this.info.request_tokens
+ response_tokens=this.info.response_tokens
}}
- {{#if this.conversationCacheLabel}}
+ {{#if this.turnCacheLabel}}
<span
class="ai-debug-modal__stats-line__cache"
- >{{this.conversationCacheLabel}}</span>
+ >{{this.turnCacheLabel}}</span>
{{/if}}
- {{#if this.formattedConversationSpending}}
+ {{#if this.formattedSpending}}
<span class="ai-debug-modal__stats-line__cost">:
- {{this.formattedConversationSpending}}</span>
+ {{this.formattedSpending}}</span>
{{/if}}
</p>
- {{/if}}
- </div>
- <div class="ai-debug-modal__preview">
- {{this.htmlContext}}
- </div>
+ <p class="ai-debug-modal__stats-line ai-debug-modal__duration">
+ <strong class="ai-debug-modal__stats-line__label">
+ {{i18n "discourse_ai.ai_bot.debug_ai_modal.duration_label"}}
+ </strong>
+ {{this.formattedDurationSummary}}
+ </p>
+ {{#if this.showConversationLine}}
+ <p class="ai-debug-modal__stats-line">
+ <strong class="ai-debug-modal__stats-line__label">
+ {{i18n
+ "discourse_ai.ai_bot.debug_ai_modal.whole_conversation"
+ }}
+ </strong>
+ {{i18n
+ "discourse_ai.ai_bot.debug_ai_modal.tokens_summary"
+ request_tokens=this.info.conversation_request_tokens
+ response_tokens=this.info.conversation_response_tokens
+ }}
+ {{#if this.conversationCacheLabel}}
+ <span
+ class="ai-debug-modal__stats-line__cache"
+ >{{this.conversationCacheLabel}}</span>
+ {{/if}}
+ {{#if this.formattedConversationSpending}}
+ <span class="ai-debug-modal__stats-line__cost">:
+ {{this.formattedConversationSpending}}</span>
+ {{/if}}
+ </p>
+ {{/if}}
+ </div>
+ <div class="ai-debug-modal__preview">
+ {{this.htmlContext}}
+ </div>
+ {{/if}}
</:body>
<:footer>
- <DButton
- class="btn confirm"
- @icon="copy"
- @action={{this.copyRequest}}
- @label="discourse_ai.ai_bot.debug_ai_modal.copy_request"
- />
- <DButton
- class="btn confirm"
- @icon="copy"
- @action={{this.copyResponse}}
- @label="discourse_ai.ai_bot.debug_ai_modal.copy_response"
- />
- {{#if this.info.prev_log_id}}
+ {{#if this.info}}
<DButton
- class="btn"
- @icon="angles-left"
- @action={{this.prevLog}}
- @label="discourse_ai.ai_bot.debug_ai_modal.previous_log"
+ class="btn confirm"
+ @icon="copy"
+ @action={{this.copyRequest}}
+ @label="discourse_ai.ai_bot.debug_ai_modal.copy_request"
/>
- {{/if}}
- {{#if this.info.next_log_id}}
<DButton
- class="btn"
- @icon="angles-right"
- @action={{this.nextLog}}
- @label="discourse_ai.ai_bot.debug_ai_modal.next_log"
+ class="btn confirm"
+ @icon="copy"
+ @action={{this.copyResponse}}
+ @label="discourse_ai.ai_bot.debug_ai_modal.copy_response"
/>
+ {{#if this.info.prev_log_id}}
+ <DButton
+ class="btn"
+ @icon="angles-left"
+ @action={{this.prevLog}}
+ @label="discourse_ai.ai_bot.debug_ai_modal.previous_log"
+ />
+ {{/if}}
+ {{#if this.info.next_log_id}}
+ <DButton
+ class="btn"
+ @icon="angles-right"
+ @action={{this.nextLog}}
+ @label="discourse_ai.ai_bot.debug_ai_modal.next_log"
+ />
+ {{/if}}
+ <span
+ class="ai-debug-modal__just-copied"
+ >{{this.justCopiedText}}</span>
{{/if}}
- <span class="ai-debug-modal__just-copied">{{this.justCopiedText}}</span>
</:footer>
</DModal>
</template>
diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.yml
index c4b00d603ef..a2d015290ff 100644
--- a/plugins/discourse-ai/config/locales/client.en.yml
+++ b/plugins/discourse-ai/config/locales/client.en.yml
@@ -1372,6 +1372,10 @@ en:
this_turn: "This turn:"
whole_conversation: "Conversation:"
tokens_summary: "%{request_tokens} in → %{response_tokens} out"
+ duration_label: "Duration:"
+ duration: "%{duration_seconds} secs (first token: %{first_token_seconds} secs)"
+ duration_without_first_token: "%{duration_seconds} secs (first token: —)"
+ duration_unavailable: "— (first token: —)"
cache_both: "(cache %{read} read, %{write} write)"
cache_read_only: "(cache %{read} read)"
cache_write_only: "(cache %{write} write)"
diff --git a/plugins/discourse-ai/db/migrate/20260817054044_add_time_to_first_token_to_ai_api_audit_logs.rb b/plugins/discourse-ai/db/migrate/20260817054044_add_time_to_first_token_to_ai_api_audit_logs.rb
new file mode 100644
index 00000000000..b27b2242723
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20260817054044_add_time_to_first_token_to_ai_api_audit_logs.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddTimeToFirstTokenToAiApiAuditLogs < ActiveRecord::Migration[8.0]
+ def change
+ add_column :ai_api_audit_logs, :time_to_first_token_msecs, :integer
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
index 3c20ead5955..83722d8fcbd 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
@@ -38,6 +38,12 @@ module DiscourseAi
LlmCreditAllocation.check_credits!(@llm_model, feature_name)
start_time = Time.now
+ start_time_msecs = monotonic_milliseconds
+ time_to_first_token_msecs = nil
+ record_first_token =
+ lambda do
+ time_to_first_token_msecs ||= (monotonic_milliseconds - start_time_msecs).round
+ end
return if cancel_manager&.cancelled?
@@ -131,13 +137,17 @@ module DiscourseAi
orig_blk = blk
blk =
lambda do |partial|
+ should_record_first_token = !partial.is_a?(String) || !partial.empty?
partials_raw << partial.to_s
response_data << partial if partial.is_a?(String)
if partial.is_a?(String) && structured_output.present?
structured_output << partial if !partial.empty?
partial = structured_output
end
- orig_blk.call(partial) if partial
+ if partial
+ record_first_token.call if should_record_first_token
+ orig_blk.call(partial)
+ end
end
if cancel_manager
@@ -175,15 +185,18 @@ module DiscourseAi
results = processor.process_message(resp.to_h.deep_symbolize_keys)
results.each { |partial| partials_raw << partial.to_s }
+ # Non-streaming callers receive their first output only after the response is complete.
if structured_output.present?
results.each { |data| structured_output << data if data.is_a?(String) }
structured_output.finish
+ record_first_token.call
call_status = :success
return structured_output
end
response_data = results.length == 1 ? results.first : results
response_data = "" if response_data.nil?
+ record_first_token.call
call_status = :success
response_data
end
@@ -208,6 +221,7 @@ module DiscourseAi
log.created_at = start_time
log.updated_at = Time.now
log.duration_msecs = (Time.now - start_time) * 1000
+ log.time_to_first_token_msecs = time_to_first_token_msecs
log.save!
execution_context&.token_usage_tracker&.add_from_audit_log(log)
diff --git a/plugins/discourse-ai/lib/completions/endpoints/base.rb b/plugins/discourse-ai/lib/completions/endpoints/base.rb
index badce0f8ea1..49628f5d14a 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/base.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/base.rb
@@ -259,6 +259,14 @@ module DiscourseAi
execution_context:
)
request_started_at = Time.now
+ request_started_at_msecs = monotonic_milliseconds
+ time_to_first_token_msecs = nil
+ # This is end-to-end latency for the audited request, including retries and backoff.
+ record_first_token =
+ lambda do
+ time_to_first_token_msecs ||=
+ (monotonic_milliseconds - request_started_at_msecs).round
+ end
cancelled = false
call_status = :error
retry_count_429 = 0
@@ -346,7 +354,13 @@ module DiscourseAi
xml_stripper = build_xml_stripper(dialect)
if @streaming_mode
- blk = streaming_partial_handler(orig_blk, xml_stripper, structured_output)
+ blk =
+ streaming_partial_handler(
+ orig_blk,
+ xml_stripper,
+ structured_output,
+ record_first_token,
+ )
end
if !@streaming_mode
@@ -359,6 +373,8 @@ module DiscourseAi
response_raw: response_raw,
structured_output: structured_output,
)
+ # Non-streaming callers receive their first output only after the response is complete.
+ record_first_token.call
call_status = :success
return response_data
end
@@ -461,6 +477,7 @@ module DiscourseAi
request_attempts: request_attempts.presence,
call_status: call_status,
start_time: request_started_at,
+ time_to_first_token_msecs: time_to_first_token_msecs,
feature_name: feature_name,
user: user,
execution_context: execution_context,
@@ -708,8 +725,13 @@ module DiscourseAi
DiscourseAi::Completions::XmlTagStripper.new(to_strip)
end
- def streaming_partial_handler(orig_blk, xml_stripper, structured_output)
+ def monotonic_milliseconds
+ Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+ end
+
+ def streaming_partial_handler(orig_blk, xml_stripper, structured_output, record_first_token)
lambda do |partial|
+ should_record_first_token = !partial.is_a?(String) || !partial.empty?
if partial.is_a?(String)
partial = xml_stripper << partial if xml_stripper && !partial.empty?
@@ -718,7 +740,10 @@ module DiscourseAi
partial = structured_output
end
end
- orig_blk.call(partial) if partial
+ if partial
+ record_first_token.call if should_record_first_token
+ orig_blk.call(partial)
+ end
end
end
@@ -809,6 +834,7 @@ module DiscourseAi
request_attempts:,
call_status:,
start_time:,
+ time_to_first_token_msecs:,
feature_name:,
user:,
execution_context:
@@ -822,6 +848,7 @@ module DiscourseAi
log.created_at = start_time
log.updated_at = Time.now
log.duration_msecs = (Time.now - start_time) * 1000
+ log.time_to_first_token_msecs = time_to_first_token_msecs
log.save!
execution_context&.token_usage_tracker&.add_from_audit_log(log)
diff --git a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
index 22ff8878fb6..a53697308a5 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
@@ -150,6 +150,8 @@ module DiscourseAi
end
def thinking_template_kwargs
+ return {} if thinking_config&.explicit_none?
+
override = active_custom_param("thinking_override")
if override
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
index 6ceb04bed00..000a3f996a1 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
@@ -158,6 +158,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
describe "non-streaming completion" do
it "completes a simple prompt" do
+ described_class.any_instance.stubs(:monotonic_milliseconds).returns(1_000, 1_180)
response = mock_converse_response(text: "Test response", input_tokens: 15, output_tokens: 8)
client = stub_sdk_client(response: response)
@@ -165,8 +166,11 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
result = llm.generate("hello", user: user)
expect(result).to eq("Test response")
- expect(AiApiAuditLog.last.request_tokens).to eq(15)
- expect(AiApiAuditLog.last.response_tokens).to eq(8)
+ expect(AiApiAuditLog.last).to have_attributes(
+ request_tokens: 15,
+ response_tokens: 8,
+ time_to_first_token_msecs: 180,
+ )
end
it "passes thinking config for Claude models" do
@@ -260,6 +264,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
describe "streaming completion" do
it "streams text responses" do
+ described_class.any_instance.stubs(:monotonic_milliseconds).returns(1_000, 1_090, 9_999)
partials = []
stub_sdk_client do |listeners|
@@ -276,8 +281,42 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
llm.generate("hello", user: user) { |partial| partials << partial }
expect(partials).to eq(["Hello", " ", "world"])
- expect(AiApiAuditLog.last.request_tokens).to eq(10)
- expect(AiApiAuditLog.last.response_tokens).to eq(3)
+ expect(AiApiAuditLog.last).to have_attributes(
+ request_tokens: 10,
+ response_tokens: 3,
+ time_to_first_token_msecs: 90,
+ )
+ end
+
+ it "waits for non-empty structured output before recording time to first token" do
+ described_class.any_instance.stubs(:monotonic_milliseconds).returns(1_000, 1_120)
+ stub_sdk_client do |listeners|
+ fire_message_start(listeners)
+ fire_content_block_delta(listeners, text: "")
+ fire_content_block_delta(listeners, text: '{"message":"Hello"}')
+ fire_content_block_stop(listeners)
+ fire_message_stop(listeners)
+ fire_metadata(listeners)
+ end
+
+ llm = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")
+ llm.generate(
+ "hello",
+ user: user,
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ ) { |_partial| }
+
+ expect(AiApiAuditLog.last.time_to_first_token_msecs).to eq(120)
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
index 0375eb24e8d..063782f49a4 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
@@ -179,7 +179,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Vllm do
before { enable_current_plugin }
- def capture_payload_for_completion
+ def capture_payload_for_completion(**generation_args)
captured_payload = nil
stub =
stub_request(:post, "https://test.dev/v1/chat/completions")
@@ -189,7 +189,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Vllm do
end
.to_return(status: 200, body: completion_response_body)
- llm.generate("say hello", user: Discourse.system_user)
+ llm.generate("say hello", user: Discourse.system_user, **generation_args)
expect(stub).to have_been_requested
captured_payload
@@ -395,6 +395,21 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Vllm do
expect(payload["chat_template_kwargs"]).to eq("enable_thinking" => true)
end
+ it "suppresses thinking overrides when reasoning is explicitly disabled" do
+ llm_model.update!(
+ provider_params: {
+ "reasoning_parser" => "deepseek_v4",
+ "reasoning_effort" => "high",
+ "thinking_override" => "on",
+ },
+ )
+
+ payload = capture_payload_for_completion(thinking_effort: "none")
+
+ expect(payload).not_to have_key("chat_template_kwargs")
+ expect(payload["reasoning_effort"]).to eq("none")
+ end
+
it "sends thinking for Granite thinking overrides" do
llm_model.update!(
provider_params: {
diff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
index e25aadbdd2b..0f10acd496f 100644
--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
@@ -162,7 +162,36 @@ RSpec.describe DiscourseAi::Completions::Llm do
expect { llm.generate("Hello", user:) }.to raise_error(
DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
)
- expect(AiApiAuditLog.last).to have_attributes(response_status: 401, response_tokens: 0)
+ expect(AiApiAuditLog.last).to have_attributes(
+ response_status: 401,
+ response_tokens: 0,
+ time_to_first_token_msecs: nil,
+ )
+ end
+
+ it "records time to the complete response for non-streaming requests" do
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .stubs(:monotonic_milliseconds)
+ .returns(1_000, 1_125)
+ stub_response(body: success_body(content: "Hello"))
+
+ expect(llm.generate("Hello", user:)).to eq("Hello")
+ expect(AiApiAuditLog.last.time_to_first_token_msecs).to eq(125)
+ end
+
+ it "records time to the first emitted partial for streaming requests" do
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .stubs(:monotonic_milliseconds)
+ .returns(1_000, 1_075)
+ stub_response(body: streaming_body(content: "Hello"))
+
+ response = +""
+ llm.generate("Hello", user:) { |partial| response << partial }
+
+ expect(response).to eq("Hello")
+ expect(AiApiAuditLog.last.time_to_first_token_msecs).to eq(75)
end
it "creates usage stats" do
@@ -351,6 +380,10 @@ RSpec.describe DiscourseAi::Completions::Llm do
end
it "retries streaming responses after rate limits" do
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .stubs(:monotonic_milliseconds)
+ .returns(1_000, 3_500)
request =
WebMock.stub_request(:post, model.url).to_return(
{ status: 429, body: "rate limited" },
@@ -365,6 +398,7 @@ RSpec.describe DiscourseAi::Completions::Llm do
expect(AiApiAuditLog.last.request_attempts).to eq(
[{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],
)
+ expect(AiApiAuditLog.last.time_to_first_token_msecs).to eq(2500)
end
it "does not retry streaming responses after output has started" do
diff --git a/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb b/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb
index 48e13aa3f25..f4ba98a57ab 100644
--- a/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb
+++ b/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb
@@ -20,6 +20,8 @@ RSpec.describe AiApiAuditLogSerializer do
AiApiAuditLog.new(
provider_id: AiApiAuditLog::Provider::OpenAI,
response_status: 200,
+ duration_msecs: 22_300,
+ time_to_first_token_msecs: 125,
request_attempts: [
{ "status" => 429, "delay_ms" => 0 },
{ "status" => 200, "delay_ms" => 2000 },
@@ -32,5 +34,15 @@ RSpec.describe AiApiAuditLogSerializer do
[{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],
)
expect(serialized[:response_status]).to eq(200)
+ expect(serialized[:duration_msecs]).to eq(22_300)
+ expect(serialized[:time_to_first_token_msecs]).to eq(125)
+ end
+
+ it "includes a null time to first token for historical logs" do
+ log = AiApiAuditLog.new(provider_id: AiApiAuditLog::Provider::OpenAI)
+
+ serialized = described_class.new(log).as_json[:ai_api_audit_log]
+
+ expect(serialized[:time_to_first_token_msecs]).to be_nil
end
end
diff --git a/plugins/discourse-ai/test/javascripts/acceptance/debug-ai-modal-test.gjs b/plugins/discourse-ai/test/javascripts/acceptance/debug-ai-modal-test.gjs
new file mode 100644
index 00000000000..d458e0edcae
--- /dev/null
+++ b/plugins/discourse-ai/test/javascripts/acceptance/debug-ai-modal-test.gjs
@@ -0,0 +1,60 @@
+import { getOwner } from "@ember/owner";
+import { settled, visit } from "@ember/test-helpers";
+import { test } from "qunit";
+import { acceptance } from "discourse/tests/helpers/qunit-helpers";
+import DebugAiModal from "discourse/plugins/discourse-ai/discourse/components/modal/debug-ai-modal";
+
+const MODEL = { id: 42 };
+
+function auditLog(timeToFirstToken, duration = 22_300) {
+ return {
+ id: 1,
+ request_tokens: 10,
+ response_tokens: 5,
+ raw_request_payload: "{}",
+ raw_response_payload: "response",
+ duration_msecs: duration,
+ time_to_first_token_msecs: timeToFirstToken,
+ };
+}
+
+acceptance("AI debug modal", function (needs) {
+ let timeToFirstToken;
+
+ needs.user();
+ needs.pretender((server, helper) => {
+ server.get("/discourse-ai/ai-bot/post/42/show-debug-info.json", () =>
+ helper.response(auditLog(timeToFirstToken))
+ );
+ });
+
+ test("it displays total duration and time to first token", async function (assert) {
+ timeToFirstToken = 10_300;
+ await visit("/");
+
+ getOwner(this).lookup("service:modal").show(DebugAiModal, { model: MODEL });
+ await settled();
+
+ assert
+ .dom(".ai-debug-modal__duration")
+ .hasText(
+ "Duration: 22.3 secs (first token: 10.3 secs)",
+ "it presents the two latency measurements together"
+ );
+ });
+
+ test("it identifies unavailable historical timing", async function (assert) {
+ timeToFirstToken = null;
+ await visit("/");
+
+ getOwner(this).lookup("service:modal").show(DebugAiModal, { model: MODEL });
+ await settled();
+
+ assert
+ .dom(".ai-debug-modal__duration")
+ .hasText(
+ "Duration: 22.3 secs (first token: —)",
+ "it preserves total duration for historical logs"
+ );
+ });
+});