Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Track incomplete AI completions (#44506)

Sam79be4ca49252
diff --git a/db/structure.sql b/db/structure.sql
index 50547ee317e..c3367d63c77 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -540,7 +540,10 @@ CREATE TABLE public.ai_api_audit_logs (
response_status integer,
request_attempts jsonb,
estimated_cost numeric(20,10),
- time_to_first_token_msecs integer
+ time_to_first_token_msecs integer,
+ completion_state smallint,
+ reasoning_tokens integer,
+ stop_reason character varying(50)
);
@@ -19437,6 +19440,13 @@ CREATE INDEX idx_ai_api_audit_logs_failed_id ON public.ai_api_audit_logs USING b
CREATE INDEX idx_ai_api_audit_logs_feature_name_id ON public.ai_api_audit_logs USING btree (feature_name, id) WHERE (feature_name IS NOT NULL);
+--
+-- Name: idx_ai_api_audit_logs_incomplete_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_ai_api_audit_logs_incomplete_id ON public.ai_api_audit_logs USING btree (completion_state, id) WHERE (completion_state = ANY (ARRAY[2, 3]));
+
+
--
-- Name: idx_ai_api_audit_logs_payload_id; Type: INDEX; Schema: public; Owner: -
--
@@ -25832,6 +25842,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20261013010026'),
+('20261013010022'),
('20261012021959'),
('20261012015332'),
('20261009144947'),
diff --git a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-logs.gjs b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-logs.gjs
index 0df3cd9968d..2c522b9e38e 100644
--- a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-logs.gjs
+++ b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-logs.gjs
@@ -129,6 +129,8 @@ export default class AiLogs extends Component {
label: i18n("discourse_ai.logs.successful"),
},
{ value: "failed", label: i18n("discourse_ai.logs.failed") },
+ { value: "empty", label: i18n("discourse_ai.logs.empty") },
+ { value: "truncated", label: i18n("discourse_ai.logs.truncated") },
];
}
diff --git a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/modal/ai-log-detail-modal.gjs b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/modal/ai-log-detail-modal.gjs
index ea8d905a3fd..e914d80e3bb 100644
--- a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/modal/ai-log-detail-modal.gjs
+++ b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/modal/ai-log-detail-modal.gjs
@@ -129,13 +129,7 @@ export default class AiLogDetailModal extends Component {
}
get formattedOutcome() {
- const successful =
- (this.log?.response_status >= 200 && this.log?.response_status <= 299) ||
- (this.log?.response_status == null &&
- Number(this.log?.response_tokens || 0) > 0);
- const outcome = i18n(
- `discourse_ai.logs.${successful ? "successful" : "failed"}`
- );
+ const outcome = i18n(`discourse_ai.logs.${this.log.outcome}`);
return this.log?.response_status == null
? outcome
@@ -145,6 +139,10 @@ export default class AiLogDetailModal extends Component {
});
}
+ get hasReasoningTokens() {
+ return this.log?.reasoning_tokens != null;
+ }
+
get formattedCreatedAt() {
return this.log?.created_at
? moment(this.log.created_at).format("LLL")
@@ -269,6 +267,16 @@ export default class AiLogDetailModal extends Component {
>{{this.formattedCreatedAt}}</dd></div>
<div><dt>{{i18n "discourse_ai.logs.outcome"}}</dt><dd
>{{this.formattedOutcome}}</dd></div>
+ {{#if this.log.stop_reason}}
+ <div class="ai-log-detail-modal__metadata-stop-reason"><dt>{{i18n
+ "discourse_ai.logs.stop_reason"
+ }}</dt><dd>{{this.log.stop_reason}}</dd></div>
+ {{/if}}
+ {{#if this.hasReasoningTokens}}
+ <div class="ai-log-detail-modal__metadata-reasoning"><dt>{{i18n
+ "discourse_ai.logs.reasoning_tokens"
+ }}</dt><dd>{{this.log.reasoning_tokens}}</dd></div>
+ {{/if}}
<div><dt>{{i18n "discourse_ai.logs.feature"}}</dt><dd>{{or
this.log.feature_name
"—"
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_logs_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_logs_controller.rb
index f67d9c5eb10..b54bf1b39ac 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_logs_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_logs_controller.rb
@@ -18,6 +18,9 @@ module DiscourseAi
user_id
request_tokens
response_tokens
+ completion_state
+ reasoning_tokens
+ stop_reason
created_at
topic_id
post_id
@@ -45,6 +48,7 @@ module DiscourseAi
post_id
request_tokens
response_tokens
+ completion_state
response_status
duration_msecs
].freeze
@@ -58,13 +62,7 @@ module DiscourseAi
end
logs =
logs
- .select(
- *LIST_COLUMNS,
- Arel.sql("request_attempts IS NOT NULL AS has_retries"),
- Arel.sql(
- "CASE WHEN #{AiApiAuditLog::SUCCESS_CONDITION} THEN 'successful' ELSE 'failed' END AS outcome",
- ),
- )
+ .select(*LIST_COLUMNS, Arel.sql("request_attempts IS NOT NULL AS has_retries"))
.order(id: :desc)
.limit(limit + 1)
.preload(:user, :llm_model)
@@ -330,6 +328,10 @@ module DiscourseAi
scope.successful_requests
when "failed"
scope.failed_requests
+ when "empty"
+ scope.empty_responses
+ when "truncated"
+ scope.truncated_responses
else
raise Discourse::InvalidParameters.new(:outcome)
end
diff --git a/plugins/discourse-ai/app/models/ai_api_audit_log.rb b/plugins/discourse-ai/app/models/ai_api_audit_log.rb
index b1278d40fcd..3a48d147bb3 100644
--- a/plugins/discourse-ai/app/models/ai_api_audit_log.rb
+++ b/plugins/discourse-ai/app/models/ai_api_audit_log.rb
@@ -20,6 +20,16 @@ class AiApiAuditLog < ActiveRecord::Base
scope :failed_requests, -> { where(FAILURE_CONDITION) }
scope :successful_requests, -> { where(SUCCESS_CONDITION) }
+ module CompletionState
+ COMPLETE = 1
+ TRUNCATED = 2
+ EMPTY = 3
+ end
+
+ scope :empty_responses, -> { successful_requests.where(completion_state: CompletionState::EMPTY) }
+ scope :truncated_responses,
+ -> { successful_requests.where(completion_state: CompletionState::TRUNCATED) }
+
module Provider
OpenAI = 1
Anthropic = 2
@@ -76,6 +86,7 @@ end
# id :bigint not null, primary key
# cache_read_tokens :integer
# cache_write_tokens :integer
+# completion_state :integer
# duration_msecs :integer
# estimated_cost :decimal(20, 10)
# feature_context :jsonb
@@ -83,10 +94,12 @@ end
# language_model :string(255)
# raw_request_payload :string
# raw_response_payload :string
+# reasoning_tokens :integer
# request_attempts :jsonb
# request_tokens :integer
# response_status :integer
# response_tokens :integer
+# stop_reason :string(50)
# time_to_first_token_msecs :integer
# created_at :datetime not null
# updated_at :datetime not null
@@ -100,6 +113,7 @@ end
#
# idx_ai_api_audit_logs_failed_id (id) WHERE (((response_status IS NOT NULL) AND ((response_status < 200) OR (response_status > 299))) OR ((response_status IS NULL) AND (COALESCE(response_tokens, 0) <= 0)))
# idx_ai_api_audit_logs_feature_name_id (feature_name,id) WHERE (feature_name IS NOT NULL)
+# idx_ai_api_audit_logs_incomplete_id (completion_state,id) WHERE (completion_state = ANY (ARRAY[2, 3]))
# idx_ai_api_audit_logs_payload_id (id) WHERE ((raw_request_payload IS NOT NULL) OR (raw_response_payload IS NOT NULL))
# idx_ai_api_audit_logs_retried_id (id) WHERE (request_attempts IS NOT NULL)
# idx_ai_api_audit_logs_user_id_id (user_id,id)
diff --git a/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_detail_serializer.rb b/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_detail_serializer.rb
index e8b911abcde..ce4723932c4 100644
--- a/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_detail_serializer.rb
+++ b/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_detail_serializer.rb
@@ -19,9 +19,12 @@ module DiscourseAi
:post_id,
:request_tokens,
:response_tokens,
+ :reasoning_tokens,
:cache_read_tokens,
:cache_write_tokens,
:response_status,
+ :outcome,
+ :stop_reason,
:duration_msecs,
:time_to_first_token_msecs,
:spending,
@@ -42,6 +45,10 @@ module DiscourseAi
)
end
+ def outcome
+ AiLogSerializerHelpers.outcome(object)
+ end
+
def provider_name
AiLogSerializerHelpers.provider_name(object.provider_id)
end
diff --git a/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_list_serializer.rb b/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_list_serializer.rb
index f84b509a203..687c706757a 100644
--- a/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_list_serializer.rb
+++ b/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_list_serializer.rb
@@ -23,6 +23,10 @@ module DiscourseAi
:has_retries,
:outcome
+ def outcome
+ AiLogSerializerHelpers.outcome(object)
+ end
+
def provider_name
AiLogSerializerHelpers.provider_name(object.provider_id)
end
diff --git a/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_serializer_helpers.rb b/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_serializer_helpers.rb
index 1301ecf45da..605d555b575 100644
--- a/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_serializer_helpers.rb
+++ b/plugins/discourse-ai/app/serializers/discourse_ai/admin/ai_log_serializer_helpers.rb
@@ -17,6 +17,25 @@ module DiscourseAi
AiApiAuditLog::Provider::BedrockConverse => "Amazon Bedrock",
}.freeze
+ def self.outcome(log)
+ successful =
+ if log.response_status.nil?
+ log.response_tokens.to_i > 0
+ else
+ log.response_status.between?(200, 299)
+ end
+ return "failed" if !successful
+
+ case log.completion_state
+ when AiApiAuditLog::CompletionState::EMPTY
+ "empty"
+ when AiApiAuditLog::CompletionState::TRUNCATED
+ "truncated"
+ else
+ "successful"
+ end
+ end
+
def self.provider_name(provider_id)
PROVIDER_NAMES[provider_id]
end
diff --git a/plugins/discourse-ai/assets/javascripts/discourse/components/ai-log-row.gjs b/plugins/discourse-ai/assets/javascripts/discourse/components/ai-log-row.gjs
index 5161f8516a2..c2db771791c 100644
--- a/plugins/discourse-ai/assets/javascripts/discourse/components/ai-log-row.gjs
+++ b/plugins/discourse-ai/assets/javascripts/discourse/components/ai-log-row.gjs
@@ -10,6 +10,25 @@ import dConcatClass from "discourse/ui-kit/helpers/d-concat-class";
import dIcon from "discourse/ui-kit/helpers/d-icon";
import I18n, { i18n } from "discourse-i18n";
+const STATUS_PRESENTATION = {
+ successful: {
+ icon: "circle-check",
+ className: "--success",
+ labelKey: "successful",
+ },
+ failed: { icon: "circle-xmark", className: "--failed", labelKey: "failed" },
+ empty: {
+ icon: "triangle-exclamation",
+ className: "--warning",
+ labelKey: "empty",
+ },
+ truncated: {
+ icon: "triangle-exclamation",
+ className: "--warning",
+ labelKey: "truncated_description",
+ },
+};
+
function compactTokenCount(value) {
if (value < 1_000) {
return I18n.toNumber(value, { precision: 0 });
@@ -30,7 +49,7 @@ function compactTokenCount(value) {
export default class AiLogRow extends Component {
get status() {
const log = this.args.log;
- if (["successful", "failed"].includes(log.outcome)) {
+ if (Object.hasOwn(STATUS_PRESENTATION, log.outcome)) {
return log.outcome;
}
@@ -41,15 +60,17 @@ export default class AiLogRow extends Component {
}
get statusLabel() {
- return i18n(`discourse_ai.logs.${this.status}`);
+ return i18n(
+ `discourse_ai.logs.${STATUS_PRESENTATION[this.status].labelKey}`
+ );
}
get statusClass() {
- return this.status === "successful" ? "--success" : "--failed";
+ return STATUS_PRESENTATION[this.status].className;
}
get statusIcon() {
- return this.status === "successful" ? "circle-check" : "circle-xmark";
+ return STATUS_PRESENTATION[this.status].icon;
}
get featureName() {
diff --git a/plugins/discourse-ai/assets/stylesheets/admin/ai-logs.scss b/plugins/discourse-ai/assets/stylesheets/admin/ai-logs.scss
index 7ae6084b53c..62ff244eb55 100644
--- a/plugins/discourse-ai/assets/stylesheets/admin/ai-logs.scss
+++ b/plugins/discourse-ai/assets/stylesheets/admin/ai-logs.scss
@@ -161,6 +161,10 @@
&.--failed {
color: var(--danger);
}
+
+ &.--warning {
+ color: color-mix(in srgb, var(--highlight-high) 80%, var(--primary));
+ }
}
&__flag {
diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.yml
index 8a18f13315a..7db850e308b 100644
--- a/plugins/discourse-ai/config/locales/client.en.yml
+++ b/plugins/discourse-ai/config/locales/client.en.yml
@@ -773,6 +773,11 @@ en:
tokens: "Tokens"
tokens_pair: "%{request} → %{response}"
tokens_aria: "%{request} input tokens and %{response} output tokens"
+ reasoning_tokens: "Reasoning tokens"
+ stop_reason: "Stop reason"
+ empty: "Empty response"
+ truncated: "Truncated"
+ truncated_description: "Truncated at output limit"
tokens_unavailable: "Token counts unavailable"
cache_read_tokens: "Cache read"
cache_write_tokens: "Cache write"
diff --git a/plugins/discourse-ai/db/migrate/20261013010022_add_completion_state_to_ai_api_audit_logs.rb b/plugins/discourse-ai/db/migrate/20261013010022_add_completion_state_to_ai_api_audit_logs.rb
new file mode 100644
index 00000000000..bfc156fd2e3
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20261013010022_add_completion_state_to_ai_api_audit_logs.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddCompletionStateToAiApiAuditLogs < ActiveRecord::Migration[8.0]
+ def change
+ add_column :ai_api_audit_logs, :completion_state, :integer, limit: 2
+ add_column :ai_api_audit_logs, :reasoning_tokens, :integer
+ add_column :ai_api_audit_logs, :stop_reason, :string, limit: 50
+ end
+end
diff --git a/plugins/discourse-ai/db/migrate/20261013010026_add_incomplete_index_to_ai_api_audit_logs.rb b/plugins/discourse-ai/db/migrate/20261013010026_add_incomplete_index_to_ai_api_audit_logs.rb
new file mode 100644
index 00000000000..85473f00847
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20261013010026_add_incomplete_index_to_ai_api_audit_logs.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddIncompleteIndexToAiApiAuditLogs < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ INDEX_NAME = "idx_ai_api_audit_logs_incomplete_id"
+
+ def up
+ remove_index :ai_api_audit_logs, name: INDEX_NAME, algorithm: :concurrently, if_exists: true
+ add_index :ai_api_audit_logs,
+ %i[completion_state id],
+ name: INDEX_NAME,
+ where: "completion_state IN (2, 3)",
+ algorithm: :concurrently
+ end
+
+ def down
+ remove_index :ai_api_audit_logs, name: INDEX_NAME, algorithm: :concurrently, if_exists: true
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/anthropic_message_processor.rb b/plugins/discourse-ai/lib/completions/anthropic_message_processor.rb
index 7b481495b8d..6fce23b91ed 100644
--- a/plugins/discourse-ai/lib/completions/anthropic_message_processor.rb
+++ b/plugins/discourse-ai/lib/completions/anthropic_message_processor.rb
@@ -50,7 +50,8 @@ class DiscourseAi::Completions::AnthropicMessageProcessor
:output_tokens,
:cache_creation_input_tokens,
:cache_read_input_tokens,
- :output_thinking
+ :output_thinking,
+ :stop_reason
def initialize(streaming_mode:, partial_tool_calls: false, output_thinking: false)
@streaming_mode = streaming_mode
@@ -65,6 +66,14 @@ class DiscourseAi::Completions::AnthropicMessageProcessor
@thinking = nil
end
+ def output_limit_reached
+ @stop_reason == "max_tokens"
+ end
+
+ def reasoning_tokens
+ nil
+ end
+
def to_tool_calls
@tool_calls.map { |tool_call| tool_call.to_tool_call }
end
@@ -204,6 +213,7 @@ class DiscourseAi::Completions::AnthropicMessageProcessor
@cache_creation_input_tokens = usage[:cache_creation_input_tokens]
@cache_read_input_tokens = usage[:cache_read_input_tokens]
elsif parsed[:type] == "message_delta"
+ @stop_reason = parsed.dig(:delta, :stop_reason) || @stop_reason
@output_tokens =
parsed.dig(:usage, :output_tokens) || parsed.dig(:delta, :usage, :output_tokens)
elsif parsed[:type] == "message_stop"
@@ -222,6 +232,7 @@ class DiscourseAi::Completions::AnthropicMessageProcessor
parsed = payload
parsed = JSON.parse(payload, symbolize_names: true) if payload.is_a?(String)
+ @stop_reason = parsed[:stop_reason]
content = parsed.dig(:content)
@anthropic_content_blocks = content.deep_dup if content.is_a?(Array)
if content.is_a?(Array)
diff --git a/plugins/discourse-ai/lib/completions/context_preparation.rb b/plugins/discourse-ai/lib/completions/context_preparation.rb
index 31478f83284..9cf8e90d4e3 100644
--- a/plugins/discourse-ai/lib/completions/context_preparation.rb
+++ b/plugins/discourse-ai/lib/completions/context_preparation.rb
@@ -4,7 +4,6 @@ module DiscourseAi
module Completions
class ContextPreparation
MAX_CALLS = 8
- MAX_OUTPUT_TOKENS = 2048
MAX_SECONDS = 90
MAX_ALLOWANCE_TOKENS = 1_000_000
RETAINED_ATTACHMENTS_ACK = "The preceding attachments are retained historical evidence."
@@ -87,7 +86,6 @@ module DiscourseAi
def initialize(llm, threshold: 80, calls: 0, spent_tokens: 0, elapsed_seconds: 0)
@llm = llm
@threshold = threshold || 80
- @summary_output_tokens = [MAX_OUTPUT_TOKENS, [llm.max_prompt_tokens.to_i / 8, 1].max].min
allowance = [[llm.max_prompt_tokens.to_i * 2, 64_000].max, MAX_ALLOWANCE_TOKENS].min
@budget =
Budget.new(
@@ -358,8 +356,7 @@ module DiscourseAi
high = remaining.length
while low < high
midpoint = (low + high + 1) / 2
- size, capacity =
- @llm.prompt_capacity(build_prompt.call(midpoint), max_tokens: @summary_output_tokens)
+ size, capacity = @llm.prompt_capacity(build_prompt.call(midpoint))
if size <= capacity
low = midpoint
else
@@ -376,8 +373,7 @@ module DiscourseAi
compression_prompt = build_prompt.call(low)
remaining.slice!(0, low)
offset += low
- input_size, input_capacity, output_reserve =
- @llm.prompt_capacity(compression_prompt, max_tokens: @summary_output_tokens)
+ input_size, input_capacity, output_reserve = @llm.prompt_capacity(compression_prompt)
raise Error.new("summary_input_overflow") if input_size > input_capacity
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@budget.record_elapsed(now - started_at)
@@ -390,11 +386,11 @@ module DiscourseAi
!options[:subagent_execution_state].reserve_completion
raise Error.new("completion_limit")
end
+ # Uncapped: thinking tokens count against max_tokens and could leave no summary.
response =
@llm.generate(
compression_prompt,
user: options[:user],
- max_tokens: @summary_output_tokens,
feature_name: "context_compression",
feature_context: options[:feature_context],
execution_context: options[:execution_context],
diff --git a/plugins/discourse-ai/lib/completions/converse_message_processor.rb b/plugins/discourse-ai/lib/completions/converse_message_processor.rb
index 347a15c3630..cf437d50d87 100644
--- a/plugins/discourse-ai/lib/completions/converse_message_processor.rb
+++ b/plugins/discourse-ai/lib/completions/converse_message_processor.rb
@@ -49,7 +49,8 @@ class DiscourseAi::Completions::ConverseMessageProcessor
:output_tokens,
:cache_read_input_tokens,
:cache_write_input_tokens,
- :output_thinking
+ :output_thinking,
+ :stop_reason
def initialize(streaming_mode:, partial_tool_calls: false, output_thinking: false)
@streaming_mode = streaming_mode
@@ -60,6 +61,14 @@ class DiscourseAi::Completions::ConverseMessageProcessor
@thinking = nil
end
+ def output_limit_reached
+ @stop_reason == "max_tokens"
+ end
+
+ def reasoning_tokens
+ nil
+ end
+
def to_tool_calls
@tool_calls.map { |tool_call| tool_call.to_tool_call }
end
@@ -144,7 +153,7 @@ class DiscourseAi::Completions::ConverseMessageProcessor
@current_tool_call = nil
end
when :message_stop
- # nothing to do
+ @stop_reason = parsed[:stop_reason]
when :metadata
usage = parsed[:usage]
if usage
@@ -166,6 +175,7 @@ class DiscourseAi::Completions::ConverseMessageProcessor
parsed = JSON.parse(payload, symbolize_names: true) if payload.is_a?(String)
result = []
+ @stop_reason = parsed[:stop_reason]
content = parsed.dig(:output, :message, :content)
if content.is_a?(Array)
diff --git a/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb b/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb
index 1d8e2614f18..c7532ebf58d 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb
@@ -97,6 +97,20 @@ module DiscourseAi
end
end
+ protected
+
+ def output_stop_reason
+ processor.stop_reason
+ end
+
+ def output_limit_reached?
+ processor.output_limit_reached
+ end
+
+ def reported_reasoning_tokens
+ processor.reasoning_tokens
+ end
+
private
def supports_anthropic_thinking?
diff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb
index 9f602de490d..1cfc04ca757 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb
@@ -216,6 +216,23 @@ module DiscourseAi
end
end
+ def reset_attempt_state!
+ super
+ @raw_response = nil
+ end
+
+ def output_stop_reason
+ super if dialect.is_a?(DiscourseAi::Completions::Dialects::Claude)
+ end
+
+ def output_limit_reached?
+ dialect.is_a?(DiscourseAi::Completions::Dialects::Claude) && super
+ end
+
+ def reported_reasoning_tokens
+ super if dialect.is_a?(DiscourseAi::Completions::Dialects::Claude)
+ end
+
def processor
if dialect.is_a?(DiscourseAi::Completions::Dialects::Claude)
claude_processor
diff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
index 46007ebe099..aa61405d32d 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
@@ -91,6 +91,7 @@ module DiscourseAi
end
end
+ reset_attempt_state!
call_status = :error
cancelled = false
cancel_manager_callback = nil
@@ -124,11 +125,12 @@ module DiscourseAi
return if cancel_manager&.cancelled?
processor =
- DiscourseAi::Completions::ConverseMessageProcessor.new(
- streaming_mode: @streaming_mode,
- partial_tool_calls: partial_tool_calls,
- output_thinking: output_thinking,
- )
+ @processor =
+ DiscourseAi::Completions::ConverseMessageProcessor.new(
+ streaming_mode: @streaming_mode,
+ partial_tool_calls: partial_tool_calls,
+ output_thinking: output_thinking,
+ )
client = build_sdk_client
@@ -146,6 +148,7 @@ module DiscourseAi
end
if partial
record_first_token.call if should_record_first_token
+ @visible_output ||= visible_partial?(partial)
orig_blk.call(partial)
end
end
@@ -168,6 +171,7 @@ module DiscourseAi
unless cancelled
if structured_output
structured_output.finish
+ @visible_output ||= structured_output.content?
if structured_output.broken?
blk.call("")
else
@@ -191,9 +195,11 @@ module DiscourseAi
structured_output.finish
record_first_token.call
call_status = :success
+ @visible_output = visible_partial?(structured_output)
return structured_output
end
+ @visible_output = visible_partial?(results)
response_data = results.length == 1 ? results.first : results
response_data = "" if response_data.nil?
record_first_token.call
@@ -215,6 +221,7 @@ module DiscourseAi
processor.cache_read_input_tokens if processor.cache_read_input_tokens
log.cache_write_tokens =
processor.cache_write_input_tokens if processor.cache_write_input_tokens
+ update_completion_state(log, call_status, cancelled: cancelled)
execution_context&.settle_generation(
tokens: log.response_tokens,
tokenizer: tokenizer,
@@ -443,6 +450,18 @@ module DiscourseAi
end
end
+ def output_stop_reason
+ @processor.stop_reason
+ end
+
+ def output_limit_reached?
+ @processor.output_limit_reached
+ end
+
+ def reported_reasoning_tokens
+ @processor.reasoning_tokens
+ end
+
def build_stream_handler(processor, blk, &cancel_check)
handler = Aws::BedrockRuntime::EventStreams::ConverseStreamOutput.new
@@ -471,7 +490,10 @@ module DiscourseAi
handler.on_message_start_event { |_event| cancel_check.call }
- handler.on_message_stop_event { |_event| cancel_check.call }
+ handler.on_message_stop_event do |event|
+ cancel_check.call
+ processor.process_streamed_message(type: :message_stop, stop_reason: event.stop_reason)
+ end
handler.on_metadata_event do |event|
usage = event.usage
diff --git a/plugins/discourse-ai/lib/completions/endpoints/base.rb b/plugins/discourse-ai/lib/completions/endpoints/base.rb
index 90a5a29bcc4..dff333a29f1 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/base.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/base.rb
@@ -265,6 +265,58 @@ module DiscourseAi
protected
+ def output_stop_reason
+ nil
+ end
+
+ def output_limit_reached?
+ false
+ end
+
+ def reported_reasoning_tokens
+ nil
+ end
+
+ def reset_attempt_state!
+ @processor = nil
+ @decoder = nil
+ @json_decoder = nil
+ @visible_output = false
+ end
+
+ def visible_partial?(partial)
+ case partial
+ when Thinking
+ false
+ when String
+ partial.present?
+ when ToolCall
+ true
+ when StructuredOutput
+ partial.content?
+ when Array
+ partial.any? { |item| visible_partial?(item) }
+ else
+ false
+ end
+ end
+
+ def update_completion_state(log, call_status, cancelled: false)
+ log.stop_reason = output_stop_reason&.to_s&.slice(0, 50).presence
+ log.reasoning_tokens = reported_reasoning_tokens
+ # Cancellation is a user action, not an incomplete completion.
+ return if cancelled || call_status != :success
+
+ log.completion_state =
+ if !@visible_output
+ AiApiAuditLog::CompletionState::EMPTY
+ elsif output_limit_reached?
+ AiApiAuditLog::CompletionState::TRUNCATED
+ else
+ AiApiAuditLog::CompletionState::COMPLETE
+ end
+ end
+
def tokenizer
llm_model.tokenizer_class
end
@@ -448,6 +500,7 @@ module DiscourseAi
)
loop do
+ reset_attempt_state!
call_status = :error
response_data = +""
response_raw = +""
@@ -508,7 +561,7 @@ module DiscourseAi
# Some providers rely on prefill to return structured outputs, so the start
# of the JSON won't be included in the response. Supply it to keep JSON valid.
- structured_output << +"{" if structured_output && @forced_json_through_prefill
+ structured_output.prefill if structured_output && @forced_json_through_prefill
xml_tool_processor = build_xml_tool_processor(dialect, partial_tool_calls)
xml_stripper = build_xml_stripper(dialect)
@@ -648,6 +701,7 @@ module DiscourseAi
partials_raw: partials_raw,
request_attempts: request_attempts.presence,
call_status: call_status,
+ cancelled: cancelled,
start_time: request_started_at,
time_to_first_token_msecs: time_to_first_token_msecs,
feature_name: feature_name,
@@ -759,13 +813,14 @@ module DiscourseAi
if partial.is_a?(String)
partial = xml_stripper << partial if xml_stripper && !partial.empty?
- if structured_output.present?
+ if partial && structured_output.present?
structured_output << partial if !partial.empty?
partial = structured_output
end
end
if partial
record_first_token.call if should_record_first_token
+ @visible_output ||= visible_partial?(partial)
orig_blk.call(partial)
end
end
@@ -825,12 +880,12 @@ module DiscourseAi
structured_output:
)
if xml_stripper
- stripped = xml_stripper.finish
- if stripped.present?
- response_data << stripped
- result = []
- result = (xml_tool_processor << stripped) if xml_tool_processor
- result.each { |partial| blk.call(partial) }
+ # Preserve legacy return data without exposing buffered reasoning through the block.
+ unstripped = xml_stripper.finish
+ response_data << unstripped if unstripped.present?
+ stripped = xml_stripper.finish_visible
+ if stripped.present? && xml_tool_processor
+ (xml_tool_processor << stripped).each { |partial| blk.call(partial) }
end
end
@@ -842,6 +897,7 @@ module DiscourseAi
def finish_structured_output_streaming(structured_output, blk)
structured_output.finish
+ @visible_output ||= structured_output.content?
if structured_output.broken?
# signal last partial output which will get parsed
# by best effort json parser
@@ -858,6 +914,7 @@ module DiscourseAi
partials_raw:,
request_attempts:,
call_status:,
+ cancelled:,
start_time:,
time_to_first_token_msecs:,
feature_name:,
@@ -867,6 +924,7 @@ module DiscourseAi
log.raw_response_payload = response_raw
log.request_attempts = request_attempts if log.has_attribute?(:request_attempts)
final_log_update(log)
+ update_completion_state(log, call_status, cancelled: cancelled)
execution_context&.settle_generation(
tokens: log.response_tokens,
tokenizer: tokenizer,
@@ -1132,12 +1190,22 @@ module DiscourseAi
end
end
+ # Legacy return values include stripped-only text; audit visibility must not.
+ visible_response_data = response_data
if xml_stripper
+ visible_response_data = []
response_data.map! do |partial|
- stripped = (xml_stripper << partial) if partial.is_a?(String)
- stripped.presence || partial
+ if partial.is_a?(String)
+ stripped = xml_stripper << partial
+ visible_response_data << stripped
+ stripped.presence || partial
+ else
+ visible_response_data << partial
+ partial
+ end
end
response_data << xml_stripper.finish
+ visible_response_data << xml_stripper.finish_visible
end
response_data.reject!(&:blank?)
@@ -1146,9 +1214,21 @@ module DiscourseAi
response_data.each { |data| structured_output << data if data.is_a?(String) }
structured_output.finish
+ visible_structured_output = structured_output
+ if xml_stripper
+ visible_structured_output = StructuredOutput.new({})
+ visible_structured_output.prefill if @forced_json_through_prefill
+ visible_response_data.each do |partial|
+ visible_structured_output << partial if partial.is_a?(String)
+ end
+ visible_structured_output.finish
+ end
+ @visible_output = visible_structured_output.content?
return structured_output
end
+ @visible_output = visible_partial?(visible_response_data)
+
# this is to keep stuff backwards compatible
response_data = response_data.first if response_data.length == 1
response_data = "" if response_data.nil?
diff --git a/plugins/discourse-ai/lib/completions/endpoints/cohere.rb b/plugins/discourse-ai/lib/completions/endpoints/cohere.rb
index 8f571e09cdf..ff081d25c2c 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/cohere.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/cohere.rb
@@ -22,6 +22,16 @@ module DiscourseAi
AiApiAuditLog::Provider::Cohere
end
+ protected
+
+ def reset_attempt_state!
+ super
+ @tool_idx = nil
+ @has_tool = nil
+ @input_tokens = nil
+ @output_tokens = nil
+ end
+
private
def model_uri
diff --git a/plugins/discourse-ai/lib/completions/endpoints/gemini.rb b/plugins/discourse-ai/lib/completions/endpoints/gemini.rb
index e03e41cf28f..32fdc9c3034 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/gemini.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/gemini.rb
@@ -414,6 +414,8 @@ module DiscourseAi
update_usage(json)
candidate = json.dig(:candidates, 0)
+ @stop_reason =
+ candidate&.dig(:finishReason) || json.dig(:promptFeedback, :blockReason) || @stop_reason
parts = candidate&.dig(:content, :parts)
batch_token = current_batch_token_for(parts)
@@ -428,6 +430,9 @@ module DiscourseAi
check_for_failure!(parsed)
update_usage(parsed)
candidate = parsed.dig(:candidates, 0)
+ @stop_reason =
+ candidate&.dig(:finishReason) || parsed.dig(:promptFeedback, :blockReason) ||
+ @stop_reason
parts = candidate&.dig(:content, :parts)
batch_token = current_batch_token_for(parts)
decode_parts(parts, batch_token:, streaming: true) +
@@ -603,6 +608,33 @@ module DiscourseAi
end
end
+ def output_stop_reason
+ @stop_reason
+ end
+
+ def output_limit_reached?
+ @stop_reason == "MAX_TOKENS"
+ end
+
+ def reported_reasoning_tokens
+ @thought_token_count
+ end
+
+ def reset_attempt_state!
+ super
+ @prompt_token_count = nil
+ @thought_token_count = nil
+ @candidate_token_count = nil
+ @stop_reason = nil
+ @tool_index = nil
+ @thought_summary = nil
+ @current_batch_token = nil
+ @thought_signature_parts = nil
+ @emitted_thought_signature_parts_count = nil
+ @emitted_grounding_metadata_thinking = nil
+ @emitted_url_context_metadata_thinking = nil
+ end
+
def final_log_update(log)
log.request_tokens = @prompt_token_count if @prompt_token_count
log.response_tokens =
diff --git a/plugins/discourse-ai/lib/completions/endpoints/gemini_interactions.rb b/plugins/discourse-ai/lib/completions/endpoints/gemini_interactions.rb
index 0fd5c307624..193d81e58c8 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/gemini_interactions.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/gemini_interactions.rb
@@ -150,13 +150,13 @@ module DiscourseAi
level if THINKING_LEVELS.include?(level)
end
- def prepare_request(payload)
- if @streaming_mode
- @processor = nil
- @streaming_decoder = nil
- @pending_stream_error = nil
- end
+ def reset_attempt_state!
+ super
+ @streaming_decoder = nil
+ @pending_stream_error = nil
+ end
+ def prepare_request(payload)
headers = { "Content-Type" => "application/json", "x-goog-api-key" => llm_model.api_key }
headers.merge!(extra_request_headers)
Net::HTTP::Post.new(model_uri, headers).tap { |request| request.body = payload }
@@ -239,6 +239,10 @@ module DiscourseAi
end
end
+ def reported_reasoning_tokens
+ processor.reasoning_tokens
+ end
+
def final_log_update(log)
log.request_tokens = processor.prompt_tokens if processor.prompt_tokens
log.response_tokens = processor.completion_tokens if processor.completion_tokens
diff --git a/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb b/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb
index 027311d391d..2a90d2fb3fd 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb
@@ -69,6 +69,20 @@ module DiscourseAi
model_params
end
+ protected
+
+ def output_stop_reason
+ processor.stop_reason
+ end
+
+ def output_limit_reached?
+ processor.output_limit_reached
+ end
+
+ def reported_reasoning_tokens
+ processor.reasoning_tokens
+ end
+
private
def disable_streaming?
diff --git a/plugins/discourse-ai/lib/completions/endpoints/samba_nova.rb b/plugins/discourse-ai/lib/completions/endpoints/samba_nova.rb
index dbaf8265a9d..1b46e0580fd 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/samba_nova.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/samba_nova.rb
@@ -27,6 +27,14 @@ module DiscourseAi
AiApiAuditLog::Provider::SambaNova
end
+ protected
+
+ def reset_attempt_state!
+ super
+ @prompt_tokens = nil
+ @completion_tokens = nil
+ end
+
private
def model_uri
diff --git a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
index 07c360f4d1b..37cff68a03c 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
@@ -120,6 +120,14 @@ module DiscourseAi
)
end
+ protected
+
+ def reset_attempt_state!
+ super
+ @thinking = nil
+ @tool_batch_id = nil
+ end
+
private
def attach_tool_batch(result, response_id)
diff --git a/plugins/discourse-ai/lib/completions/gemini_interactions_message_processor.rb b/plugins/discourse-ai/lib/completions/gemini_interactions_message_processor.rb
index bf700a6a8fd..116786584a4 100644
--- a/plugins/discourse-ai/lib/completions/gemini_interactions_message_processor.rb
+++ b/plugins/discourse-ai/lib/completions/gemini_interactions_message_processor.rb
@@ -19,7 +19,7 @@ module DiscourseAi
url_context_result
].freeze
- attr_reader :prompt_tokens, :completion_tokens, :cache_read_tokens
+ attr_reader :prompt_tokens, :completion_tokens, :cache_read_tokens, :reasoning_tokens
def initialize(partial_tool_calls: false, output_thinking: false, &content_decoder)
@partial_tool_calls = partial_tool_calls
@@ -376,6 +376,7 @@ module DiscourseAi
return if usage.blank?
usage = normalize(usage)
+ @reasoning_tokens = usage[:total_thought_tokens] if usage.key?(:total_thought_tokens)
cached = usage[:total_cached_tokens].to_i
@prompt_tokens = [usage[:total_input_tokens].to_i - cached, 0].max
@cache_read_tokens = cached
diff --git a/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb b/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb
index be0621335e2..095c4f486c3 100644
--- a/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb
+++ b/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb
@@ -1,7 +1,12 @@
# frozen_string_literal: true
module DiscourseAi::Completions
class OpenAiMessageProcessor
- attr_reader :prompt_tokens, :completion_tokens, :cache_read_tokens, :cache_write_tokens
+ attr_reader :prompt_tokens,
+ :completion_tokens,
+ :cache_read_tokens,
+ :cache_write_tokens,
+ :stop_reason,
+ :reasoning_tokens
def initialize(partial_tool_calls: false)
@tools = {}
@@ -13,6 +18,10 @@ module DiscourseAi::Completions
@partial_tool_calls = partial_tool_calls
end
+ def output_limit_reached
+ @stop_reason == "length"
+ end
+
def process_message(json)
result = []
tool_calls = json.dig(:choices, 0, :message, :tool_calls)
@@ -30,6 +39,7 @@ module DiscourseAi::Completions
end
end
+ @stop_reason = json.dig(:choices, 0, :finish_reason) || @stop_reason
update_usage(json)
result
@@ -99,6 +109,7 @@ module DiscourseAi::Completions
result.concat(finish) if finished_tools
# Content and tool calls may share a delta; preserve whitespace-only text.
result.unshift(content) if !content.to_s.empty?
+ @stop_reason = json.dig(:choices, 0, :finish_reason) || @stop_reason
update_usage(json)
result.length > 1 ? result : result.first
@@ -155,6 +166,9 @@ module DiscourseAi::Completions
usage = json.dig(:usage)
return if !usage
+ @reasoning_tokens = usage.dig(:completion_tokens_details, :reasoning_tokens) if usage[
+ :completion_tokens_details
+ ]&.key?(:reasoning_tokens)
token_details = usage[:prompt_tokens_details] || {}
cached_tokens = (token_details[:cached_tokens] || usage[:prompt_cache_hit_tokens]).to_i
cache_write_tokens = token_details[:cache_write_tokens].to_i
diff --git a/plugins/discourse-ai/lib/completions/open_ai_responses_message_processor.rb b/plugins/discourse-ai/lib/completions/open_ai_responses_message_processor.rb
index 6c05692fac8..8d330ccec5c 100644
--- a/plugins/discourse-ai/lib/completions/open_ai_responses_message_processor.rb
+++ b/plugins/discourse-ai/lib/completions/open_ai_responses_message_processor.rb
@@ -3,7 +3,12 @@ module DiscourseAi::Completions
class OpenAiResponsesMessageProcessor
PROVIDER_KEY = :open_ai_responses
- attr_reader :prompt_tokens, :completion_tokens, :cache_read_tokens, :cache_write_tokens
+ attr_reader :prompt_tokens,
+ :completion_tokens,
+ :cache_read_tokens,
+ :cache_write_tokens,
+ :stop_reason,
+ :reasoning_tokens
def initialize(partial_tool_calls: false, output_thinking: false)
@tool = nil # currently streaming ToolCall
@@ -23,6 +28,10 @@ module DiscourseAi::Completions
@emitted_response_output_items = false
end
+ def output_limit_reached
+ @stop_reason == "max_output_tokens"
+ end
+
# @param json [Hash] full JSON response from responses.create / retrieve
# @return [Array<String,ToolCall>] pieces in the order they were produced
def process_message(json)
@@ -57,6 +66,7 @@ module DiscourseAi::Completions
end
end
+ update_stop_reason(json[:response] || json)
update_usage(json)
result
end
@@ -108,6 +118,7 @@ module DiscourseAi::Completions
track_completed_response(json[:response])
end
+ update_stop_reason(json[:response] || json)
update_usage(json)
rval
end
@@ -281,18 +292,29 @@ module DiscourseAi::Completions
# leave arguments empty; caller can decide how to handle
end
+ def update_stop_reason(response)
+ if response[:status] == "incomplete"
+ @stop_reason = response.dig(:incomplete_details, :reason)
+ elsif response[:status] == "completed"
+ @stop_reason = "completed"
+ end
+ end
+
def update_usage(json)
usage = json.dig(:response, :usage) || json[:usage]
return if !usage
+ @reasoning_tokens = usage.dig(:output_tokens_details, :reasoning_tokens) if usage[
+ :output_tokens_details
+ ]&.key?(:reasoning_tokens)
token_details = usage[:input_tokens_details] || {}
cached_tokens = token_details[:cached_tokens].to_i
cache_write_tokens = token_details[:cache_write_tokens].to_i
- @prompt_tokens ||= usage[:input_tokens] - cached_tokens - cache_write_tokens
- @completion_tokens ||= usage[:output_tokens]
- @cache_read_tokens ||= cached_tokens
- @cache_write_tokens ||= cache_write_tokens
+ @prompt_tokens = usage[:input_tokens] - cached_tokens - cache_write_tokens
+ @completion_tokens = usage[:output_tokens]
+ @cache_read_tokens = cached_tokens
+ @cache_write_tokens = cache_write_tokens
end
def build_partial_reasoning_delta(json, prefix: nil)
diff --git a/plugins/discourse-ai/lib/completions/structured_output.rb b/plugins/discourse-ai/lib/completions/structured_output.rb
index 1f98ca16f5f..07d8820927d 100644
--- a/plugins/discourse-ai/lib/completions/structured_output.rb
+++ b/plugins/discourse-ai/lib/completions/structured_output.rb
@@ -24,6 +24,19 @@ module DiscourseAi
@done = false
end
+ def content?
+ raw = @raw_response.strip
+ return false if raw.empty? || raw.match?(/\A\{\s*\}\z/)
+ return false if raw == "{" && (@prefilled || !finished?)
+
+ true
+ end
+
+ def prefill
+ @prefilled = true
+ self << "{"
+ end
+
def to_s
# we may want to also normalize the JSON here for the broken case
@raw_response.to_s
diff --git a/plugins/discourse-ai/lib/completions/xml_tag_stripper.rb b/plugins/discourse-ai/lib/completions/xml_tag_stripper.rb
index c1e6e6413b1..ffbc3cf339c 100644
--- a/plugins/discourse-ai/lib/completions/xml_tag_stripper.rb
+++ b/plugins/discourse-ai/lib/completions/xml_tag_stripper.rb
@@ -26,6 +26,12 @@ module DiscourseAi
@parsed.map { |node| node[:content] }.join
end
+ def finish_visible
+ return "" if @parsed.any? { |node| node[:type] == :open_tag }
+
+ @parsed.filter_map { |node| node[:content] if node[:type] == :maybe_tag }.join
+ end
+
def process_parsed(parsed)
output = []
buffer = []
diff --git a/plugins/discourse-ai/spec/lib/completions/anthropic_message_processor_spec.rb b/plugins/discourse-ai/spec/lib/completions/anthropic_message_processor_spec.rb
index cc85daafdd8..0c68adf36a0 100644
--- a/plugins/discourse-ai/spec/lib/completions/anthropic_message_processor_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/anthropic_message_processor_spec.rb
@@ -3,6 +3,35 @@
describe DiscourseAi::Completions::AnthropicMessageProcessor do
before { enable_current_plugin }
+ it "captures raw stop reasons and output limits in both modes without inventing reasoning usage" do
+ processor = described_class.new(streaming_mode: false)
+ processor.process_message(content: [], stop_reason: "max_tokens")
+ expect(processor).to have_attributes(
+ stop_reason: "max_tokens",
+ output_limit_reached: true,
+ reasoning_tokens: nil,
+ )
+
+ processor = described_class.new(streaming_mode: true)
+ processor.process_streamed_message(
+ type: "message_delta",
+ delta: {
+ stop_reason: "max_tokens",
+ },
+ usage: {
+ output_tokens: 100,
+ },
+ )
+ processor.process_streamed_message(type: "message_stop")
+ expect(processor).to have_attributes(
+ stop_reason: "max_tokens",
+ output_limit_reached: true,
+ reasoning_tokens: nil,
+ )
+ processor.process_streamed_message(type: "message_delta", delta: { stop_reason: "end_turn" })
+ expect(processor).to have_attributes(stop_reason: "end_turn", output_limit_reached: false)
+ end
+
it "correctly handles and combines partial thinking chunks into complete thinking objects" do
processor =
DiscourseAi::Completions::AnthropicMessageProcessor.new(
diff --git a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
index 6bac250e2db..64ba2df982b 100644
--- a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
@@ -90,6 +90,7 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
expect(prompt.messages.last).to eq(original.last)
expect(prompts.first.skip_trim).to eq(true)
expect(options.first).to include(user: user, feature_name: "context_compression")
+ expect(options.first[:max_tokens]).to be_nil
evidence = prompts.first.messages.last[:content]
expect(evidence).to include(original[1][:content], original[3][:content])
expect(evidence).not_to include(original.last[:content])
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb
index f5293bba883..924433748ad 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb
@@ -84,6 +84,223 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
before { enable_current_plugin }
+ it "captures only the final Claude attempt's stop reason after retrying an unfinished stream" do
+ response = instance_double(Net::HTTPResponse, code: "200")
+ reads = 0
+ allow(response).to receive(:read_body) do |&block|
+ reads += 1
+ if reads == 1
+ block.call(
+ "data: #{{ type: "message_delta", delta: { stop_reason: "max_tokens" }, usage: { output_tokens: 0 } }.to_json}\n\ndata: {",
+ )
+ raise EOFError, "Unexpected EOF"
+ end
+ events = [
+ { type: "content_block_delta", delta: { type: "text_delta", text: "Recovered answer" } },
+ { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 4 } },
+ ]
+ block.call(events.map { |event| "data: #{event.to_json}\n\n" }.join)
+ end
+ http = instance_double(Net::HTTP)
+ allow(http).to receive(:request).and_yield(response)
+ allow(FinalDestination::HTTP).to receive(:start).and_yield(http)
+ endpoint = described_class.new(model)
+ endpoint.stubs(:sleep_before_retry)
+ dialect = DiscourseAi::Completions::Dialects::Claude.new(prompt, model)
+ emitted = []
+ endpoint.perform_completion!(dialect, Discourse.system_user) { |partial| emitted << partial }
+ expect(emitted).to eq(["Recovered answer"])
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: "end_turn",
+ reasoning_tokens: nil,
+ response_tokens: 4,
+ )
+ end
+
+ describe "completion state" do
+ it "classifies non-streaming thinking-only output as empty" do
+ stub_request(:post, url).to_return(
+ body: {
+ content: [{ type: "thinking", thinking: "Considering", signature: "sig" }],
+ stop_reason: "max_tokens",
+ usage: {
+ input_tokens: 10,
+ output_tokens: 100,
+ },
+ }.to_json,
+ )
+ result = llm.generate(prompt, user: Discourse.system_user, output_thinking: true)
+ expect(result).to be_a(DiscourseAi::Completions::Thinking)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "max_tokens",
+ reasoning_tokens: nil,
+ response_tokens: 100,
+ )
+ end
+
+ it "classifies streamed thinking-only output as empty" do
+ events = [
+ { type: "content_block_start", content_block: { type: "thinking", thinking: "" } },
+ { type: "content_block_delta", delta: { type: "thinking_delta", thinking: "Considering" } },
+ { type: "content_block_stop" },
+ {
+ type: "message_delta",
+ delta: {
+ stop_reason: "max_tokens",
+ },
+ usage: {
+ output_tokens: 100,
+ },
+ },
+ ]
+ stub_request(:post, url).to_return(
+ body: events.map { |event| "data: #{event.to_json}\n\n" }.join,
+ )
+ emitted = []
+ llm.generate(prompt, user: Discourse.system_user, output_thinking: true) do |partial|
+ emitted << partial
+ end
+ expect(emitted).to all(be_a(DiscourseAi::Completions::Thinking))
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "max_tokens",
+ reasoning_tokens: nil,
+ response_tokens: 100,
+ )
+ end
+
+ it "preserves stripped-only non-streaming return values while classifying them as empty" do
+ text = "<thinking>Hidden</thinking>"
+ stub_request(:post, url).to_return(body: { content: [{ type: "text", text: text }] }.to_json)
+ expect(llm.generate(prompt_with_echo_tool, user: Discourse.system_user)).to eq(text)
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::EMPTY)
+ end
+
+ it "preserves stripped-only streaming return values without yielding hidden text" do
+ text = "<thinking>Hidden</thinking>"
+ stub_request(:post, url).to_return(
+ body:
+ "data: #{{ type: "content_block_delta", delta: { type: "text_delta", text: text } }.to_json}\n\n",
+ )
+ emitted = []
+ result =
+ llm.generate(prompt_with_echo_tool, user: Discourse.system_user) do |partial|
+ emitted << partial
+ end
+ expect(result).to eq(text)
+ expect(emitted).to eq([])
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::EMPTY)
+ end
+
+ it "does not flush reasoning from a truncated unclosed thinking block" do
+ events = [
+ {
+ type: "content_block_delta",
+ delta: {
+ type: "text_delta",
+ text: "<thinking>Hidden reasoning",
+ },
+ },
+ {
+ type: "message_delta",
+ delta: {
+ stop_reason: "max_tokens",
+ },
+ usage: {
+ output_tokens: 100,
+ },
+ },
+ ]
+ stub_request(:post, url).to_return(
+ body: events.map { |event| "data: #{event.to_json}\n\n" }.join,
+ )
+ emitted = []
+ llm.generate(prompt_with_echo_tool, user: Discourse.system_user) do |partial|
+ emitted << partial
+ end
+ expect(emitted).to eq([])
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "max_tokens",
+ )
+ end
+
+ it "does not flush reasoning from a truncated thinking block in an XML tool dialect" do
+ model.update!(provider_params: { disable_native_tools: true })
+ events = [
+ {
+ type: "content_block_delta",
+ delta: {
+ type: "text_delta",
+ text: "<thinking>Hidden reasoning",
+ },
+ },
+ {
+ type: "message_delta",
+ delta: {
+ stop_reason: "max_tokens",
+ },
+ usage: {
+ output_tokens: 100,
+ },
+ },
+ ]
+ stub_request(:post, url).to_return(
+ body: events.map { |event| "data: #{event.to_json}\n\n" }.join,
+ )
+ emitted = []
+ llm.generate(prompt_with_echo_tool, user: Discourse.system_user) do |partial|
+ emitted << partial
+ end
+ expect(emitted).to eq([])
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "max_tokens",
+ )
+ end
+
+ it "preserves dangling tag prefixes in non-streaming return values" do
+ stub_request(:post, url).to_return(
+ body: { content: [{ type: "text", text: "<think" }] }.to_json,
+ )
+ expect(llm.generate(prompt_with_echo_tool, user: Discourse.system_user)).to eq(
+ %w[<think <think],
+ )
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::COMPLETE)
+ end
+
+ it "preserves the existing native-tool streaming flush behavior for dangling tag prefixes" do
+ stub_request(:post, url).to_return(
+ body:
+ "data: #{{ type: "content_block_delta", delta: { type: "text_delta", text: "<think" } }.to_json}\n\n",
+ )
+ emitted = []
+ result =
+ llm.generate(prompt_with_echo_tool, user: Discourse.system_user) do |partial|
+ emitted << partial
+ end
+ expect(result).to eq("<think<think")
+ expect(emitted).to eq([])
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::EMPTY)
+ end
+
+ it "preserves stripped-only disable-streaming replay while keeping its classification empty" do
+ model.update!(provider_params: { disable_streaming: true })
+ text = "<thinking>Hidden</thinking>"
+ stub_request(:post, url).to_return(body: { content: [{ type: "text", text: text }] }.to_json)
+ emitted = []
+ result =
+ llm.generate(prompt_with_echo_tool, user: Discourse.system_user) do |partial|
+ emitted << partial
+ end
+ expect(result).to eq(text)
+ expect(emitted).to eq([text])
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::EMPTY)
+ end
+ end
+
it "settles pending cancelled reasoning without final output usage" do
thinking =
"Consider each independent source carefully before deciding what the evidence actually supports."
@@ -132,6 +349,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
end
expect(execution.work_budget.used).to eq(llm.tokenizer.size(thinking))
expect(execution.token_usage_tracker.response).to be > 0
+ expect(AiApiAuditLog.last.completion_state).to be_nil
expect(AiApiAuditLog.last.raw_response_payload).to include(thinking)
end
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
index 75f45885a1a..b61805a383c 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
@@ -10,6 +10,141 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
before { enable_current_plugin }
+ it "captures streamed truncation with SDK snake-case metadata" do
+ stub_sdk_client do |listeners|
+ fire_message_start(listeners)
+ fire_content_block_delta(listeners, text: "Answer")
+ fire_event(
+ listeners,
+ :message_stop,
+ Aws::BedrockRuntime::Types::MessageStopEvent.new(stop_reason: "max_tokens"),
+ )
+ fire_metadata(listeners)
+ end
+ model.to_llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ stop_reason: "max_tokens",
+ completion_state: AiApiAuditLog::CompletionState::TRUNCATED,
+ )
+ end
+
+ it "captures streamed normal completion with SDK snake-case metadata" do
+ stub_sdk_client do |listeners|
+ fire_message_start(listeners)
+ fire_content_block_delta(listeners, text: "Answer")
+ fire_event(
+ listeners,
+ :message_stop,
+ Aws::BedrockRuntime::Types::MessageStopEvent.new(stop_reason: "end_turn"),
+ )
+ fire_metadata(listeners)
+ end
+ model.to_llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ stop_reason: "end_turn",
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ )
+ end
+
+ describe "completion metadata" do
+ it "records SDK stop reasons and limits in non-streaming responses" do
+ response = mock_converse_response(text: "Partial")
+ response.stop_reason = "max_tokens"
+ stub_sdk_client(response: response)
+ model.to_llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::TRUNCATED,
+ stop_reason: "max_tokens",
+ reasoning_tokens: nil,
+ )
+ end
+
+ it "forwards streamed message-stop reasons even without content" do
+ stub_sdk_client do |listeners|
+ fire_message_start(listeners)
+ fire_event(
+ listeners,
+ :message_stop,
+ Aws::BedrockRuntime::Types::MessageStopEvent.new(stop_reason: "max_tokens"),
+ )
+ fire_metadata(listeners, output_tokens: 100)
+ end
+ model.to_llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "max_tokens",
+ response_tokens: 100,
+ )
+ end
+
+ it "classifies non-streaming structured output before returning early" do
+ response = mock_converse_response(text: '{"message":"Answer"}')
+ response.stop_reason = "max_tokens"
+ stub_sdk_client(response: response)
+ prompt = DiscourseAi::Completions::Prompt.new("Input")
+ dialect = DiscourseAi::Completions::Dialects::Converse.new(prompt, model)
+ endpoint.perform_completion!(
+ dialect,
+ user,
+ {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ },
+ )
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::TRUNCATED,
+ stop_reason: "max_tokens",
+ )
+ end
+ end
+
+ it "bounds unexpected SDK stop reasons before persisting" do
+ reason = "x" * 100
+ response = mock_converse_response
+ response.stop_reason = reason
+ stub_sdk_client(response: response)
+ model.to_llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last.stop_reason).to eq(reason.first(50))
+ end
+
+ it "classifies broken structured streaming output after finishing its buffer" do
+ stub_sdk_client do |listeners|
+ fire_message_start(listeners)
+ fire_content_block_delta(listeners, text: "{broken")
+ fire_message_stop(listeners)
+ fire_metadata(listeners)
+ end
+ partials = []
+ prompt = DiscourseAi::Completions::Prompt.new("Input")
+ dialect = DiscourseAi::Completions::Dialects::Converse.new(prompt, model)
+ params = {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ }
+ endpoint.perform_completion!(dialect, user, params) { |partial| partials << partial }
+ expect(partials.last).to be_a(DiscourseAi::Completions::StructuredOutput)
+ expect(partials.last.to_s).to eq("{broken")
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::COMPLETE)
+ end
+
def mock_converse_response(text: "Hello world", input_tokens: 10, output_tokens: 5)
response =
Aws::BedrockRuntime::Types::ConverseResponse.new(
@@ -290,6 +425,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
end
expect(execution.work_budget.used).to eq(model.to_llm.tokenizer.size("Partial answer"))
expect(execution.token_usage_tracker.response).to be > 0
+ expect(AiApiAuditLog.last.completion_state).to be_nil
expect(AiApiAuditLog.last.raw_request_payload).to be_present
end
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_interactions_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_interactions_spec.rb
index 689128b74dc..15d1e611ed2 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_interactions_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_interactions_spec.rb
@@ -20,6 +20,137 @@ RSpec.describe DiscourseAi::Completions::Endpoints::GeminiInteractions do
before { enable_current_plugin }
+ it "persists non-streaming absent thought usage" do
+ usage = { total_input_tokens: 10, total_output_tokens: 4 }
+ stub_request(:post, url).to_return(body: interaction_response(steps: [], usage: usage).to_json)
+ llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ reasoning_tokens: nil,
+ response_tokens: 4,
+ stop_reason: nil,
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ )
+ end
+
+ it "persists non-streaming zero thought usage" do
+ usage = { total_input_tokens: 10, total_output_tokens: 4, total_thought_tokens: 0 }
+ stub_request(:post, url).to_return(body: interaction_response(steps: [], usage: usage).to_json)
+ llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ reasoning_tokens: 0,
+ response_tokens: 4,
+ stop_reason: nil,
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ )
+ end
+
+ it "persists non-streaming positive thought usage" do
+ usage = { total_input_tokens: 10, total_output_tokens: 4, total_thought_tokens: 12 }
+ stub_request(:post, url).to_return(body: interaction_response(steps: [], usage: usage).to_json)
+ llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ reasoning_tokens: 12,
+ response_tokens: 16,
+ stop_reason: nil,
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ )
+ end
+
+ it "persists streamed absent thought usage" do
+ usage = { total_input_tokens: 10, total_output_tokens: 4 }
+ stub_request(:post, url).to_return(
+ body: sse_event("interaction.completed", interaction: { status: "completed", usage: usage }),
+ )
+ llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ reasoning_tokens: nil,
+ response_tokens: 4,
+ stop_reason: nil,
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ )
+ end
+
+ it "persists streamed zero thought usage" do
+ usage = { total_input_tokens: 10, total_output_tokens: 4, total_thought_tokens: 0 }
+ stub_request(:post, url).to_return(
+ body: sse_event("interaction.completed", interaction: { status: "completed", usage: usage }),
+ )
+ llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ reasoning_tokens: 0,
+ response_tokens: 4,
+ stop_reason: nil,
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ )
+ end
+
+ it "persists streamed positive thought usage" do
+ usage = { total_input_tokens: 10, total_output_tokens: 4, total_thought_tokens: 12 }
+ stub_request(:post, url).to_return(
+ body: sse_event("interaction.completed", interaction: { status: "completed", usage: usage }),
+ )
+ llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ reasoning_tokens: 12,
+ response_tokens: 16,
+ stop_reason: nil,
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ )
+ end
+
+ it "resets its usage and unfinished streaming decoder before a retried request" do
+ response = instance_double(Net::HTTPResponse, code: "200")
+ reads = 0
+ allow(response).to receive(:read_body) do |&block|
+ reads += 1
+ if reads == 1
+ block.call(
+ sse_event(
+ "interaction.completed",
+ interaction: {
+ status: "completed",
+ usage: {
+ total_input_tokens: 10,
+ total_output_tokens: 0,
+ total_thought_tokens: 0,
+ },
+ },
+ ) + 'data: {"event_type":',
+ )
+ raise EOFError, "Unexpected EOF"
+ end
+ block.call(
+ sse_event("step.start", step: { type: "model_output" }) +
+ sse_event("step.delta", delta: { type: "text", text: "Recovered answer" }) +
+ sse_event(
+ "interaction.completed",
+ interaction: {
+ status: "completed",
+ usage: {
+ total_input_tokens: 10,
+ total_output_tokens: 4,
+ },
+ },
+ ),
+ )
+ end
+ http = instance_double(Net::HTTP)
+ allow(http).to receive(:request).and_yield(response)
+ allow(FinalDestination::HTTP).to receive(:start).and_yield(http)
+ endpoint.stubs(:sleep_before_retry)
+ prompt = DiscourseAi::Completions::Prompt.new("Input")
+ dialect = DiscourseAi::Completions::Dialects::GeminiInteractions.new(prompt, model)
+ emitted = []
+ endpoint.perform_completion!(dialect, user) { |partial| emitted << partial }
+ expect(emitted).to eq(["Recovered answer"])
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: nil,
+ reasoning_tokens: nil,
+ response_tokens: 4,
+ )
+ end
+
def interaction_response(steps:, usage: nil, status: "completed")
{
object: "interaction",
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb
index ffd94041268..454567175a8 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb
@@ -225,6 +225,148 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Gemini do
before { enable_current_plugin }
+ describe "completion metadata" do
+ it "captures non-streaming no visible output metadata" do
+ payload = {
+ candidates: [{ content: { parts: [] }, finishReason: "MAX_TOKENS" }],
+ usageMetadata: {
+ promptTokenCount: 10,
+ candidatesTokenCount: 4,
+ thoughtsTokenCount: 12,
+ },
+ }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(body: payload.to_json)
+ model.to_llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "MAX_TOKENS",
+ reasoning_tokens: 12,
+ response_tokens: 16,
+ )
+ end
+
+ it "captures non-streaming limited text metadata" do
+ payload = {
+ candidates: [{ content: { parts: [{ text: "Partial" }] }, finishReason: "MAX_TOKENS" }],
+ usageMetadata: {
+ promptTokenCount: 10,
+ candidatesTokenCount: 4,
+ thoughtsTokenCount: 0,
+ },
+ }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(body: payload.to_json)
+ model.to_llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::TRUNCATED,
+ stop_reason: "MAX_TOKENS",
+ reasoning_tokens: 0,
+ response_tokens: 4,
+ )
+ end
+
+ it "captures non-streaming normal text metadata" do
+ payload = {
+ candidates: [{ content: { parts: [{ text: "Answer" }] }, finishReason: "STOP" }],
+ usageMetadata: {
+ promptTokenCount: 10,
+ candidatesTokenCount: 4,
+ },
+ }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(body: payload.to_json)
+ model.to_llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: "STOP",
+ reasoning_tokens: nil,
+ response_tokens: 4,
+ )
+ end
+
+ it "records non-streaming prompt blocking without a candidate finish reason" do
+ payload = { promptFeedback: { blockReason: "SAFETY" } }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(body: payload.to_json)
+ model.to_llm.generate("Input", user: user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "SAFETY",
+ )
+ end
+
+ it "captures streamed no visible output metadata" do
+ payload = {
+ candidates: [{ content: { parts: [] }, finishReason: "MAX_TOKENS" }],
+ usageMetadata: {
+ promptTokenCount: 10,
+ candidatesTokenCount: 4,
+ thoughtsTokenCount: 12,
+ },
+ }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(
+ body: "data: #{payload.to_json}\n\n",
+ )
+ model.to_llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "MAX_TOKENS",
+ reasoning_tokens: 12,
+ response_tokens: 16,
+ )
+ end
+
+ it "captures streamed limited text metadata" do
+ payload = {
+ candidates: [{ content: { parts: [{ text: "Partial" }] }, finishReason: "MAX_TOKENS" }],
+ usageMetadata: {
+ promptTokenCount: 10,
+ candidatesTokenCount: 4,
+ thoughtsTokenCount: 0,
+ },
+ }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(
+ body: "data: #{payload.to_json}\n\n",
+ )
+ model.to_llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::TRUNCATED,
+ stop_reason: "MAX_TOKENS",
+ reasoning_tokens: 0,
+ response_tokens: 4,
+ )
+ end
+
+ it "captures streamed normal text metadata" do
+ payload = {
+ candidates: [{ content: { parts: [{ text: "Answer" }] }, finishReason: "STOP" }],
+ usageMetadata: {
+ promptTokenCount: 10,
+ candidatesTokenCount: 4,
+ },
+ }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(
+ body: "data: #{payload.to_json}\n\n",
+ )
+ model.to_llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: "STOP",
+ reasoning_tokens: nil,
+ response_tokens: 4,
+ )
+ end
+
+ it "records streamed prompt blocking without a candidate finish reason" do
+ payload = { promptFeedback: { blockReason: "SAFETY" } }
+ stub_request(:post, /generativelanguage.googleapis.com/).to_return(
+ body: "data: #{payload.to_json}\n\n",
+ )
+ model.to_llm.generate("Input", user: user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "SAFETY",
+ )
+ end
+ end
+
it "uses Gemini RetryInfo retryDelay from rate limit response bodies" do
DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
DiscourseAi::Completions::Endpoints::Gemini
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
index 1cc87c8e580..361faf889ad 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
@@ -211,6 +211,337 @@ RSpec.describe DiscourseAi::Completions::Endpoints::OpenAi do
before { enable_current_plugin }
+ def stub_audited_completion(text, finish_reason: "stop", streaming: false)
+ usage = {
+ prompt_tokens: 10,
+ completion_tokens: 5,
+ completion_tokens_details: {
+ reasoning_tokens: 4,
+ },
+ }
+ response =
+ if streaming
+ "data: #{{ choices: [{ delta: { content: text }, finish_reason: finish_reason }] }.to_json}\n\n" +
+ "data: #{{ choices: [], usage: usage }.to_json}\n\ndata: [DONE]\n\n"
+ else
+ {
+ choices: [{ message: { content: text }, finish_reason: finish_reason }],
+ usage: usage,
+ }.to_json
+ end
+ open_ai_mock.stub_raw(response)
+ end
+
+ it "resets stop reasons, usage, and an unfinished decoder across retried streaming attempts" do
+ response = instance_double(Net::HTTPResponse, code: "200")
+ reads = 0
+ allow(response).to receive(:read_body) do |&block|
+ reads += 1
+ if reads == 1
+ block.call(
+ "data: #{{ choices: [{ delta: {}, finish_reason: "length" }], usage: { prompt_tokens: 10, completion_tokens: 0, completion_tokens_details: { reasoning_tokens: 7 } } }.to_json}\n\ndata: {\"choices\":",
+ )
+ raise EOFError, "Unexpected EOF"
+ end
+ block.call(
+ "data: #{{ choices: [{ delta: { content: "Recovered answer" }, finish_reason: "stop" }] }.to_json}\n\ndata: [DONE]\n\n",
+ )
+ end
+ http = instance_double(Net::HTTP)
+ allow(http).to receive(:request).and_yield(response)
+ allow(FinalDestination::HTTP).to receive(:start).and_yield(http)
+ endpoint.stubs(:sleep_before_retry)
+ emitted = []
+ endpoint.perform_completion!(
+ compliance.dialect(prompt: compliance.generic_prompt),
+ user,
+ ) { |partial| emitted << partial }
+ expect(emitted).to eq(["Recovered answer"])
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: "stop",
+ reasoning_tokens: nil,
+ )
+ end
+
+ describe "completion state" do
+ it "classifies non-streaming normal text" do
+ stub_audited_completion("Answer", finish_reason: "stop")
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: "stop",
+ reasoning_tokens: 4,
+ response_tokens: 5,
+ )
+ end
+
+ it "classifies non-streaming limited text" do
+ stub_audited_completion("Partial", finish_reason: "length")
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::TRUNCATED,
+ stop_reason: "length",
+ reasoning_tokens: 4,
+ response_tokens: 5,
+ )
+ end
+
+ it "classifies non-streaming blank output" do
+ stub_audited_completion(" ", finish_reason: "length")
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "length",
+ reasoning_tokens: 4,
+ response_tokens: 5,
+ )
+ end
+
+ it "classifies non-streaming structured empty objects" do
+ stub_audited_completion("{}", finish_reason: "length")
+ params = {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ }
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user, params)
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::EMPTY)
+ end
+
+ it "classifies non-streaming structured broken JSON" do
+ stub_audited_completion("{broken", finish_reason: "length")
+ params = {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ }
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user, params)
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::TRUNCATED)
+ end
+
+ it "classifies non-streaming structured a finished incomplete JSON prefix" do
+ stub_audited_completion("{", finish_reason: "length")
+ params = {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ }
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user, params)
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::TRUNCATED)
+ end
+
+ it "classifies streamed normal text" do
+ stub_audited_completion("Answer", finish_reason: "stop", streaming: true)
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: "stop",
+ reasoning_tokens: 4,
+ response_tokens: 5,
+ )
+ end
+
+ it "classifies streamed limited text" do
+ stub_audited_completion("Partial", finish_reason: "length", streaming: true)
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::TRUNCATED,
+ stop_reason: "length",
+ reasoning_tokens: 4,
+ response_tokens: 5,
+ )
+ end
+
+ it "classifies streamed blank output" do
+ stub_audited_completion(" ", finish_reason: "length", streaming: true)
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "length",
+ reasoning_tokens: 4,
+ response_tokens: 5,
+ )
+ end
+
+ it "classifies streamed structured empty objects" do
+ stub_audited_completion("{}", finish_reason: "length", streaming: true)
+ params = {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ }
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user, params) { |partial| }
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::EMPTY)
+ end
+
+ it "classifies streamed structured broken JSON" do
+ stub_audited_completion("{broken", finish_reason: "length", streaming: true)
+ params = {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ }
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user, params) { |partial| }
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::TRUNCATED)
+ end
+
+ it "classifies streamed structured a finished incomplete JSON prefix" do
+ stub_audited_completion("{", finish_reason: "length", streaming: true)
+ params = {
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ }
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user, params) { |partial| }
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::TRUNCATED)
+ end
+
+ it "classifies disable_streaming replay using the processed non-streaming output" do
+ model.update!(provider_params: { disable_streaming: true })
+ open_ai_mock.stub_raw(
+ { choices: [{ message: { content: "Partial" }, finish_reason: "length" }] }.to_json,
+ )
+ emitted = []
+ endpoint.perform_completion!(
+ compliance.dialect(prompt: compliance.generic_prompt),
+ user,
+ ) { |partial| emitted << partial }
+ expect(emitted).to eq(["Partial"])
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::TRUNCATED)
+ end
+
+ it "leaves failed requests unclassified and records only final-attempt metadata after a retry" do
+ endpoint.stubs(:sleep_before_retry)
+ stub_request(:post, model.url).to_return(
+ { status: 500, body: "Server error" },
+ {
+ status: 200,
+ body: { choices: [{ message: { content: "Answer" }, finish_reason: "stop" }] }.to_json,
+ },
+ )
+ endpoint.perform_completion!(compliance.dialect(prompt: compliance.generic_prompt), user)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ stop_reason: "stop",
+ reasoning_tokens: nil,
+ )
+ expect(AiApiAuditLog.last.request_attempts.map { |attempt| attempt["status"] }).to eq(
+ [500, 200],
+ )
+
+ stub_request(:post, model.url).to_return(status: 500, body: "Server error")
+ stub_const(DiscourseAi::Completions::Endpoints::Base, :TRANSIENT_ERROR_RETRY_DELAYS, []) do
+ expect {
+ endpoint.perform_completion!(compliance.dialect(prompt: compliance.generic_prompt), user)
+ }.to raise_error(DiscourseAi::Completions::Endpoints::Base::CompletionFailed)
+ end
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: nil,
+ stop_reason: nil,
+ reasoning_tokens: nil,
+ )
+ end
+
+ it "bounds unexpected provider stop reasons before persisting" do
+ reason = "x" * 100
+ stub_audited_completion("Answer", finish_reason: reason)
+ endpoint.perform_completion!(compliance.dialect(prompt: compliance.generic_prompt), user)
+ expect(AiApiAuditLog.last.stop_reason).to eq(reason.first(50))
+ end
+
+ it "preserves successful cancellation settlement while leaving completion state unknown" do
+ partial = "A long partial answer with more than one token"
+ payload = {
+ choices: [{ delta: { content: partial } }],
+ usage: {
+ prompt_tokens: 10,
+ completion_tokens: 1,
+ },
+ }
+ open_ai_mock.stub_raw("data: #{payload.to_json}\n\n")
+ execution =
+ DiscourseAi::Completions::ExecutionContext.new(
+ work_budget: DiscourseAi::Completions::TurnWorkBudget.new(limit: 4000),
+ )
+ cancel = DiscourseAi::Completions::CancelManager.new
+ model
+ .to_llm
+ .generate(
+ "Input",
+ user: user,
+ execution_context: execution,
+ cancel_manager: cancel,
+ ) { |fragment| cancel.cancel! }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: nil,
+ response_status: 200,
+ response_tokens: 1,
+ )
+ expect(execution.work_budget.used).to eq(1)
+ end
+ end
+
describe "DNS SRV endpoints" do
let(:domain) { "_openai._tcp.example.com" }
let(:resolved_url) { "https://llm.example.com:8443" }
@@ -739,6 +1070,20 @@ RSpec.describe DiscourseAi::Completions::Endpoints::OpenAi do
}
end
+ it "counts streamed XML-only tool output as visible" do
+ model.update!(provider_params: { disable_native_tools: true })
+ response =
+ "data: #{{ choices: [{ delta: { content: xml_tool_call_response }, finish_reason: "stop" }] }.to_json}\n\ndata: [DONE]\n\n"
+ open_ai_mock.stub_raw(response)
+ emitted = []
+ endpoint.perform_completion!(
+ compliance.dialect(prompt: compliance.generic_prompt(tools: [weather_tool])),
+ user,
+ ) { |partial| emitted << partial }
+ expect(emitted.grep(DiscourseAi::Completions::ToolCall).map(&:name)).to eq(["get_weather"])
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::COMPLETE)
+ end
+
it "parses XML tool calls" do
response = {
id: "chatcmpl-6sZfAb30Rnv9Q7ufzFwvQsMpjZh8S",
@@ -774,6 +1119,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::OpenAi do
expect(body_parsed[:messages][0][:content]).to include("<function_calls>")
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::COMPLETE)
expect(tool_call.name).to eq("get_weather")
expect(tool_call.parameters).to eq({ location: "Sydney", unit: "c", is_it_hot: true })
end
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
index 83ffec39e47..b8f7d48dd72 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
@@ -179,6 +179,54 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Vllm do
before { enable_current_plugin }
+ it "classifies non-streaming reasoning-only output as empty" do
+ payload = {
+ choices: [
+ { message: { reasoning_content: "Considering", content: "" }, finish_reason: "length" },
+ ],
+ usage: {
+ prompt_tokens: 10,
+ completion_tokens: 100,
+ completion_tokens_details: {
+ reasoning_tokens: 100,
+ },
+ },
+ }
+ stub_request(:post, llm_model.url).to_return(body: payload.to_json)
+ llm_model.to_llm.generate("Input", user: user, output_thinking: true)
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "length",
+ reasoning_tokens: 100,
+ response_tokens: 100,
+ )
+ end
+
+ it "classifies streamed reasoning-only output as empty" do
+ payload = {
+ choices: [
+ { delta: { reasoning_content: "Considering", content: "" }, finish_reason: "length" },
+ ],
+ usage: {
+ prompt_tokens: 10,
+ completion_tokens: 100,
+ completion_tokens_details: {
+ reasoning_tokens: 100,
+ },
+ },
+ }
+ stub_request(:post, llm_model.url).to_return(
+ body: "data: #{payload.to_json}\n\ndata: [DONE]\n\n",
+ )
+ llm_model.to_llm.generate("Input", user: user, output_thinking: true) { |partial| }
+ expect(AiApiAuditLog.last).to have_attributes(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ stop_reason: "length",
+ reasoning_tokens: 100,
+ response_tokens: 100,
+ )
+ end
+
def capture_payload_for_completion(**generation_args)
captured_payload = nil
stub =
diff --git a/plugins/discourse-ai/spec/lib/completions/open_ai_message_processor_spec.rb b/plugins/discourse-ai/spec/lib/completions/open_ai_message_processor_spec.rb
index cbb0fd430f0..3716128acf6 100644
--- a/plugins/discourse-ai/spec/lib/completions/open_ai_message_processor_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/open_ai_message_processor_spec.rb
@@ -3,6 +3,46 @@
RSpec.describe DiscourseAi::Completions::OpenAiMessageProcessor do
before { enable_current_plugin }
+ describe "completion metadata" do
+ it "captures the consumed choice's stop reason in both modes and retains it through usage-only chunks" do
+ %i[process_message process_streamed_message].each do |method|
+ processor = described_class.new
+ processor.public_send(
+ method,
+ choices: [{ finish_reason: "length" }, { finish_reason: "stop" }],
+ )
+ processor.public_send(
+ method,
+ choices: [],
+ usage: {
+ prompt_tokens: 10,
+ completion_tokens: 5,
+ completion_tokens_details: {
+ reasoning_tokens: 4,
+ },
+ },
+ )
+ expect(processor).to have_attributes(
+ stop_reason: "length",
+ output_limit_reached: true,
+ reasoning_tokens: 4,
+ )
+ end
+ end
+
+ it "distinguishes missing reasoning usage from explicit zero and other stop reasons" do
+ processor = described_class.new
+ processor.process_message(choices: [{ finish_reason: "content_filter" }])
+ expect(processor).to have_attributes(
+ stop_reason: "content_filter",
+ output_limit_reached: false,
+ reasoning_tokens: nil,
+ )
+ processor.process_message(usage: { completion_tokens_details: { reasoning_tokens: 0 } })
+ expect(processor.reasoning_tokens).to eq(0)
+ end
+ end
+
def chunk(delta:, finish_reason: nil)
choice = { delta: delta }
choice[:finish_reason] = finish_reason if finish_reason
diff --git a/plugins/discourse-ai/spec/lib/completions/open_ai_responses_message_processor_spec.rb b/plugins/discourse-ai/spec/lib/completions/open_ai_responses_message_processor_spec.rb
index cec6088671a..d4a0fb5eba2 100644
--- a/plugins/discourse-ai/spec/lib/completions/open_ai_responses_message_processor_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/open_ai_responses_message_processor_spec.rb
@@ -1,6 +1,126 @@
# frozen_string_literal: true
RSpec.describe DiscourseAi::Completions::OpenAiResponsesMessageProcessor do
+ describe "completion metadata" do
+ it "captures non-streaming incomplete response metadata" do
+ processor = described_class.new
+ response = {
+ status: "incomplete",
+ incomplete_details: {
+ reason: "max_output_tokens",
+ },
+ usage: {
+ input_tokens: 10,
+ output_tokens: 5,
+ output_tokens_details: {
+ reasoning_tokens: 5,
+ },
+ },
+ }
+ processor.process_message(response)
+ expect(processor).to have_attributes(
+ stop_reason: "max_output_tokens",
+ output_limit_reached: true,
+ reasoning_tokens: 5,
+ )
+ end
+
+ it "captures streamed incomplete response metadata" do
+ processor = described_class.new
+ response = {
+ status: "incomplete",
+ incomplete_details: {
+ reason: "max_output_tokens",
+ },
+ usage: {
+ input_tokens: 10,
+ output_tokens: 5,
+ output_tokens_details: {
+ reasoning_tokens: 5,
+ },
+ },
+ }
+ processor.process_streamed_message(type: "response.incomplete", response: response)
+ expect(processor).to have_attributes(
+ stop_reason: "max_output_tokens",
+ output_limit_reached: true,
+ reasoning_tokens: 5,
+ )
+ end
+
+ it "captures incomplete metadata without replaying terminal native output items" do
+ processor = described_class.new(output_thinking: true)
+ processor.process_streamed_message(
+ type: "response.incomplete",
+ response: {
+ status: "incomplete",
+ incomplete_details: {
+ reason: "max_output_tokens",
+ },
+ output: [
+ { type: "web_search_call", id: "ws_1", action: { type: "search", query: "Query" } },
+ ],
+ usage: {
+ input_tokens: 10,
+ output_tokens: 5,
+ output_tokens_details: {
+ reasoning_tokens: 5,
+ },
+ },
+ },
+ )
+ expect(processor.finish).to eq([])
+ expect(processor).to have_attributes(
+ stop_reason: "max_output_tokens",
+ output_limit_reached: true,
+ reasoning_tokens: 5,
+ )
+ end
+
+ it "updates usage on completion and distinguishes zero from missing reasoning usage" do
+ processor = described_class.new
+ processor.process_streamed_message(
+ type: "response.created",
+ response: {
+ usage: {
+ input_tokens: 10,
+ output_tokens: 0,
+ },
+ },
+ )
+ expect(processor.reasoning_tokens).to be_nil
+ processor.process_streamed_message(
+ type: "response.completed",
+ response: {
+ status: "completed",
+ usage: {
+ input_tokens: 12,
+ output_tokens: 8,
+ output_tokens_details: {
+ reasoning_tokens: 0,
+ },
+ },
+ },
+ )
+ expect(processor).to have_attributes(
+ stop_reason: "completed",
+ output_limit_reached: false,
+ completion_tokens: 8,
+ reasoning_tokens: 0,
+ )
+ processor.process_message(
+ status: "incomplete",
+ incomplete_details: {
+ reason: "content_filter",
+ },
+ )
+ expect(processor).to have_attributes(
+ stop_reason: "content_filter",
+ output_limit_reached: false,
+ )
+ end
+ end
+
describe "native web tool activity" do
it "emits a thinking item for streamed web search calls" do
processor = described_class.new(output_thinking: true)
diff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
index 9b1b48eb06e..7c126bb6fcd 100644
--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
@@ -28,6 +28,40 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
before { enable_current_plugin }
+ describe "#content?" do
+ it "ignores whitespace and empty objects" do
+ ["", " \n", "{ }"].each do |raw|
+ output = described_class.new(message: { type: "string" })
+ output << raw
+ expect(output.content?).to eq(false)
+ end
+ end
+
+ it "distinguishes synthetic prefill from a broken brace-only provider response" do
+ structured_output << "{"
+ expect(structured_output.content?).to eq(false)
+ structured_output.finish
+ expect(structured_output.content?).to eq(true)
+
+ prefilled = described_class.new(message: { type: "string" })
+ prefilled.prefill
+ prefilled << " "
+ prefilled.finish
+ expect(prefilled.content?).to eq(false)
+ end
+
+ it "counts false, zero, and broken content without consuming property cursors" do
+ %w[{"bool":false} {"number":0} {broken].each do |raw|
+ output = described_class.new(bool: { type: "boolean" }, number: { type: "integer" })
+ output << raw
+ expect(output.content?).to eq(true)
+ end
+ structured_output << '{"message":"Answer"}'
+ expect(structured_output.content?).to eq(true)
+ expect(structured_output.read_buffered_property(:message)).to eq("Answer")
+ end
+ end
+
describe "#read_buffered_property_chunk" do
it "yields non-empty chunks, including whitespace-only ones" do
yielded = []
diff --git a/plugins/discourse-ai/spec/lib/completions/xml_tag_stripper_spec.rb b/plugins/discourse-ai/spec/lib/completions/xml_tag_stripper_spec.rb
index 6ce2985cd91..9e96d2f0ca7 100644
--- a/plugins/discourse-ai/spec/lib/completions/xml_tag_stripper_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/xml_tag_stripper_spec.rb
@@ -5,6 +5,19 @@ describe DiscourseAi::Completions::PromptMessagesBuilder do
before { enable_current_plugin }
+ describe "#finish_visible" do
+ it "drops unclosed stripped blocks, including their dangling tag tails" do
+ expect(tag_stripper << "<thinking>Hidden<res").to be_nil
+ expect(tag_stripper.finish_visible).to eq("")
+ expect(tag_stripper.finish).to eq("<thinking>Hidden<res")
+ end
+
+ it "returns a dangling tag prefix outside stripped blocks" do
+ expect(tag_stripper << "<thinking>Hidden</thinking><res").to be_nil
+ expect(tag_stripper.finish_visible).to eq("<res")
+ end
+ end
+
it "strips tags correctly in simple cases" do
result = tag_stripper << "x<thinking>hello</thinki"
expect(result).to eq("x")
diff --git a/plugins/discourse-ai/spec/requests/admin/ai_logs_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_logs_controller_spec.rb
index 2c470844e02..2b7e3f501bb 100644
--- a/plugins/discourse-ai/spec/requests/admin/ai_logs_controller_spec.rb
+++ b/plugins/discourse-ai/spec/requests/admin/ai_logs_controller_spec.rb
@@ -13,6 +13,59 @@ RSpec.describe DiscourseAi::Admin::AiLogsController do
sign_in(admin) if !example.metadata[:anonymous]
end
+ it "keeps list and detail outcomes in parity and filters incomplete responses within successful requests" do
+ records =
+ [
+ [nil, 200, 0, "successful"],
+ [nil, nil, 1, "successful"],
+ [nil, nil, 0, "failed"],
+ [AiApiAuditLog::CompletionState::EMPTY, 200, 100, "empty"],
+ [AiApiAuditLog::CompletionState::TRUNCATED, 200, 100, "truncated"],
+ [AiApiAuditLog::CompletionState::EMPTY, 500, 100, "failed"],
+ ].map do |state, status, tokens, outcome|
+ [
+ Fabricate(
+ :ai_api_audit_log,
+ completion_state: state,
+ response_status: status,
+ response_tokens: tokens,
+ reasoning_tokens: 0,
+ stop_reason: "length",
+ ),
+ outcome,
+ ]
+ end
+ get index_path
+ listed = response.parsed_body["logs"].index_by { |log| log["id"] }
+ records.each do |log, outcome|
+ expect(listed[log.id]["outcome"]).to eq(outcome)
+ get "/admin/plugins/discourse-ai/ai-logs/#{log.id}.json"
+ expect(response.parsed_body).to include(
+ "outcome" => outcome,
+ "reasoning_tokens" => 0,
+ "stop_reason" => "length",
+ )
+ end
+ historical_success, historical_positive, historical_failure, empty, truncated, failed_empty =
+ records.map(&:first)
+ get index_path, params: { outcome: "empty" }
+ expect(response.parsed_body["logs"].map { |log| log["id"] }).to eq([empty.id])
+ get index_path, params: { outcome: "truncated" }
+ expect(response.parsed_body["logs"].map { |log| log["id"] }).to eq([truncated.id])
+ get index_path, params: { outcome: "successful" }
+ expect(response.parsed_body["logs"].map { |log| log["id"] }).to contain_exactly(
+ historical_success.id,
+ historical_positive.id,
+ empty.id,
+ truncated.id,
+ )
+ get index_path, params: { outcome: "failed" }
+ expect(response.parsed_body["logs"].map { |log| log["id"] }).to contain_exactly(
+ historical_failure.id,
+ failed_empty.id,
+ )
+ end
+
def request_ai_logs_endpoint(endpoint, log, headers: {})
case endpoint
when :index
diff --git a/plugins/discourse-ai/spec/system/ai_logs_spec.rb b/plugins/discourse-ai/spec/system/ai_logs_spec.rb
index f4eafad10f2..d55ef95143b 100644
--- a/plugins/discourse-ai/spec/system/ai_logs_spec.rb
+++ b/plugins/discourse-ai/spec/system/ai_logs_spec.rb
@@ -64,6 +64,30 @@ RSpec.describe "AI logs admin page" do
sign_in(admin)
end
+ it "surfaces empty completions with reasoning usage and the raw stop reason in details" do
+ successful_log =
+ Fabricate(
+ :ai_api_audit_log,
+ response_status: 200,
+ completion_state: AiApiAuditLog::CompletionState::COMPLETE,
+ )
+ log.update!(
+ completion_state: AiApiAuditLog::CompletionState::EMPTY,
+ reasoning_tokens: 10,
+ stop_reason: "length",
+ )
+ ai_logs_page.visit
+ expect(ai_logs_page).to have_log(successful_log)
+ ai_logs_page.select_outcome(I18n.t("js.discourse_ai.logs.empty"))
+ expect(ai_logs_page).to have_no_log(successful_log)
+ expect(ai_logs_page).to have_log(log)
+ expect(page).to have_css(".ai-logs__status-icon.--warning.d-icon-triangle-exclamation")
+ ai_logs_page.open_log(log)
+ expect(page).to have_css(".ai-log-detail-modal__metadata", text: "Empty response (200)")
+ expect(page).to have_css(".ai-log-detail-modal__metadata-reasoning", text: "10")
+ expect(page).to have_css(".ai-log-detail-modal__metadata-stop-reason", text: "length")
+ end
+
it "applies outcome, feature, and ID filters" do
Fabricate(:ai_api_audit_log, feature_name: "other-feature")
ai_logs_page.visit
diff --git a/plugins/discourse-ai/test/javascripts/integration/components/ai-log-detail-modal-test.gjs b/plugins/discourse-ai/test/javascripts/integration/components/ai-log-detail-modal-test.gjs
new file mode 100644
index 00000000000..928fff54f78
--- /dev/null
+++ b/plugins/discourse-ai/test/javascripts/integration/components/ai-log-detail-modal-test.gjs
@@ -0,0 +1,54 @@
+import { render, settled } from "@ember/test-helpers";
+import { module, test } from "qunit";
+import ModalContainer from "discourse/components/modal-container";
+import { setupRenderingTest } from "discourse/tests/helpers/component-test";
+import pretender, { response } from "discourse/tests/helpers/create-pretender";
+import AiLogDetailModal from "discourse/plugins/discourse-ai/discourse/components/modal/ai-log-detail-modal";
+
+module("Integration | Component | AiLogDetailModal", function (hooks) {
+ setupRenderingTest(hooks);
+
+ test("renders incomplete outcomes, raw stop reasons, and explicit zero reasoning tokens", async function (assert) {
+ pretender.get("/admin/plugins/discourse-ai/ai-logs/1.json", () =>
+ response({
+ id: 1,
+ outcome: "empty",
+ response_status: 200,
+ reasoning_tokens: 0,
+ stop_reason: "length",
+ })
+ );
+ const model = { logId: 1 };
+ await render(<template><ModalContainer /></template>);
+ this.owner.lookup("service:modal").show(AiLogDetailModal, { model });
+ await settled();
+ assert
+ .dom(".ai-log-detail-modal__metadata")
+ .includesText(
+ "Empty response (200)",
+ "the server outcome is used in details"
+ );
+ assert
+ .dom(".ai-log-detail-modal__metadata-reasoning dd")
+ .hasText("0", "explicit zero reasoning usage is retained in details");
+ assert
+ .dom(".ai-log-detail-modal__metadata-stop-reason dd")
+ .hasText("length", "the raw provider stop reason is shown");
+ });
+
+ test("omits absent provider metadata", async function (assert) {
+ pretender.get("/admin/plugins/discourse-ai/ai-logs/1.json", () =>
+ response({ id: 1, outcome: "successful", response_status: 200 })
+ );
+ const model = { logId: 1 };
+ await render(<template><ModalContainer /></template>);
+ this.owner.lookup("service:modal").show(AiLogDetailModal, { model });
+ await settled();
+ assert
+ .dom(".ai-log-detail-modal__metadata-reasoning")
+ .doesNotExist("unknown reasoning usage is omitted");
+ assert
+ .dom(".ai-log-detail-modal__metadata-stop-reason")
+ .doesNotExist("unknown stop reasons are omitted");
+ });
+});
diff --git a/plugins/discourse-ai/test/javascripts/integration/components/ai-log-row-test.gjs b/plugins/discourse-ai/test/javascripts/integration/components/ai-log-row-test.gjs
index e4f06226395..20aa74c18b9 100644
--- a/plugins/discourse-ai/test/javascripts/integration/components/ai-log-row-test.gjs
+++ b/plugins/discourse-ai/test/javascripts/integration/components/ai-log-row-test.gjs
@@ -6,6 +6,64 @@ import AiLogRow from "discourse/plugins/discourse-ai/discourse/components/ai-log
module("Integration | Component | AiLogRow", function (hooks) {
setupRenderingTest(hooks);
+ test("renders empty outcomes as warnings", async function (assert) {
+ this.log = {
+ id: 1,
+ outcome: "empty",
+ response_status: 200,
+ request_tokens: 3000,
+ response_tokens: 2000,
+ reasoning_tokens: 1900,
+ };
+ await render(
+ <template>
+ <table><tbody><AiLogRow @log={{this.log}} /></tbody></table>
+ </template>
+ );
+ assert
+ .dom(".ai-logs__status-icon.--warning.d-icon-triangle-exclamation")
+ .hasAttribute(
+ "aria-label",
+ "Empty response",
+ "warnings have an accessible label"
+ );
+ assert
+ .dom(".ai-logs__token-summary .sr-only")
+ .hasText(
+ "3,000 input tokens and 2,000 output tokens",
+ "reasoning usage is left to the detail view"
+ );
+ });
+
+ test("renders truncated outcomes as warnings", async function (assert) {
+ this.log = {
+ id: 1,
+ outcome: "truncated",
+ response_status: 200,
+ request_tokens: 3000,
+ response_tokens: 2000,
+ reasoning_tokens: 1900,
+ };
+ await render(
+ <template>
+ <table><tbody><AiLogRow @log={{this.log}} /></tbody></table>
+ </template>
+ );
+ assert
+ .dom(".ai-logs__status-icon.--warning.d-icon-triangle-exclamation")
+ .hasAttribute(
+ "aria-label",
+ "Truncated at output limit",
+ "warnings have an accessible label"
+ );
+ assert
+ .dom(".ai-logs__token-summary .sr-only")
+ .hasText(
+ "3,000 input tokens and 2,000 output tokens",
+ "reasoning usage is left to the detail view"
+ );
+ });
+
test("renders statusless positive-token calls as successful", async function (assert) {
this.log = {
id: 43,
diff --git a/plugins/discourse-ai/test/javascripts/integration/components/ai-logs-test.gjs b/plugins/discourse-ai/test/javascripts/integration/components/ai-logs-test.gjs
index 4f3d90dbd46..eb275bf6ab4 100644
--- a/plugins/discourse-ai/test/javascripts/integration/components/ai-logs-test.gjs
+++ b/plugins/discourse-ai/test/javascripts/integration/components/ai-logs-test.gjs
@@ -296,3 +296,33 @@ module("Integration | Component | ai-logs new-log banner", function (hooks) {
);
});
});
+
+module("Integration | Component | AiLogs", function (hooks) {
+ setupRenderingTest(hooks);
+
+ hooks.beforeEach(function () {
+ this.owner.unregister("service:router");
+ this.owner.register("service:router", RouterStub);
+ });
+
+ test("offers incomplete outcome filters", async function (assert) {
+ const model = {
+ logs: [],
+ meta: { has_more: false },
+ models: [],
+ features: [],
+ };
+ const queryParams = {};
+ await render(
+ <template>
+ <AiLogs @model={{model}} @queryParams={{queryParams}} />
+ </template>
+ );
+ assert
+ .dom('.d-filter-controls__dropdown--outcome option[value="empty"]')
+ .hasText("Empty response", "empty completions can be filtered");
+ assert
+ .dom('.d-filter-controls__dropdown--outcome option[value="truncated"]')
+ .hasText("Truncated", "truncated completions can be filtered");
+ });
+});