Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Separate response and thinking budgets (#44513)

Sam5f275e7ed787
diff --git a/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb b/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb
index f5243d1c3d8..f85c432ed0b 100644
--- a/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb
+++ b/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb
@@ -51,7 +51,7 @@ module DiscourseAi
top_p: agent.top_p,
thinking_effort: agent.thinking_effort,
response_format: response_format(agent),
- max_tokens: 8000,
+ max_response_tokens: 8000,
)
result = JSON.parse(DiscourseAi::Completions::Llm.text_from_response(response))
ids = asks.map(&:first)
diff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rb
index 0fcff1d6941..5b6d654f815 100644
--- a/plugins/discourse-ai/lib/agents/bot.rb
+++ b/plugins/discourse-ai/lib/agents/bot.rb
@@ -24,6 +24,7 @@ module DiscourseAi
TOKEN_BUDGET_FINAL_ANSWER_HINT = <<~TEXT.strip
[The turn work allowance has been reached — no further tool calls are available.]
+ Answer directly from what you have; do not spend tokens deliberating at length.
Provide your final response now using the information already gathered.
Do not describe additional tool calls or future work as though you will perform them.
If the task is not fully complete, clearly state what was accomplished
@@ -31,15 +32,18 @@ module DiscourseAi
TEXT
TOOL_INVOCATION_BUDGET_FINAL_ANSWER_HINT = <<~TEXT.strip
[The tool invocation limit has been reached — no further tool calls are available.]
+ Answer directly from what you have; do not spend tokens deliberating at length.
Provide your final response now using the information already gathered.
If the evidence is insufficient, say so instead of attempting another tool call.
TEXT
COMPLETION_LIMIT_FINAL_ANSWER_HINT = <<~TEXT.strip
[The shared completion safety limit has been reached — no further tool calls are available.]
+ Answer directly from what you have; do not spend tokens deliberating at length.
Provide one final response using the information already gathered.
TEXT
LEGACY_BUDGET_EXHAUSTED_HINT = <<~TEXT.strip
[Turn budget exhausted — you cannot call any more tools.]
+ Answer directly from what you have; do not spend tokens deliberating at length.
Provide your final response using the information already gathered.
If the task is not fully complete, clearly state what was accomplished
and what still needs to be done so the user can continue in a follow-up message.
@@ -307,8 +311,14 @@ module DiscourseAi
root_generation = context.subagent_depth.to_i.zero?
generation_options =
work_budget.generation_options(
- llm_kwargs,
- maximum: llm_kwargs[:max_tokens] || model.max_output_tokens.presence || 2500,
+ llm_kwargs.merge(
+ thinking_allowance:
+ DiscourseAi::Completions::ThinkingConfig.response_allowance_for(
+ llm_model: model,
+ effort: llm_kwargs[:thinking_effort],
+ ),
+ ),
+ maximum: llm_kwargs[:max_tokens] || model.max_output_tokens.presence,
final: final_answer_requested,
tools: prompt.tools.present? && prompt.tool_choice != :none,
root: root_generation,
diff --git a/plugins/discourse-ai/lib/agents/tool_runner/llm.rb b/plugins/discourse-ai/lib/agents/tool_runner/llm.rb
index 9fe6445f37d..713144e6f50 100644
--- a/plugins/discourse-ai/lib/agents/tool_runner/llm.rb
+++ b/plugins/discourse-ai/lib/agents/tool_runner/llm.rb
@@ -35,6 +35,7 @@ module DiscourseAi
temperature: options["temperature"],
top_p: options["top_p"],
max_tokens: options["max_tokens"],
+ max_response_tokens: options["max_response_tokens"],
thinking_effort: options["thinking_effort"],
stop_sequences: options["stop_sequences"],
)
diff --git a/plugins/discourse-ai/lib/agents/tools/summarize.rb b/plugins/discourse-ai/lib/agents/tools/summarize.rb
index 09001a1ac16..a77db2e22e5 100644
--- a/plugins/discourse-ai/lib/agents/tools/summarize.rb
+++ b/plugins/discourse-ai/lib/agents/tools/summarize.rb
@@ -135,7 +135,7 @@ module DiscourseAi
DiscourseAi::Completions::Llm.text_from_response(
llm.generate(
prompt,
- max_tokens: 400,
+ max_response_tokens: 400,
user: bot_user,
feature_name: "summarize_tool",
execution_context: context.execution_context,
@@ -158,7 +158,7 @@ module DiscourseAi
result =
llm.generate(
concatenation_prompt,
- max_tokens: 500,
+ max_response_tokens: 500,
user: bot_user,
feature_name: "summarize_tool",
execution_context: context.execution_context,
diff --git a/plugins/discourse-ai/lib/agents/tools/view_image.rb b/plugins/discourse-ai/lib/agents/tools/view_image.rb
index 8476c7c74b4..7e82df255ae 100644
--- a/plugins/discourse-ai/lib/agents/tools/view_image.rb
+++ b/plugins/discourse-ai/lib/agents/tools/view_image.rb
@@ -101,7 +101,7 @@ module DiscourseAi
ai_agent_id: agent&.id,
)
.compact,
- max_tokens: MAX_OUTPUT_TOKENS,
+ max_response_tokens: MAX_OUTPUT_TOKENS,
cancel_manager: context.cancel_manager,
execution_context: context.execution_context,
),
diff --git a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
index 0f014073ee0..880fc3f6ec4 100644
--- a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
+++ b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
@@ -477,8 +477,14 @@ module DiscourseAi
def generate_with_work_budget!(execution_context, options, final: false, &block)
options =
@work_budget.generation_options(
- options,
- maximum: @bot.model.max_output_tokens.presence || 2500,
+ options.merge(
+ thinking_allowance:
+ DiscourseAi::Completions::ThinkingConfig.response_allowance_for(
+ llm_model: @bot.model,
+ effort: options[:thinking_effort],
+ ),
+ ),
+ maximum: @bot.model.max_output_tokens.presence,
final: final,
tools: @prompt.tools.present?,
root: true,
diff --git a/plugins/discourse-ai/lib/ai_helper/assistant.rb b/plugins/discourse-ai/lib/ai_helper/assistant.rb
index a4c7e5d326f..27af5cd8719 100644
--- a/plugins/discourse-ai/lib/ai_helper/assistant.rb
+++ b/plugins/discourse-ai/lib/ai_helper/assistant.rb
@@ -290,7 +290,7 @@ module DiscourseAi
end
end
- bot.reply(context, llm_args: { max_tokens: 1024 }, &buffer_blk)
+ bot.reply(context, llm_args: { max_response_tokens: 1024 }, &buffer_blk)
raw_caption = ""
diff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rb
index a07aa665892..f4713f2c7cb 100644
--- a/plugins/discourse-ai/lib/automation/llm_triage.rb
+++ b/plugins/discourse-ai/lib/automation/llm_triage.rb
@@ -157,7 +157,7 @@ module DiscourseAi
result = nil
llm_args = {
- max_tokens: max_output_tokens,
+ max_response_tokens: max_output_tokens,
stop_sequences: stop_sequences,
feature_context: {
automation_id: automation&.id,
diff --git a/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb b/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb
index 7b09df9af88..efd8f0f6926 100644
--- a/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb
@@ -61,7 +61,7 @@ module DiscourseAi
!legacy_reasoning_model? && llm_model.provider == "open_ai" &&
(
llm_model.name.start_with?("o1") || llm_model.name.start_with?("o3") ||
- llm_model.name.start_with?("gpt-5")
+ llm_model.name.start_with?("gpt-5") || llm_model.name.start_with?("gpt-6")
)
end
diff --git a/plugins/discourse-ai/lib/completions/endpoints/anthropic.rb b/plugins/discourse-ai/lib/completions/endpoints/anthropic.rb
index dba3477adab..9642fc41112 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/anthropic.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/anthropic.rb
@@ -12,36 +12,10 @@ module DiscourseAi
end
def default_options(dialect)
- mapped_model =
- case llm_model.name
- when "claude-2"
- "claude-2.1"
- when "claude-instant-1"
- "claude-instant-1.2"
- when "claude-3-haiku"
- "claude-3-haiku-20240307"
- when "claude-3-sonnet"
- "claude-3-sonnet-20240229"
- when "claude-3-opus"
- "claude-3-opus-20240229"
- when "claude-3-5-sonnet"
- "claude-3-5-sonnet-latest"
- when "claude-3-7-sonnet"
- "claude-3-7-sonnet-latest"
- when "claude-4-opus"
- "claude-4-opus-20250514"
- when "claude-4-sonnet"
- "claude-4-sonnet-20250514"
- else
- llm_model.name
- end
+ mapped_model = canonical_claude_model_name
# Note: Anthropic requires this param
- max_tokens = 4096
- # 3.5 and 3.7 models have a higher token limit
- max_tokens = 8192 if mapped_model.match?(/3.[57]/)
-
- options = { model: mapped_model, max_tokens: max_tokens }
+ options = { model: mapped_model, max_tokens: effective_max_output_tokens }
apply_anthropic_thinking_config!(options)
apply_anthropic_effort_config!(options)
diff --git a/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb b/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb
index c7532ebf58d..781c47c548c 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/anthropic_shared.rb
@@ -13,10 +13,36 @@ module DiscourseAi
"xhigh" => 32_768,
"max" => 32_768,
}.freeze
- DEFAULT_VISIBLE_OUTPUT_TOKENS = 30_000
- DEFAULT_ADAPTIVE_OUTPUT_TOKENS = 32_000
MIN_THINKING_BUDGET = 1024
MIN_VISIBLE_OUTPUT_TOKENS = 1024
+ # Models we cannot identify get a generous assumed cap so an
+ # undersized guess surfaces as an error when an admin tests the
+ # model, instead of silently truncating every response.
+ UNKNOWN_MODEL_MAX_OUTPUT_TOKENS = 64_000
+ # Friendly or legacy names mapped to the canonical Anthropic API
+ # model name. Dated canonical names pass through unchanged.
+ CLAUDE_MODEL_ALIASES = {
+ "claude-2" => "claude-2.1",
+ "claude-instant-1" => "claude-instant-1.2",
+ "claude-instant" => "claude-instant-1.2",
+ "claude-3-haiku" => "claude-3-haiku-20240307",
+ "claude-3-sonnet" => "claude-3-sonnet-20240229",
+ "claude-3-opus" => "claude-3-opus-20240229",
+ "claude-3-5-sonnet" => "claude-3-5-sonnet-20241022",
+ "claude-3-5-sonnet-latest" => "claude-3-5-sonnet-20241022",
+ "claude-3-5-haiku" => "claude-3-5-haiku-20241022",
+ "claude-3-5-haiku-latest" => "claude-3-5-haiku-20241022",
+ "claude-3-7-sonnet" => "claude-3-7-sonnet-20250219",
+ "claude-3-7-sonnet-latest" => "claude-3-7-sonnet-20250219",
+ "claude-4-opus" => "claude-opus-4-20250514",
+ "claude-4-sonnet" => "claude-sonnet-4-20250514",
+ "claude-opus-4" => "claude-opus-4-20250514",
+ "claude-sonnet-4" => "claude-sonnet-4-20250514",
+ "claude-opus-4-1" => "claude-opus-4-1-20250805",
+ "claude-opus-4-5" => "claude-opus-4-5-20251101",
+ "claude-sonnet-4-5" => "claude-sonnet-4-5-20250929",
+ "claude-haiku-4-5" => "claude-haiku-4-5-20251001",
+ }.freeze
def normalize_model_params(model_params)
model_params = model_params.dup
@@ -153,28 +179,116 @@ module DiscourseAi
end
end
+ # Canonical Anthropic API name for the configured model, accepting
+ # friendly aliases and Bedrock-style IDs (profile prefixes and
+ # version suffixes are stripped).
+ def canonical_claude_model_name
+ raw = llm_model.name.to_s.strip
+ return "claude-2.1" if raw.match?(/claude-v2(?::1)?\z/i)
+ return "claude-instant-1.2" if raw.match?(/claude-instant-v1\z/i)
+
+ stripped = raw.sub(/\A(?:[a-z]{2,}\.)?anthropic\./i, "").sub(/-v\d+(?::\d+)?\z/i, "")
+
+ CLAUDE_MODEL_ALIASES[stripped.downcase] || stripped
+ end
+
+ # Bedrock model ID for the configured model. Already-qualified
+ # Bedrock IDs, inference profile IDs and ARNs pass through
+ # unchanged, as do non-Claude models.
+ def bedrock_model_id
+ raw = llm_model.name.to_s.strip
+ return raw if raw.start_with?("arn:") || raw.match?(/\A(?:[a-z]{2,}\.)?anthropic\./i)
+
+ canonical = canonical_claude_model_name
+
+ case canonical
+ when "claude-2.1"
+ "anthropic.claude-v2:1"
+ when "claude-instant-1.2"
+ "anthropic.claude-instant-v1"
+ when "claude-3-5-sonnet-20241022"
+ "anthropic.claude-3-5-sonnet-20241022-v2:0"
+ when "claude-opus-4-6"
+ "anthropic.claude-opus-4-6-v1"
+ when /\Aclaude-.+-\d{8}\z/
+ "anthropic.#{canonical}-v1:0"
+ when /\Aclaude-/
+ "anthropic.#{canonical}"
+ else
+ raw
+ end
+ end
+
+ def bedrock_provider?
+ %w[aws_bedrock aws_bedrock_converse].include?(llm_model.provider)
+ end
+
+ # A configured limit always wins. When it is missing we fall back to
+ # the known cap for the model, and to a generous assumed cap for
+ # models we do not recognise.
+ def effective_max_output_tokens
+ configured = llm_model.max_output_tokens.to_i
+ return configured if configured.positive?
+
+ known_model_max_output_tokens || UNKNOWN_MODEL_MAX_OUTPUT_TOKENS
+ end
+
+ def enforce_max_output_tokens(value)
+ return super if !supports_anthropic_thinking?
+
+ limit = effective_max_output_tokens
+ value = limit if (value.to_i > limit) || (value.to_i <= 0)
+ value
+ end
+
+ def known_model_max_output_tokens
+ name = canonical_claude_model_name.to_s.downcase
+
+ case name
+ when /claude-(?:fable|mythos|opus|sonnet|haiku)-5(?:-|\z)/,
+ /claude-opus-4-(?:[6-9]|\d{2,})(?:-|\z)/
+ 128_000
+ when /claude-sonnet-4-6(?:-|\z)/
+ # Bedrock caps Sonnet 4.6 at 64k; the Anthropic API allows 128k.
+ bedrock_provider? ? 64_000 : 128_000
+ when /claude-(?:opus|sonnet|haiku)-4-5(?:-|\z)/, /claude-sonnet-4(?:-|\z)/,
+ /claude-4-sonnet(?:-|\z)/, /claude-3-7(?:-|\z)/
+ 64_000
+ when /claude-opus-4-1(?:-|\z)/, /claude-opus-4(?:-|\z)/, /claude-4-opus(?:-|\z)/
+ 32_000
+ when /claude-3-5(?:-|\z)/
+ 8_192
+ when /claude-3(?:-|\z)/, /claude-(?:v2|2)(?:[-.:]|\z)/, /claude-instant/
+ 4_096
+ end
+ end
+
+ # Prompt admission must reserve what this endpoint can actually
+ # send: the explicit ceiling when given, otherwise the effective
+ # cap. Thinking configs already reserve their provider total.
+ def apply_thinking_config_to_model_params(model_params)
+ result = super
+ if supports_anthropic_thinking? && result[:reserved_output_tokens].blank?
+ result[:reserved_output_tokens] = model_params[:max_tokens].presence ||
+ effective_max_output_tokens
+ end
+ result
+ end
+
def adaptive_total_output_tokens(model_params)
requested_output_tokens = model_params[:max_tokens].presence&.to_i
- output_token_limit = llm_model.max_output_tokens.to_i
+ output_token_limit = effective_max_output_tokens
- if output_token_limit.positive?
- if requested_output_tokens&.positive?
- [requested_output_tokens, output_token_limit].min
- else
- output_token_limit
- end
+ if requested_output_tokens&.positive?
+ [requested_output_tokens, output_token_limit].min
else
- if requested_output_tokens&.positive?
- requested_output_tokens
- else
- DEFAULT_ADAPTIVE_OUTPUT_TOKENS
- end
+ output_token_limit
end
end
def budget_thinking_config(canonical_effort:, budget:, model_params:)
requested_visible_output_tokens = model_params[:max_tokens].presence&.to_i
- output_token_limit = llm_model.max_output_tokens.to_i
+ output_token_limit = effective_max_output_tokens
if model_params[:max_tokens_is_total] && requested_visible_output_tokens&.positive?
provider_output_tokens = requested_visible_output_tokens
@@ -193,22 +307,26 @@ module DiscourseAi
budget = [budget, provider_output_tokens - visible_floor].min
visible_output_tokens = provider_output_tokens - budget
elsif output_token_limit.positive?
- provider_output_tokens = output_token_limit
- return if provider_output_tokens <= MIN_THINKING_BUDGET
+ return if output_token_limit <= MIN_THINKING_BUDGET
- max_visible_output_tokens = provider_output_tokens - MIN_THINKING_BUDGET
+ max_visible_output_tokens = output_token_limit - MIN_THINKING_BUDGET
visible_output_floor = [MIN_VISIBLE_OUTPUT_TOKENS, max_visible_output_tokens].min
- visible_output_tokens = provider_output_tokens - budget
+ # The requested thinking budget stays authoritative: visible
+ # output and thinking together form the provider total, shrunk
+ # only when that total would exceed the effective cap.
visible_output_tokens =
- requested_visible_output_tokens if requested_visible_output_tokens&.positive?
+ if requested_visible_output_tokens&.positive?
+ requested_visible_output_tokens
+ else
+ output_token_limit - budget
+ end
visible_output_tokens = [visible_output_tokens, visible_output_floor].max
visible_output_tokens = [visible_output_tokens, max_visible_output_tokens].min
- budget = provider_output_tokens - visible_output_tokens
+ budget = [budget, output_token_limit - visible_output_tokens].min
return if budget < MIN_THINKING_BUDGET
- else
- visible_output_tokens = requested_visible_output_tokens || DEFAULT_VISIBLE_OUTPUT_TOKENS
+
provider_output_tokens = visible_output_tokens + budget
end
diff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb
index 1cfc04ca757..5cd2b64921e 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock.rb
@@ -21,10 +21,10 @@ module DiscourseAi
def default_options(dialect)
options =
if dialect.is_a?(DiscourseAi::Completions::Dialects::Claude)
- max_tokens = 4096
- max_tokens = 8192 if bedrock_model_id.match?(/3.[57]/)
-
- result = { anthropic_version: "bedrock-2023-05-31", max_tokens: max_tokens }
+ result = {
+ anthropic_version: "bedrock-2023-05-31",
+ max_tokens: effective_max_output_tokens,
+ }
apply_anthropic_thinking_config!(result)
apply_anthropic_effort_config!(result)
@@ -44,45 +44,6 @@ module DiscourseAi
bedrock_model_id.include?("anthropic") || bedrock_model_id.include?("claude")
end
- def bedrock_model_id
- case llm_model.name
- when "claude-2"
- "anthropic.claude-v2:1"
- when "claude-3-haiku", "claude-3-haiku-20240307"
- "anthropic.claude-3-haiku-20240307-v1:0"
- when "claude-3-sonnet"
- "anthropic.claude-3-sonnet-20240229-v1:0"
- when "claude-instant-1"
- "anthropic.claude-instant-v1"
- when "claude-3-opus"
- "anthropic.claude-3-opus-20240229-v1:0"
- when "claude-3-5-sonnet", "claude-3-5-sonnet-20241022", "claude-3-5-sonnet-latest"
- "anthropic.claude-3-5-sonnet-20241022-v2:0"
- when "claude-3-5-sonnet-20240620"
- "anthropic.claude-3-5-sonnet-20240620-v1:0"
- when "claude-3-5-haiku", "claude-3-5-haiku-20241022", "claude-3-5-haiku-latest"
- "anthropic.claude-3-5-haiku-20241022-v1:0"
- when "claude-3-7-sonnet", "claude-3-7-sonnet-20250219", "claude-3-7-sonnet-latest"
- "anthropic.claude-3-7-sonnet-20250219-v1:0"
- when "claude-opus-4-1", "claude-opus-4-1-20250805"
- "anthropic.claude-opus-4-1-20250805-v1:0"
- when "claude-opus-4", "claude-opus-4-20250514"
- "anthropic.claude-opus-4-20250514-v1:0"
- when "claude-sonnet-4", "claude-sonnet-4-20250514"
- "anthropic.claude-sonnet-4-20250514-v1:0"
- when "claude-opus-4-5"
- "anthropic.claude-opus-4-5-20250918-v1:0"
- when "claude-sonnet-4-5"
- "anthropic.claude-sonnet-4-5-20250514-v1:0"
- when "claude-opus-4-6"
- "anthropic.claude-opus-4-6-v1"
- when "claude-sonnet-4-6"
- "anthropic.claude-sonnet-4-6-v1"
- else
- llm_model.name
- end
- end
-
def model_uri
region = llm_model.lookup_custom_param("region")
inference_profile_arn = llm_model.lookup_custom_param("inference_profile_arn")
diff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
index aa61405d32d..113feff2171 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
@@ -221,11 +221,16 @@ module DiscourseAi
processor.cache_read_input_tokens if processor.cache_read_input_tokens
log.cache_write_tokens =
processor.cache_write_input_tokens if processor.cache_write_input_tokens
- update_completion_state(log, call_status, cancelled: cancelled)
+ update_completion_state(
+ log,
+ call_status,
+ cancelled: cancelled,
+ response_limit_tripped: !!execution_context&.response_limit_tripped,
+ )
execution_context&.settle_generation(
tokens: log.response_tokens,
tokenizer: tokenizer,
- complete: call_status == :success,
+ complete: call_status == :success && !cancelled,
)
log.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?
log.response_status ||= 200 if call_status == :success
@@ -235,6 +240,7 @@ module DiscourseAi
log.duration_msecs = (Time.now - start_time) * 1000
log.time_to_first_token_msecs = time_to_first_token_msecs
log.save!
+ execution_context.audit_log_id = log.id if execution_context
execution_context&.token_usage_tracker&.add_from_audit_log(
log,
@@ -284,8 +290,11 @@ module DiscourseAi
def default_options(dialect)
options = {}
- apply_anthropic_thinking_config!(options) if supports_anthropic_thinking?
- apply_anthropic_effort_config!(options) if supports_anthropic_thinking?
+ if supports_anthropic_thinking?
+ options[:max_tokens] = effective_max_output_tokens
+ apply_anthropic_thinking_config!(options)
+ apply_anthropic_effort_config!(options)
+ end
options
end
@@ -346,7 +355,7 @@ module DiscourseAi
options = default_options(dialect).merge(model_params.except(:response_format))
apply_anthropic_thinking_config!(options)
- params = { model_id: llm_model.name, messages: prompt.messages }
+ params = { model_id: bedrock_model_id, messages: prompt.messages }
params[:system] = prompt.system if prompt.system.present?
diff --git a/plugins/discourse-ai/lib/completions/endpoints/base.rb b/plugins/discourse-ai/lib/completions/endpoints/base.rb
index dff333a29f1..f227d50992b 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/base.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/base.rb
@@ -301,9 +301,23 @@ module DiscourseAi
end
end
- def update_completion_state(log, call_status, cancelled: false)
- log.stop_reason = output_stop_reason&.to_s&.slice(0, 50).presence
+ def update_completion_state(
+ log,
+ call_status,
+ cancelled: false,
+ response_limit_tripped: false
+ )
log.reasoning_tokens = reported_reasoning_tokens
+
+ # A response-limit cut is our own doing: the reply was trimmed to
+ # the caller's visible-response cap and generation aborted.
+ if response_limit_tripped
+ log.stop_reason = "max_response_tokens"
+ log.completion_state = AiApiAuditLog::CompletionState::TRUNCATED if @visible_output
+ return
+ end
+
+ log.stop_reason = output_stop_reason&.to_s&.slice(0, 50).presence
# Cancellation is a user action, not an incomplete completion.
return if cancelled || call_status != :success
@@ -924,11 +938,16 @@ module DiscourseAi
log.raw_response_payload = response_raw
log.request_attempts = request_attempts if log.has_attribute?(:request_attempts)
final_log_update(log)
- update_completion_state(log, call_status, cancelled: cancelled)
+ update_completion_state(
+ log,
+ call_status,
+ cancelled: cancelled,
+ response_limit_tripped: !!execution_context&.response_limit_tripped,
+ )
execution_context&.settle_generation(
tokens: log.response_tokens,
tokenizer: tokenizer,
- complete: call_status == :success,
+ complete: call_status == :success && !cancelled,
)
log.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?
log.response_status ||= 200 if call_status == :success
@@ -938,6 +957,7 @@ module DiscourseAi
log.duration_msecs = (Time.now - start_time) * 1000
log.time_to_first_token_msecs = time_to_first_token_msecs
log.save!
+ execution_context.audit_log_id = log.id if execution_context
execution_context&.token_usage_tracker&.add_from_audit_log(
log,
diff --git a/plugins/discourse-ai/lib/completions/endpoints/hugging_face.rb b/plugins/discourse-ai/lib/completions/endpoints/hugging_face.rb
index b44b21add53..803db24eddf 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/hugging_face.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/hugging_face.rb
@@ -39,9 +39,12 @@ module DiscourseAi
.merge(messages: prompt)
.tap do |payload|
if !payload[:max_tokens]
+ # Unlike most providers, HF/TGI defaults an omitted limit
+ # to 100 tokens, so "no limit" is expressed explicitly as
+ # whatever the context window has left.
token_limit = llm_model.max_prompt_tokens
- payload[:max_tokens] = token_limit - prompt_size(prompt)
+ payload[:max_tokens] = [token_limit - prompt_size(prompt), 1].max
end
payload[:stream] = true if @streaming_mode
diff --git a/plugins/discourse-ai/lib/completions/endpoints/ollama.rb b/plugins/discourse-ai/lib/completions/endpoints/ollama.rb
index 768cb6a461b..96949507c82 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/ollama.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/ollama.rb
@@ -20,7 +20,7 @@ module DiscourseAi
end
def default_options
- { max_tokens: 2000, model: llm_model.name }
+ { model: llm_model.name }
end
def provider_id
diff --git a/plugins/discourse-ai/lib/completions/endpoints/open_ai.rb b/plugins/discourse-ai/lib/completions/endpoints/open_ai.rb
index f747a9e2837..5e3ab0e2ee2 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/open_ai.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/open_ai.rb
@@ -17,7 +17,7 @@ module DiscourseAi
# max_tokens is deprecated however we still need to support it
# on older OpenAI models and older Azure models, so we will only normalize
# if our model name starts with o (to denote all the reasoning models)
- if llm_model.name.starts_with?(/o|gpt-5/)
+ if llm_model.name.starts_with?(/o|gpt-[56]/)
max_tokens = normalized.delete(:max_tokens)
normalized[:max_completion_tokens] = max_tokens if max_tokens
end
diff --git a/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb b/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb
index 2a90d2fb3fd..799dfb8d90b 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/open_ai_shared.rb
@@ -20,6 +20,9 @@ module DiscourseAi
gpt-5.5
gpt-5.5-pro
].freeze
+ # GPT-6 Astra and GPT-6.1 Sol reject none/minimal; low is the lowest
+ # effort they support.
+ OPEN_AI_LOW_EFFORT_FLOOR_MODELS = %w[gpt-6-astra gpt-6.1-sol].freeze
OPEN_AI_MODEL_SNAPSHOT_SUFFIX = /-\d{4}-\d{2}-\d{2}\z/
OPEN_AI_REASONING_OUTPUT_RESERVATION = 25_000
# "minimal" isn't a distinct tier on every OpenAI reasoning model and can
@@ -103,6 +106,7 @@ module DiscourseAi
provider_effort = OPEN_AI_EFFORT_ALIASES[effort] || effort
provider_effort = "xhigh" if provider_effort == "max" && max_effort_maps_to_xhigh?
+ provider_effort = "low" if provider_effort == "none" && low_effort_floor_model?
if !OPEN_AI_REASONING_EFFORTS.include?(provider_effort)
return DiscourseAi::Completions::ThinkingConfig.unsupported(canonical_effort: effort)
end
@@ -162,8 +166,15 @@ module DiscourseAi
end
def max_effort_maps_to_xhigh?
- model_name = llm_model.name.to_s.sub(OPEN_AI_MODEL_SNAPSHOT_SUFFIX, "")
- OPEN_AI_MAX_TO_XHIGH_MODELS.include?(model_name)
+ OPEN_AI_MAX_TO_XHIGH_MODELS.include?(normalized_model_name)
+ end
+
+ def low_effort_floor_model?
+ OPEN_AI_LOW_EFFORT_FLOOR_MODELS.include?(normalized_model_name)
+ end
+
+ def normalized_model_name
+ llm_model.name.to_s.sub(OPEN_AI_MODEL_SNAPSHOT_SUFFIX, "")
end
def legacy_reasoning_effort
diff --git a/plugins/discourse-ai/lib/completions/execution_context.rb b/plugins/discourse-ai/lib/completions/execution_context.rb
index a0efd0f3ed5..373d3f67653 100644
--- a/plugins/discourse-ai/lib/completions/execution_context.rb
+++ b/plugins/discourse-ai/lib/completions/execution_context.rb
@@ -9,7 +9,9 @@ module DiscourseAi
:generated_output,
:generation_event_id,
:generation_settled,
- :root_reply_completed
+ :root_reply_completed,
+ :response_limit_tripped,
+ :audit_log_id
attr_reader :audit_logger, :structured_audit_logger, :upload_skips
def initialize(
diff --git a/plugins/discourse-ai/lib/completions/llm.rb b/plugins/discourse-ai/lib/completions/llm.rb
index 1392d56d0f6..df50c501e81 100644
--- a/plugins/discourse-ai/lib/completions/llm.rb
+++ b/plugins/discourse-ai/lib/completions/llm.rb
@@ -161,6 +161,10 @@ module DiscourseAi
# @param temperature { Float - Optional } - The temperature to use for the completion.
# @param top_p { Float - Optional } - The top_p to use for the completion.
# @param max_tokens { Integer - Optional } - The maximum number of tokens to generate.
+ # @param max_response_tokens { Integer - Optional } - Cap on visible response (text) tokens.
+ # Unlike max_tokens this expresses caller intent for the reply itself: thinking tokens do
+ # not count against it, streamed text is trimmed at the cap, and generation is aborted
+ # once the cap is reached. Tool calls and structured output are exempt.
# @param stop_sequences { Array<String> - Optional } - The stop sequences to use for the completion.
# @param feature_name { String - Optional } - The feature name to use for the completion.
# @param feature_context { Hash - Optional } - The feature context to use for the completion.
@@ -183,6 +187,7 @@ module DiscourseAi
temperature: nil,
top_p: nil,
max_tokens: nil,
+ max_response_tokens: nil,
stop_sequences: nil,
user:,
feature_name: nil,
@@ -198,12 +203,16 @@ module DiscourseAi
max_tokens_is_total: false,
&partial_read_blk
)
+ response_limit = max_response_tokens&.to_i
+ response_limit = nil if response_limit && !response_limit.positive?
+
self.class.record_prompt(
prompt,
{
temperature: temperature,
top_p: top_p,
max_tokens: max_tokens,
+ max_response_tokens: response_limit,
stop_sequences: stop_sequences,
user: user,
feature_name: feature_name,
@@ -259,10 +268,18 @@ module DiscourseAi
work = execution_context.work_budget
helper_options =
work.generation_options(
- { thinking_effort: thinking_effort, response_format: response_format },
- maximum: max_tokens || llm_model.max_output_tokens.presence || 2500,
+ {
+ thinking_effort: thinking_effort,
+ response_format: response_format,
+ max_response_tokens: response_limit,
+ thinking_allowance: thinking_response_allowance(thinking_effort),
+ },
+ # Helper generations borrow against the root work budget, so
+ # their admission ceiling is whatever work remains when no
+ # explicit limit was given.
+ maximum: max_tokens || llm_model.max_output_tokens.presence || work.remaining,
)
- if helper_options[:max_tokens] <= 0
+ if helper_options[:max_tokens] && helper_options[:max_tokens] <= 0
raise ContextPreparation::Error.new("turn_budget_exhausted")
end
_, _, provider_output = prompt_capacity(prompt, **helper_options)
@@ -271,8 +288,20 @@ module DiscourseAi
if !execution_context.work_execution_state.reserve_completion
raise ContextPreparation::Error.new("completion_limit")
end
- model_params[:max_tokens] = helper_options[:max_tokens]
+ model_params[:max_tokens] = helper_options[:max_tokens] if helper_options[:max_tokens]
+ end
+
+ # Express the visible-response intent in provider terms when the
+ # caller did not set a provider ceiling: reserve room for thinking
+ # on top of the response cap. An explicit or work-budget ceiling
+ # stays authoritative; enforcement below still trims the reply.
+ if response_limit && model_params[:max_tokens].blank?
+ model_params[:max_tokens] = response_limit + thinking_response_allowance(thinking_effort)
+ # The derived ceiling already includes thinking room; providers
+ # with a visible/thinking split must treat it as the total.
+ model_params[:max_tokens_is_total] = true
end
+
model_params = gateway.prepare_model_params(model_params) if gateway.respond_to?(
:prepare_model_params,
)
@@ -291,6 +320,9 @@ module DiscourseAi
end
call_context = execution_context&.dup
+ # A response limit needs a context to carry the limiter flag to the
+ # endpoint's audit logging even when the caller supplied none.
+ call_context ||= ExecutionContext.new if response_limit
if call_context && feature_name != "context_compression"
call_context.generated_output = GeneratedOutput.new
call_context.generation_event_id = "generation:#{SecureRandom.uuid}"
@@ -298,11 +330,54 @@ module DiscourseAi
elsif call_context
call_context.generated_output = nil
end
+ # Visible-response enforcement: count text partials, trim the one
+ # that crosses the cap, then abort generation through an internal
+ # cancel manager so a caller's own manager is never tripped.
+ response_limiter = nil
+ limiter_cancel_manager = nil
+ user_cancel_callback = nil
+ response_limit_tripped = false
+ effective_cancel_manager = cancel_manager
+
+ if response_limit && partial_read_blk
+ limiter_cancel_manager = CancelManager.new
+ limiter_cancel_manager.cancel! if cancel_manager&.cancelled?
+ if cancel_manager
+ user_cancel_callback = -> { limiter_cancel_manager.cancel! }
+ cancel_manager.add_callback(user_cancel_callback)
+ end
+ effective_cancel_manager = limiter_cancel_manager
+
+ # Count the accumulated text, not individual deltas: tokenisation
+ # is not additive across arbitrary stream chunk boundaries.
+ visible_text = +""
+ response_limiter =
+ lambda do |partial|
+ next partial if !partial.is_a?(String)
+ next nil if response_limit_tripped
+
+ candidate = visible_text + partial
+ if tokenizer.size(candidate) <= response_limit
+ visible_text.replace(candidate)
+ partial
+ else
+ trimmed = truncate_response_text(candidate, response_limit)
+ forwarded = trimmed[visible_text.length..] || +""
+ visible_text.replace(trimmed)
+ response_limit_tripped = true
+ call_context.response_limit_tripped = true if call_context
+ limiter_cancel_manager.cancel!
+ forwarded.empty? ? nil : forwarded
+ end
+ end
+ end
+
read_block =
if partial_read_blk
proc do |partial, *args|
call_context&.generated_output&.<<(partial)
- partial_read_blk.call(partial, *args)
+ partial = response_limiter.call(partial) if response_limiter
+ partial_read_blk.call(partial, *args) if !partial.nil?
end
end
@@ -315,13 +390,38 @@ module DiscourseAi
feature_context: feature_context,
partial_tool_calls: partial_tool_calls,
output_thinking: output_thinking,
- cancel_manager: cancel_manager,
+ cancel_manager: effective_cancel_manager,
execution_context: call_context,
&read_block
)
Array(result).each { |part| call_context&.generated_output&.<<(part) } if !partial_read_blk
+
+ # Structured output is exempt: reclassifying its serialized form as
+ # plain text could chop JSON the caller already received intact.
+ result_truncated = false
+ if response_limit && !response_format
+ result_truncated = response_result_over_limit?(result, response_limit)
+ result = truncate_response_result(result, response_limit) if result_truncated
+ end
+
+ # The endpoint persists its audit log before non-streamed (or
+ # replayed) output reaches the limiter, so reconcile the exact log
+ # this call created (handed back through the execution context).
+ if response_limit && (response_limit_tripped || result_truncated) &&
+ call_context&.audit_log_id
+ log = AiApiAuditLog.find_by(id: call_context.audit_log_id)
+ if log
+ log.completion_state = AiApiAuditLog::CompletionState::TRUNCATED
+ log.stop_reason = "max_response_tokens"
+ log.save!
+ end
+ end
+
result
ensure
+ if user_cancel_callback && cancel_manager
+ cancel_manager.remove_callback(user_cancel_callback)
+ end
call_context&.settle_generation(tokenizer: tokenizer)
execution_context&.work_budget&.release_output(generation_reservation)
end
@@ -490,6 +590,62 @@ module DiscourseAi
private
+ def thinking_response_allowance(thinking_effort)
+ ThinkingConfig.response_allowance_for(llm_model: llm_model, effort: thinking_effort)
+ end
+
+ def truncate_response_text(text, tokens)
+ return +"" if tokens <= 0
+ return text if tokenizer.size(text) <= tokens
+
+ tokenizer.truncate(text, tokens, strict: SiteSetting.ai_strict_token_counting)
+ end
+
+ def response_result_over_limit?(result, limit)
+ text =
+ case result
+ when String
+ result
+ when Array
+ result.select { |part| part.is_a?(String) }.join
+ else
+ return false
+ end
+
+ tokenizer.size(text) > limit
+ end
+
+ def truncate_response_result(result, limit)
+ case result
+ when String
+ truncate_response_text(result, limit)
+ when Array
+ visible_text = +""
+ tripped = false
+ result.filter_map do |part|
+ if part.is_a?(String)
+ next nil if tripped
+
+ candidate = visible_text + part
+ if tokenizer.size(candidate) <= limit
+ visible_text.replace(candidate)
+ part
+ else
+ tripped = true
+ trimmed = truncate_response_text(candidate, limit)
+ forwarded = trimmed[visible_text.length..] || +""
+ visible_text.replace(trimmed)
+ forwarded.empty? ? nil : forwarded
+ end
+ else
+ part
+ end
+ end
+ else
+ result
+ end
+ end
+
def document_capacity_tokens(upload)
if upload[:mime_type] != "application/pdf"
raise ContextPreparation::Error.new("unsupported_document_type")
diff --git a/plugins/discourse-ai/lib/completions/thinking_config.rb b/plugins/discourse-ai/lib/completions/thinking_config.rb
index 57723591a14..3dfca2bf0ec 100644
--- a/plugins/discourse-ai/lib/completions/thinking_config.rb
+++ b/plugins/discourse-ai/lib/completions/thinking_config.rb
@@ -13,6 +13,45 @@ module DiscourseAi
:provider_output_tokens,
:reserved_output_tokens
+ # Rough provider-agnostic thinking allowances (mirrors the Anthropic
+ # effort budgets) used to reserve thinking room on top of a visible
+ # response cap when translating it to a provider ceiling. "none" still
+ # reserves room: some models floor at low and think anyway.
+ RESPONSE_ALLOWANCES = {
+ "none" => 4096,
+ "minimal" => 1024,
+ "low" => 4096,
+ "medium" => 8192,
+ "high" => 16_384,
+ "xhigh" => 32_768,
+ "max" => 32_768,
+ }.freeze
+
+ def self.response_allowance(effort)
+ RESPONSE_ALLOWANCES.fetch(normalize_effort(effort).to_s, 0)
+ end
+
+ # Allowance for a call on a specific model: the per-call effort when
+ # given, otherwise reasoning configured in the model's provider params.
+ def self.response_allowance_for(llm_model:, effort:)
+ return response_allowance(effort) if normalize_effort(effort)
+ return 0 if llm_model.nil?
+
+ configured_tokens = llm_model.lookup_custom_param("reasoning_tokens").to_i
+ return configured_tokens.clamp(1024, 32_768) if configured_tokens.positive?
+
+ allowance = response_allowance(llm_model.lookup_custom_param("reasoning_effort"))
+ return allowance if allowance.positive?
+
+ if llm_model.lookup_custom_param("enable_reasoning") ||
+ llm_model.lookup_custom_param("adaptive_thinking") ||
+ llm_model.lookup_custom_param("reasoning_mode")
+ response_allowance("medium")
+ else
+ 0
+ end
+ end
+
def self.normalize_effort(value)
value = value.to_s.strip
return if value.blank? || value == "default"
diff --git a/plugins/discourse-ai/lib/completions/turn_work_budget.rb b/plugins/discourse-ai/lib/completions/turn_work_budget.rb
index 33f7b1f2944..c319a72b75f 100644
--- a/plugins/discourse-ai/lib/completions/turn_work_budget.rb
+++ b/plugins/discourse-ai/lib/completions/turn_work_budget.rb
@@ -5,6 +5,11 @@ module DiscourseAi
# Work is new generation/evidence, never request usage or a monetary estimate.
class TurnWorkBudget
MAX_FINAL_ANSWER_TOKENS = 2048
+ # Thinking room on top of the visible final-answer cap, so a model
+ # that cannot disable thinking does not spend the answer budget
+ # deliberating. The final provider ceiling is the sum of the two.
+ MAX_FINAL_THINKING_ALLOWANCE = 2048
+ PLANNING_RESERVE_TOKENS = 2500
Reservation = Struct.new(:tokens, :owner, :released)
private_constant :Reservation
@@ -49,22 +54,71 @@ module DiscourseAi
end
def generation_options(options, maximum:, final: false, tools: false, root: false)
- available = remaining
- maximum = [maximum, MAX_FINAL_ANSWER_TOKENS].min if final
- maximum = [maximum, available].min if !final || !root
- maximum = [maximum, [available / 2, 1].max].min if !final && tools
- resolved = options.merge(max_tokens: maximum, max_tokens_is_total: true)
- resolved[:thinking_effort] = "none" if final
+ resolved = options.dup
+ thinking_allowance = resolved.delete(:thinking_allowance)
+
+ # Finalization is the bot's own bounded policy ("answer now"): the
+ # visible answer gets MAX_FINAL_ANSWER_TOKENS, with a separate
+ # thinking allowance on top bounding the provider ceiling.
+ if final
+ total_ceiling = MAX_FINAL_ANSWER_TOKENS + MAX_FINAL_THINKING_ALLOWANCE
+ maximum = [maximum || total_ceiling, total_ceiling].min
+ end
+
+ # A visible-response intent is not a provider ceiling: derive the
+ # provider total (visible + thinking allowance) and let any genuine
+ # provider ceiling (explicit max_tokens or model cap) bound it.
+ if !final && options[:max_response_tokens].to_i.positive?
+ derived =
+ options[:max_response_tokens].to_i +
+ (thinking_allowance || ThinkingConfig.response_allowance(options[:thinking_effort]))
+ maximum = maximum ? [maximum, derived].min : derived
+ end
+
+ # Nothing configured means no provider ceiling is sent at all; the
+ # budget still accounts for actual usage after each generation.
+ if maximum
+ available = remaining
+ maximum = [maximum, available].min if !final || !root
+ maximum = [maximum, [available / 2, 1].max].min if !final && tools
+ resolved = resolved.merge(max_tokens: maximum, max_tokens_is_total: true)
+ else
+ resolved.delete(:max_tokens)
+ resolved.delete(:max_tokens_is_total)
+ end
+
+ if final
+ existing_response = resolved[:max_response_tokens].to_i
+ resolved[:max_response_tokens] = [
+ existing_response.positive? ? existing_response : MAX_FINAL_ANSWER_TOKENS,
+ MAX_FINAL_ANSWER_TOKENS,
+ maximum,
+ ].min
+ resolved[:thinking_effort] = "none"
+ end
+
resolved
end
# Capacity/history checks must precede the root-final claim; a duplicate claim returns nil.
def reserve_generation_output(provider_output, max_tokens:, final: false, root: false)
- maximum = [provider_output.to_i, max_tokens].max
- if final && maximum > MAX_FINAL_ANSWER_TOKENS
+ # With no provider ceiling, the work reservation is a planning
+ # estimate only (context admission uses the provider output
+ # separately); it must not consume the whole remaining budget and
+ # starve helpers invoked while this generation is in flight.
+ maximum =
+ if max_tokens.nil?
+ [provider_output.to_i, PLANNING_RESERVE_TOKENS].min
+ else
+ [provider_output.to_i, max_tokens.to_i].max
+ end
+ if final && max_tokens && provider_output.to_i > max_tokens.to_i
raise ContextPreparation::Error.new("final_output_limit")
end
- reservation = reserve_output(maximum, final: final && root, allow_overshoot: root)
+ # With no provider ceiling the generation is accounted post-hoc, so
+ # admission soft-reserves what work remains instead of rejecting.
+ reservation =
+ reserve_output(maximum, final: final && root, allow_overshoot: root || max_tokens.nil?)
raise ContextPreparation::Error.new("turn_budget_exhausted") if !reservation
if final && root && !claim_final_answer
release_output(reservation)
diff --git a/plugins/discourse-ai/lib/summarization/fold_content.rb b/plugins/discourse-ai/lib/summarization/fold_content.rb
index 6d1992d2253..318c552227b 100644
--- a/plugins/discourse-ai/lib/summarization/fold_content.rb
+++ b/plugins/discourse-ai/lib/summarization/fold_content.rb
@@ -204,7 +204,7 @@ module DiscourseAi
end
end
- bot.reply(context, llm_args: { max_tokens: }, &buffer_blk)
+ bot.reply(context, llm_args: { max_response_tokens: max_tokens }, &buffer_blk)
if tool_output && summary.blank?
raise MissingToolOutput, "The model did not set a topic summary"
diff --git a/plugins/discourse-ai/lib/translation/base_translator.rb b/plugins/discourse-ai/lib/translation/base_translator.rb
index 4e6c02af721..65cca5c61b6 100644
--- a/plugins/discourse-ai/lib/translation/base_translator.rb
+++ b/plugins/discourse-ai/lib/translation/base_translator.rb
@@ -117,7 +117,7 @@ module DiscourseAi
def get_translation(text:, bot:, translation_user:, output_tokens:)
context = translation_context(text:, translation_user:)
- llm_args = { max_tokens: output_tokens }
+ llm_args = { max_response_tokens: output_tokens }
structured_output = nil
result = +""
diff --git a/plugins/discourse-ai/spec/lib/agents/bot_spec.rb b/plugins/discourse-ai/spec/lib/agents/bot_spec.rb
index 4e47276fa1c..bb1ea5542ed 100644
--- a/plugins/discourse-ai/spec/lib/agents/bot_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/bot_spec.rb
@@ -68,6 +68,40 @@ RSpec.describe DiscourseAi::Agents::Bot do
)
end
+ it "sends no max_tokens when neither the caller nor the model configures a limit" do
+ context =
+ DiscourseAi::Agents::BotContext.new(
+ user: user,
+ messages: [{ type: :user, content: "test" }],
+ )
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["Answer"]) do |_, _, _, options|
+ bot.reply(context) { |_partial| }
+
+ expect(options.last[:max_tokens]).to be_nil
+ end
+ end
+
+ it "admits child generations on unconfigured Anthropic models below the model cap" do
+ anthropic = Fabricate(:anthropic_model)
+ child_bot =
+ described_class.as(bot_user, agent: DiscourseAi::Agents::General.new, model: anthropic)
+ context =
+ DiscourseAi::Agents::BotContext.new(
+ user: user,
+ messages: [{ type: :user, content: "test" }],
+ subagent_depth: 1,
+ turn_token_budget: 2000,
+ )
+
+ raw = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses(["Answer"]) do
+ raw = child_bot.reply(context) { |_partial| }
+ end
+
+ expect(raw.map(&:first)).to include("Answer")
+ end
+
it "requests Gemini thought summaries when thinking is shown" do
gemini = Fabricate(:gemini_model)
captured_kwargs = []
@@ -350,7 +384,7 @@ RSpec.describe DiscourseAi::Agents::Bot do
"Fitting answer",
)
expect(options.map { |option| option[:max_tokens] }).to eq(
- [mode == :tools ? 2000 : mode == :final ? 2048 : 4000],
+ [mode == :tools ? 2000 : mode == :final ? 4096 : 4000],
)
expect(options.map { |option| option[:feature_name] }).to eq(["bot"])
expect(prompts.first.tool_choice).to eq(mode == :final ? :none : nil)
@@ -412,7 +446,7 @@ RSpec.describe DiscourseAi::Agents::Bot do
# Simulate a provider whose mandatory reasoning cannot be disabled for finalization.
allow_any_instance_of(DiscourseAi::Completions::Llm).to receive(
:prompt_capacity,
- ).and_return([100, 20_000, 4096])
+ ).and_return([100, 20_000, 8192])
DiscourseAi::Completions::Llm.with_prepared_responses([]) do |canned|
expect {
described_class.as(
@@ -505,7 +539,11 @@ RSpec.describe DiscourseAi::Agents::Bot do
expect(raw.select { |entry| entry[2] == "tool" }.map(&:first)).to eq([result.to_json])
expect(context.execution_context.work_budget.used).to be > 5000
expect(prompts.last.tool_choice).to eq(:none)
- expect(options.last[:max_tokens]).to be <= described_class::MAX_FINAL_ANSWER_TOKENS
+ expect(options.last[:max_response_tokens]).to be <=
+ described_class::MAX_FINAL_ANSWER_TOKENS
+ expect(options.last[:max_tokens]).to be <=
+ described_class::MAX_FINAL_ANSWER_TOKENS +
+ DiscourseAi::Completions::TurnWorkBudget::MAX_FINAL_THINKING_ALLOWANCE
expect(options.last[:thinking_effort]).to eq("none")
expect(canned.completions).to eq(2)
expect(context.execution_context.work_budget.snapshot[:final_answer_claimed]).to eq(true)
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb
index 924433748ad..751c1bebbf0 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/anthropic_spec.rb
@@ -405,8 +405,8 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
expect(execution.work_budget.remaining).to eq(15_950)
expect(execution.work_budget.limit).to eq(32_000)
llm.generate("Legacy", user: Discourse.system_user, max_tokens: 16_000, thinking_effort: "high")
- expect(bodies.last["max_tokens"]).to eq(64_000)
- expect(bodies.last["thinking"]["budget_tokens"]).to eq(48_000)
+ expect(bodies.last["max_tokens"]).to eq(32_384)
+ expect(bodies.last["thinking"]["budget_tokens"]).to eq(16_384)
expect(request).to have_been_requested.twice
ensure
execution&.work_budget&.release_output(parent_reservation)
@@ -605,6 +605,100 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
expect(log.raw_response_payload.strip).to eq(body.strip)
end
+ describe "default max_tokens" do
+ it "reserves the effective cap for prompt admission when nothing is configured" do
+ endpoint = described_class.new(model)
+
+ expect(endpoint.prepare_model_params({})[:reserved_output_tokens]).to eq(4096)
+
+ model.update!(name: "claude-sonnet-4-6")
+ expect(described_class.new(model).prepare_model_params({})[:reserved_output_tokens]).to eq(
+ 128_000,
+ )
+ end
+
+ it "maps friendly names to canonical API model names" do
+ dialect = DiscourseAi::Completions::Dialects::Claude.new(prompt, model)
+
+ {
+ "claude-4-opus" => "claude-opus-4-20250514",
+ "claude-4-sonnet" => "claude-sonnet-4-20250514",
+ "claude-3-5-sonnet" => "claude-3-5-sonnet-20241022",
+ "claude-opus-4-5" => "claude-opus-4-5-20251101",
+ "claude-sonnet-4-5-20250929" => "claude-sonnet-4-5-20250929",
+ }.each do |friendly_name, canonical_name|
+ model.update!(name: friendly_name)
+ expect(described_class.new(model).default_options(dialect)[:model]).to eq(canonical_name)
+ end
+ end
+
+ it "uses the known cap for newer models when no limit is configured" do
+ model.update!(name: "claude-sonnet-4-6")
+ dialect = DiscourseAi::Completions::Dialects::Claude.new(prompt, model)
+
+ options = described_class.new(model).default_options(dialect)
+
+ expect(options[:max_tokens]).to eq(128_000)
+ end
+
+ it "uses the known cap for dated snapshots when no limit is configured" do
+ model.update!(name: "claude-haiku-4-5-20251001")
+ dialect = DiscourseAi::Completions::Dialects::Claude.new(prompt, model)
+
+ options = described_class.new(model).default_options(dialect)
+
+ expect(options[:max_tokens]).to eq(64_000)
+ end
+
+ it "assumes a 64k cap for unrecognised models when no limit is configured" do
+ model.update!(name: "claude-sonnet-99")
+ dialect = DiscourseAi::Completions::Dialects::Claude.new(prompt, model)
+
+ options = described_class.new(model).default_options(dialect)
+
+ expect(options[:max_tokens]).to eq(64_000)
+ end
+
+ it "uses the configured limit over the known cap" do
+ model.update!(name: "claude-sonnet-4-6", max_output_tokens: 2000)
+ dialect = DiscourseAi::Completions::Dialects::Claude.new(prompt, model)
+
+ options = described_class.new(model).default_options(dialect)
+
+ expect(options[:max_tokens]).to eq(2000)
+ end
+
+ it "clamps explicit requests to the known cap when no limit is configured" do
+ model.update!(name: "claude-sonnet-4-6")
+
+ parsed_body = nil
+ stub_request(:post, url).with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(
+ status: 200,
+ body: {
+ id: "msg_123",
+ type: "message",
+ role: "assistant",
+ content: [{ type: "text", text: "test response" }],
+ model: "claude-sonnet-4-6",
+ usage: {
+ input_tokens: 10,
+ output_tokens: 5,
+ },
+ }.to_json,
+ )
+
+ llm.generate(prompt, user: Discourse.system_user, max_tokens: 200_000)
+
+ expect(parsed_body[:max_tokens]).to eq(128_000)
+ end
+ end
+
it "supports non streaming tool calls" do
tool = {
name: "calculate",
@@ -845,6 +939,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
model.provider_params["enable_reasoning"] = true
model.provider_params["reasoning_tokens"] = 10_000
+ model.max_output_tokens = 40_000
model.save!
proxy = DiscourseAi::Completions::Llm.proxy(model)
@@ -902,6 +997,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
it "configures adaptive thinking with effort" do
model.update!(
+ max_output_tokens: 32_000,
provider_params: {
enable_reasoning: true,
adaptive_thinking: true,
@@ -2279,7 +2375,13 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
describe "adaptive thinking" do
it "sends adaptive thinking config when enabled" do
- model.update!(provider_params: { enable_reasoning: true, adaptive_thinking: true })
+ model.update!(
+ max_output_tokens: 32_000,
+ provider_params: {
+ enable_reasoning: true,
+ adaptive_thinking: true,
+ },
+ )
parsed_body = nil
stub_request(:post, url).with(
@@ -2315,6 +2417,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Anthropic do
it "adaptive_thinking takes priority over enable_reasoning" do
model.update!(
+ max_output_tokens: 32_000,
provider_params: {
adaptive_thinking: true,
enable_reasoning: true,
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
index b61805a383c..2b338331861 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
@@ -241,7 +241,34 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
end
describe "default_options" do
- it "does not set max_tokens by default" do
+ it "sets max_tokens to the known model cap by default" do
+ prompt = DiscourseAi::Completions::Prompt.new("hello")
+ dialect = DiscourseAi::Completions::Dialects::Converse.new(prompt, model)
+
+ expect(endpoint.default_options(dialect)[:max_tokens]).to eq(4096)
+ end
+
+ it "assumes a 64k cap for unrecognised Claude models when no limit is configured" do
+ model.update!(name: "claude-sonnet-99")
+
+ prompt = DiscourseAi::Completions::Prompt.new("hello")
+ dialect = DiscourseAi::Completions::Dialects::Converse.new(prompt, model)
+
+ expect(endpoint.default_options(dialect)[:max_tokens]).to eq(64_000)
+ end
+
+ it "uses the configured limit over the known cap" do
+ model.update!(max_output_tokens: 2000)
+
+ prompt = DiscourseAi::Completions::Prompt.new("hello")
+ dialect = DiscourseAi::Completions::Dialects::Converse.new(prompt, model)
+
+ expect(endpoint.default_options(dialect)[:max_tokens]).to eq(2000)
+ end
+
+ it "does not set max_tokens for non-Claude models" do
+ model.update!(name: "amazon.nova-pro-v1:0")
+
prompt = DiscourseAi::Completions::Prompt.new("hello")
dialect = DiscourseAi::Completions::Dialects::Converse.new(prompt, model)
@@ -260,6 +287,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
end
it "configures reasoning with budget tokens" do
+ model.update!(max_output_tokens: 8192)
model.provider_params["enable_reasoning"] = true
model.provider_params["reasoning_tokens"] = 4096
@@ -447,6 +475,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
end
it "passes thinking config for Claude models" do
+ model.update!(max_output_tokens: 8192)
response = mock_converse_response
client = stub_sdk_client(response: response)
@@ -494,7 +523,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
end
end
- it "passes model name directly as model_id to SDK" do
+ it "translates friendly model names to Bedrock model IDs for the SDK" do
response = mock_converse_response
client = stub_sdk_client(response: response)
@@ -502,11 +531,34 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
llm.generate("hello", user: user)
expect(client).to have_received(:converse) do |params|
- expect(params[:model_id]).to eq("claude-3-sonnet")
+ expect(params[:model_id]).to eq("anthropic.claude-3-sonnet-20240229-v1:0")
expect(params[:system]).to be_present
end
end
+ it "translates newer friendly names and passes qualified IDs through" do
+ response = mock_converse_response
+ client = stub_sdk_client(response: response)
+ seen_model_ids = []
+ allow(client).to receive(:converse) do |params|
+ seen_model_ids << params[:model_id]
+ response
+ end
+
+ {
+ "claude-opus-4-1" => "anthropic.claude-opus-4-1-20250805-v1:0",
+ "claude-opus-4-5" => "anthropic.claude-opus-4-5-20251101-v1:0",
+ "claude-sonnet-4-6" => "anthropic.claude-sonnet-4-6",
+ "claude-opus-4-8" => "anthropic.claude-opus-4-8",
+ "global.anthropic.claude-sonnet-4-6" => "global.anthropic.claude-sonnet-4-6",
+ "amazon.nova-pro-v1:0" => "amazon.nova-pro-v1:0",
+ }.each do |friendly_name, bedrock_id|
+ model.update!(name: friendly_name)
+ DiscourseAi::Completions::Llm.proxy("custom:#{model.id}").generate("hello", user: user)
+ expect(seen_model_ids.last).to eq(bedrock_id)
+ end
+ end
+
it "logs a placeholder when image bytes are binary" do
model.update!(vision_enabled: true)
raw_bytes = "\x89PNG\r\n\x1a\nbinary".b
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_spec.rb
index b45cda08041..f315620a8db 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_spec.rb
@@ -43,6 +43,58 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrock do
model.name = "claude-3-5-haiku"
options = endpoint.default_options(dialect)
expect(options[:max_tokens]).to eq(8192)
+
+ # Bedrock caps Sonnet 4.6 at 64k, unlike the Anthropic API (128k)
+ model.name = "claude-sonnet-4-6"
+ expect(endpoint.default_options(dialect)[:max_tokens]).to eq(64_000)
+
+ model.name = "claude-sonnet-99"
+ expect(endpoint.default_options(dialect)[:max_tokens]).to eq(64_000)
+
+ model.update!(name: "claude-sonnet-4-6", max_output_tokens: 2000)
+ expect(endpoint.default_options(dialect)[:max_tokens]).to eq(2000)
+ end
+
+ it "translates friendly model names to Bedrock model IDs in the request URL" do
+ proxy = DiscourseAi::Completions::Llm.proxy(model)
+
+ content = {
+ content: [text: "hello sam"],
+ usage: {
+ input_tokens: 10,
+ output_tokens: 20,
+ },
+ }.to_json
+
+ requested_model_ids = []
+ stub_request(:post, %r{bedrock-runtime\.us-east-1\.amazonaws\.com/model/})
+ .with do |request|
+ requested_model_ids << URI.decode_www_form_component(
+ request.uri.path.split("/model/").last.sub(%r{/invoke\z}, ""),
+ )
+ true
+ end
+ .to_return(status: 200, body: content)
+
+ {
+ "claude-opus-4-1" => "anthropic.claude-opus-4-1-20250805-v1:0",
+ "claude-opus-4-1-20250805" => "anthropic.claude-opus-4-1-20250805-v1:0",
+ "claude-opus-4-5" => "anthropic.claude-opus-4-5-20251101-v1:0",
+ "claude-sonnet-4-5" => "anthropic.claude-sonnet-4-5-20250929-v1:0",
+ "claude-sonnet-4-5-20250929" => "anthropic.claude-sonnet-4-5-20250929-v1:0",
+ "claude-haiku-4-5" => "anthropic.claude-haiku-4-5-20251001-v1:0",
+ "claude-opus-4-6" => "anthropic.claude-opus-4-6-v1",
+ "claude-sonnet-4-6" => "anthropic.claude-sonnet-4-6",
+ "claude-opus-4-7" => "anthropic.claude-opus-4-7",
+ "claude-sonnet-5" => "anthropic.claude-sonnet-5",
+ "claude-fable-5-1" => "anthropic.claude-fable-5-1",
+ "claude-haiku-5-5" => "anthropic.claude-haiku-5-5",
+ "us.anthropic.claude-sonnet-4-6" => "us.anthropic.claude-sonnet-4-6",
+ }.each do |friendly_name, bedrock_id|
+ model.update!(name: friendly_name)
+ proxy.generate("hello world", user: user)
+ expect(requested_model_ids.last).to eq(bedrock_id)
+ end
end
describe "function calling" do
@@ -341,6 +393,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrock do
it "supports thinking" do
model.provider_params["enable_reasoning"] = true
model.provider_params["reasoning_tokens"] = 10_000
+ model.max_output_tokens = 40_000
model.save!
proxy = DiscourseAi::Completions::Llm.proxy(model)
@@ -1335,6 +1388,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrock do
describe "adaptive thinking" do
it "sends adaptive thinking config when enabled" do
model.update!(
+ max_output_tokens: 32_000,
provider_params: {
access_key_id: "123",
region: "us-east-1",
@@ -1373,6 +1427,7 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrock do
it "adaptive_thinking takes priority over enable_reasoning" do
model.update!(
+ max_output_tokens: 32_000,
provider_params: {
access_key_id: "123",
region: "us-east-1",
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/hugging_face_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/hugging_face_spec.rb
index 56ade1aadbf..5364deb37e1 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/hugging_face_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/hugging_face_spec.rb
@@ -97,6 +97,39 @@ RSpec.describe DiscourseAi::Completions::Endpoints::HuggingFace do
before { enable_current_plugin }
+ describe "output limit" do
+ it "passes an explicit max_tokens through" do
+ parsed_body = nil
+ stub_request(:post, "https://test.dev/v1/chat/completions").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { choices: [{ message: { content: "ok" } }] }.to_json)
+
+ DiscourseAi::Completions::Llm.proxy(hf_model).generate("hello", user: user, max_tokens: 777)
+
+ expect(parsed_body[:max_tokens]).to eq(777)
+ end
+
+ it "derives the remaining context window when nothing is configured" do
+ parsed_body = nil
+ stub_request(:post, "https://test.dev/v1/chat/completions").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { choices: [{ message: { content: "ok" } }] }.to_json)
+
+ DiscourseAi::Completions::Llm.proxy(hf_model).generate("hello", user: user)
+
+ expect(parsed_body[:max_tokens]).to be > 60_000
+ expect(parsed_body[:max_tokens]).to be < 64_000
+ end
+ end
+
describe "#perform_completion!" do
context "when using regular mode" do
context "with simple prompts" do
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/ollama_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/ollama_spec.rb
index 4d2def80ace..c1a62ff7691 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/ollama_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/ollama_spec.rb
@@ -137,6 +137,38 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Ollama do
before { enable_current_plugin }
+ describe "output limit" do
+ it "omits max_tokens when nothing is configured" do
+ parsed_body = nil
+ stub_request(:post, "http://api.ollama.ai/api/chat").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { message: { content: "ok" } }.to_json)
+
+ DiscourseAi::Completions::Llm.proxy(model).generate("hello", user: user)
+
+ expect(parsed_body).not_to have_key(:max_tokens)
+ end
+
+ it "passes an explicit max_tokens through" do
+ parsed_body = nil
+ stub_request(:post, "http://api.ollama.ai/api/chat").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { message: { content: "ok" } }.to_json)
+
+ DiscourseAi::Completions::Llm.proxy(model).generate("hello", user: user, max_tokens: 123)
+
+ expect(parsed_body[:max_tokens]).to eq(123)
+ end
+ end
+
describe "#perform_completion!" do
context "when using regular mode" do
it "completes a trivial prompt and logs the response" do
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_responses_api_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_responses_api_spec.rb
index 43526637701..4c536704caa 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_responses_api_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_responses_api_spec.rb
@@ -577,6 +577,31 @@ data: {"type":"response.reasoning_summary_part.added","sequence_number":3,"item_
expect(parsed_body[:reasoning]).to include(effort: "none", summary: "auto")
end
+ it "maps none to low for GPT-6.1 Sol, which does not support none" do
+ model.update!(name: "gpt-6.1-sol", provider_params: { reasoning_effort: "none" })
+
+ parsed_body = nil
+ stub_request(:post, "https://api.openai.com/v1/responses").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { output: [] }.to_json)
+
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "You are a bot",
+ messages: [type: :user, content: "hello"],
+ )
+
+ dialect = DiscourseAi::Completions::Dialects::OpenAiResponses.new(prompt, model)
+
+ endpoint.perform_completion!(dialect, Discourse.system_user)
+
+ expect(parsed_body[:reasoning]).to include(effort: "low", summary: "auto")
+ end
+
it "strips temperature and top_p when reasoning_effort is set" do
model.update!(provider_params: { reasoning_effort: "high" })
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
index 361faf889ad..066d05a8eac 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
@@ -538,7 +538,11 @@ RSpec.describe DiscourseAi::Completions::Endpoints::OpenAi do
response_status: 200,
response_tokens: 1,
)
- expect(execution.work_budget.used).to eq(1)
+ # Cancelled streams settle as incomplete: exposed output wins over an
+ # early usage snapshot.
+ expect(execution.work_budget.used).to eq(
+ DiscourseAi::Completions::Llm.proxy(model).tokenizer.size(partial),
+ )
end
end
@@ -1250,6 +1254,34 @@ RSpec.describe DiscourseAi::Completions::Endpoints::OpenAi do
expect(body_parsed["messages"].first["content"]).to eq("You are a bot")
end
+ it "supports gpt-6, remaps max_tokens, and uses developer message" do
+ model.update!(name: "gpt-6.1-sol")
+
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "You are a bot",
+ messages: [type: :user, content: "hello"],
+ )
+ dialect = compliance.dialect(prompt: prompt)
+
+ body_parsed = nil
+ stub_request(:post, "https://api.openai.com/v1/chat/completions").with(
+ body:
+ proc do |body|
+ body_parsed = JSON.parse(body)
+ true
+ end,
+ ).to_return(status: 200, body: { choices: [{ message: { content: "ok" } }] }.to_json)
+
+ endpoint.perform_completion!(dialect, user, { max_tokens: 321 })
+
+ expect(body_parsed["model"]).to eq("gpt-6.1-sol")
+ expect(body_parsed["max_completion_tokens"]).to eq(321)
+ expect(body_parsed["max_tokens"]).to be_nil
+ expect(body_parsed["messages"].first["role"]).to eq("developer")
+ expect(body_parsed["messages"].first["content"]).to eq("You are a bot")
+ end
+
context "with tools" do
it "returns a function invocation" do
compliance.streaming_mode_tools(open_ai_mock)
@@ -1688,6 +1720,74 @@ TEXT
expect(parsed_body[:reasoning_effort]).to eq("max")
end
+ it "maps none to low for GPT-6 Astra, which does not support none" do
+ model.update!(name: "gpt-6-astra")
+
+ parsed_body = nil
+ stub_request(:post, "https://api.openai.com/v1/chat/completions").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { choices: [{ message: { content: "test" } }] }.to_json)
+
+ endpoint.perform_completion!(dialect, user, { thinking_effort: "none" })
+
+ expect(parsed_body[:reasoning_effort]).to eq("low")
+ end
+
+ it "maps none to low for GPT-6.1 Sol snapshots, which do not support none" do
+ model.update!(name: "gpt-6.1-sol-2026-09-01", provider_params: { reasoning_effort: "none" })
+
+ parsed_body = nil
+ stub_request(:post, "https://api.openai.com/v1/chat/completions").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { choices: [{ message: { content: "test" } }] }.to_json)
+
+ endpoint.perform_completion!(dialect, user)
+
+ expect(parsed_body[:reasoning_effort]).to eq("low")
+ end
+
+ it "keeps none for GPT-6 Sol, which supports none" do
+ model.update!(name: "gpt-6-sol")
+
+ parsed_body = nil
+ stub_request(:post, "https://api.openai.com/v1/chat/completions").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { choices: [{ message: { content: "test" } }] }.to_json)
+
+ endpoint.perform_completion!(dialect, user, { thinking_effort: "none" })
+
+ expect(parsed_body[:reasoning_effort]).to eq("none")
+ end
+
+ it "uses native max effort for GPT-6 Astra" do
+ model.update!(name: "gpt-6-astra")
+
+ parsed_body = nil
+ stub_request(:post, "https://api.openai.com/v1/chat/completions").with(
+ body:
+ proc do |req_body|
+ parsed_body = JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: { choices: [{ message: { content: "test" } }] }.to_json)
+
+ endpoint.perform_completion!(dialect, user, { thinking_effort: "max" })
+
+ expect(parsed_body[:reasoning_effort]).to eq("max")
+ end
+
it "uses native max effort for future OpenAI models by default" do
model.update!(name: "gpt-5.7")
@@ -1750,6 +1850,14 @@ TEXT
expect(prepared_params).not_to have_key(:reserved_output_tokens)
end
+ it "reserves output room when none maps to low on GPT-6.1 Sol" do
+ model.update!(name: "gpt-6.1-sol")
+
+ prepared_params = endpoint.prepare_model_params(thinking_effort: "none")
+
+ expect(prepared_params[:reserved_output_tokens]).to eq(25_000)
+ end
+
it "treats standard reasoning mode as a no-op for Chat Completions" do
model.update!(provider_params: { reasoning_mode: "standard" })
diff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
index ac8acc2d92f..3de4b4cfa32 100644
--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
@@ -44,6 +44,136 @@ RSpec.describe DiscourseAi::Completions::Llm do
before { enable_current_plugin }
+ describe "max_response_tokens" do
+ it "trims streamed text at the cap, ignores thinking, and does not trip the caller's cancel manager" do
+ cancel = DiscourseAi::Completions::CancelManager.new
+ thinking = DiscourseAi::Completions::Thinking.new(message: "thinking " * 20)
+ received = []
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [[thinking, "one two three four five six seven eight"]],
+ ) do
+ llm.generate(
+ "Input",
+ user: user,
+ max_response_tokens: 3,
+ cancel_manager: cancel,
+ ) { |partial| received << partial }
+ end
+
+ expect(received).to include(thinking)
+ expect(llm.tokenizer.size(received.select { |partial| partial.is_a?(String) }.join)).to be <=
+ 3
+ expect(cancel.cancelled?).to eq(false)
+ end
+
+ it "truncates a non-streamed string result" do
+ result = nil
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ ["one two three four five six seven eight"],
+ ) { result = llm.generate("Input", user: user, max_response_tokens: 3) }
+
+ expect(llm.tokenizer.size(result)).to be <= 3
+ end
+
+ it "does not trim tool calls" do
+ tool_call =
+ DiscourseAi::Completions::ToolCall.new(
+ id: "call_1",
+ name: "echo",
+ parameters: {
+ text: "one two three four five six seven eight",
+ },
+ )
+ received = []
+
+ DiscourseAi::Completions::Llm.with_prepared_responses([[tool_call]]) do
+ llm.generate("Input", user: user, max_response_tokens: 3) { |partial| received << partial }
+ end
+
+ expect(received).to eq([tool_call])
+ end
+
+ it "translates the cap to a provider ceiling with a thinking allowance" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["ok"]) do |endpoint|
+ llm.generate("Input", user: user, max_response_tokens: 10, thinking_effort: "high")
+
+ expect(endpoint.model_params[:max_tokens]).to eq(10 + 16_384)
+ end
+ end
+
+ it "translates the cap to a provider ceiling when no thinking is requested" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["ok"]) do |endpoint|
+ llm.generate("Input", user: user, max_response_tokens: 10)
+
+ expect(endpoint.model_params[:max_tokens]).to eq(10)
+ end
+ end
+
+ it "records a truncated completion when the limiter cuts a real stream" do
+ stub_response(body: streaming_body(content: "one two three four five six seven eight"))
+ execution = DiscourseAi::Completions::ExecutionContext.new
+ received = +""
+
+ llm.generate(
+ "Input",
+ user: user,
+ max_response_tokens: 3,
+ execution_context: execution,
+ ) { |partial| received << partial if partial.is_a?(String) }
+
+ expect(llm.tokenizer.size(received)).to be <= 3
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::TRUNCATED)
+ expect(AiApiAuditLog.last.stop_reason).to eq("max_response_tokens")
+ end
+
+ it "records a truncated completion for a non-streamed cut without an execution context" do
+ stub_response(body: success_body(content: "one two three four five six seven eight"))
+
+ result = llm.generate("Input", user: user, max_response_tokens: 3)
+
+ expect(llm.tokenizer.size(result)).to be <= 3
+ expect(AiApiAuditLog.last.completion_state).to eq(AiApiAuditLog::CompletionState::TRUNCATED)
+ expect(AiApiAuditLog.last.stop_reason).to eq("max_response_tokens")
+ end
+
+ it "does not truncate structured output returned as a string" do
+ stub_response(body: success_body(content: "{\"answer\":\"one two three four five\"}"))
+
+ result =
+ llm.generate(
+ "Input",
+ user: user,
+ max_response_tokens: 3,
+ response_format: {
+ type: "json_schema",
+ json_schema: {
+ name: "answer",
+ schema: {
+ type: "object",
+ properties: {
+ answer: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ )
+
+ expect(DiscourseAi::Completions::Llm.text_from_response(result)).to include("four five")
+ end
+
+ it "keeps an explicit provider ceiling over the translated cap" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["ok"]) do |endpoint|
+ llm.generate("Input", user: user, max_tokens: 500, max_response_tokens: 10)
+
+ expect(endpoint.model_params[:max_tokens]).to eq(500)
+ end
+ end
+ end
+
describe "turn work settlement" do
it "blocks unadmitted helper generations when work or the shared completion bound is exhausted" do
request = stub_response
diff --git a/plugins/discourse-ai/spec/lib/completions/turn_work_budget_spec.rb b/plugins/discourse-ai/spec/lib/completions/turn_work_budget_spec.rb
index 33e321f6625..e7346cd144e 100644
--- a/plugins/discourse-ai/spec/lib/completions/turn_work_budget_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/turn_work_budget_spec.rb
@@ -26,16 +26,55 @@ RSpec.describe DiscourseAi::Completions::TurnWorkBudget do
options = { thinking_effort: "high" }
expect(budget.generation_options(options, maximum: 5000, final: true, root: true)).to eq(
- max_tokens: 2048,
+ max_tokens: 4096,
max_tokens_is_total: true,
+ max_response_tokens: 2048,
thinking_effort: "none",
)
expect(
budget.generation_options(options, maximum: 1000, final: true, root: true)[:max_tokens],
).to eq(1000)
+ expect(
+ budget.generation_options(options, maximum: 1000, final: true, root: true)[
+ :max_response_tokens
+ ],
+ ).to eq(1000)
expect(budget.generation_options(options, maximum: 5000, final: true)[:max_tokens]).to eq(500)
expect(budget.remaining).to eq(500)
end
+
+ it "derives the provider ceiling from visible intent plus thinking allowance" do
+ budget = described_class.new(limit: 100_000)
+
+ expect(
+ budget.generation_options(
+ { max_response_tokens: 400, thinking_effort: "high" },
+ maximum: nil,
+ )[
+ :max_tokens
+ ],
+ ).to eq(400 + 16_384)
+ expect(
+ budget.generation_options(
+ { max_response_tokens: 400, thinking_effort: "high" },
+ maximum: 1000,
+ )[
+ :max_tokens
+ ],
+ ).to eq(1000)
+ end
+
+ it "sends no ceiling when nothing is configured, except the bounded final answer" do
+ budget = described_class.new(limit: 4000)
+
+ expect(budget.generation_options({}, maximum: nil)).to eq({})
+ expect(budget.generation_options({}, maximum: nil, final: true, root: true)).to eq(
+ max_tokens: 4096,
+ max_tokens_is_total: true,
+ max_response_tokens: 2048,
+ thinking_effort: "none",
+ )
+ end
end
describe "#reserve_generation_output" do
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
index be5551e8221..fa84b91175d 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
@@ -818,7 +818,7 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
["Not executed — work allowance or tool batch limit reached."],
)
expect(final.tool_choice).to eq(:none)
- expect(options[1][:max_tokens]).to eq(2048)
+ expect(options[1][:max_tokens]).to eq(4096)
generation = DiscourseAi::Completions::GeneratedOutput.new
calls.each { |call| generation << call }
generation << "Batch summary"
@@ -859,7 +859,7 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
options
.select { |option| option[:feature_name] == "bot" }
.map { |option| option[:max_tokens] },
- ).to eq([2000, 2048])
+ ).to eq([2000, 4096])
expect(options.map { |option| option[:feature_name] }).not_to include("context_compression")
end
end
@@ -877,7 +877,7 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
state["work_budget"]["used"] = 5000
Discourse.redis.setex(key, described_class::RESUME_STATE_TTL_SECONDS, state.to_json)
allow_any_instance_of(DiscourseAi::Completions::Llm).to receive(:prompt_capacity).and_return(
- [100, 20_000, 4096],
+ [100, 20_000, 8192],
)
session =
build_session(
diff --git a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
index 50ac447b72e..8130c99ece4 100644
--- a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
+++ b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
@@ -122,6 +122,7 @@ describe DiscourseAi::Translation::BaseTranslator do
DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do |_, _, _, options|
expect(translator.translate).to eq(llm_response)
expect(options.last[:max_tokens]).to eq(2500)
+ expect(options.last[:max_response_tokens]).to eq(2500)
end
end