Skip to content

A platform for community discussion. Free, open, simple.

FIX: Finalize AI replies at token budget (#41833)

Sam5ccba9bbcc3e
diff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rb
index 13c1119e11a..ae3870207e4 100644
--- a/plugins/discourse-ai/lib/agents/bot.rb
+++ b/plugins/discourse-ai/lib/agents/bot.rb
@@ -14,15 +14,31 @@ module DiscourseAi
COMPRESSED_CONTEXT_ACK =
DiscourseAi::Completions::PromptMessagesBuilder::COMPRESSED_CONTEXT_ACK
- BUDGET_EXHAUSTED_HINT = <<~TEXT.strip
+ TOKEN_BUDGET_FINAL_ANSWER_HINT = <<~TEXT.strip
+ [The turn token budget has been reached — no further tool calls are available.]
+ Provide your final response now using the information already gathered.
+ Do not describe additional tool calls or future work as though you will perform them.
+ If the task is not fully complete, clearly state what was accomplished
+ and what still needs to be done so the user can continue in a follow-up message.
+ TEXT
+ LEGACY_BUDGET_EXHAUSTED_HINT = <<~TEXT.strip
[Turn budget exhausted — you cannot call any more tools.]
Provide your final response using the information already gathered.
If the task is not fully complete, clearly state what was accomplished
and what still needs to be done so the user can continue in a follow-up message.
TEXT
+ BUDGET_EXHAUSTED_HINT = TOKEN_BUDGET_FINAL_ANSWER_HINT
+ TRANSIENT_TOKEN_BUDGET_HINTS = [
+ TOKEN_BUDGET_FINAL_ANSWER_HINT,
+ LEGACY_BUDGET_EXHAUSTED_HINT,
+ ].freeze
+
+ def self.inject_token_budget_final_answer_hint(prompt)
+ prompt.push(type: :user, content: TOKEN_BUDGET_FINAL_ANSWER_HINT)
+ end
def self.inject_budget_exhausted_hint(prompt)
- prompt.push(type: :user, content: BUDGET_EXHAUSTED_HINT)
+ inject_token_budget_final_answer_hint(prompt)
end
# When an agent has no explicit max_turn_tokens, default the turn budget
@@ -130,16 +146,10 @@ module DiscourseAi
break # already ran the final text-only generate
end
- should_stop = token_usage_tracker.total >= token_budget
-
- if should_stop
- if prompt.messages.last&.dig(:type) == :tool
- self.class.inject_budget_exhausted_hint(prompt)
- prompt.tool_choice = :none
- final_answer_requested = true
- else
- break
- end
+ if token_usage_tracker.total >= token_budget
+ self.class.inject_token_budget_final_answer_hint(prompt)
+ prompt.tool_choice = :none
+ final_answer_requested = true
end
compression_result =
@@ -257,9 +267,6 @@ module DiscourseAi
total_completions += 1
if !final_answer_requested
- total_used = token_usage_tracker.total
- prompt.tool_choice = :none if total_used >= (token_budget * 0.85)
-
# Safety valve against pathological tool loops.
break if total_completions >= 100
end
@@ -570,20 +577,21 @@ module DiscourseAi
instruction = COMPRESSION_INSTRUCTION
instruction = "#{instruction}\n#{COMPRESSION_MERGE_INSTRUCTION}" if has_prior_compression
- compression_messages = prompt.messages.map(&:dup)
+ compression_messages =
+ prompt.messages.reject { |message| transient_prompt_hint?(message) }.map(&:dup)
compression_messages << { type: :user, content: instruction }
- compression_prompt =
- DiscourseAi::Completions::Prompt.new(
- messages: compression_messages,
- tools: prompt.tools,
- topic_id: prompt.topic_id,
- post_id: prompt.post_id,
- )
- compression_prompt.tool_choice = :none
-
summary =
begin
+ compression_prompt =
+ DiscourseAi::Completions::Prompt.new(
+ messages: compression_messages,
+ tools: prompt.tools,
+ topic_id: prompt.topic_id,
+ post_id: prompt.post_id,
+ )
+ compression_prompt.tool_choice = :none
+
current_llm.generate(
compression_prompt,
user: nil,
@@ -664,7 +672,7 @@ module DiscourseAi
when :tool
[message[:content], message[:id], "tool", message[:name]]
when :user
- [message[:content], message[:id], "user"]
+ [message[:content], message[:id], "user"] if !transient_prompt_hint?(message)
when :model
[message[:content], nil, "model", nil, thinking_context(message)]
end
@@ -672,6 +680,10 @@ module DiscourseAi
.compact
end
+ def transient_prompt_hint?(message)
+ message[:type] == :user && TRANSIENT_TOKEN_BUDGET_HINTS.include?(message[:content])
+ end
+
def thinking_context(message)
if message[:thinking] || message[:thinking_provider_info]
{
diff --git a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
index ff853811b5f..a5aaff1b80e 100644
--- a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
+++ b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
@@ -198,22 +198,20 @@ module DiscourseAi
feature_name: "bot",
}
- # Pre-check: if budget already exhausted on resume, force a final
- # text-only call or finalize immediately — don't overshoot.
+ # Pre-check: if budget is already exhausted on resume, force one final
+ # text-only call instead of starting another tool round.
if @accumulated_tokens >= token_budget
- if @prompt.messages.last&.dig(:type) == :tool
- DiscourseAi::Agents::Bot.inject_budget_exhausted_hint(@prompt)
- @prompt.tool_choice = :none
+ DiscourseAi::Agents::Bot.inject_token_budget_final_answer_hint(@prompt)
+ @prompt.tool_choice = :none
- final_reply = +""
- llm.generate(@prompt, **generate_options) do |partial|
- if partial.is_a?(String) && !partial.empty?
- final_reply << partial
- yield(:partial, partial)
- end
+ final_reply = +""
+ llm.generate(@prompt, **generate_options) do |partial|
+ if partial.is_a?(String) && !partial.empty?
+ final_reply << partial
+ yield(:partial, partial)
end
- @accumulated_reply << final_reply
end
+ @accumulated_reply << final_reply
persist_reply_post!
clear_resume_state!
@@ -277,7 +275,7 @@ module DiscourseAi
)
end
- DiscourseAi::Agents::Bot.inject_budget_exhausted_hint(@prompt)
+ DiscourseAi::Agents::Bot.inject_token_budget_final_answer_hint(@prompt)
@prompt.tool_choice = :none
final_reply = +""
diff --git a/plugins/discourse-ai/spec/lib/agents/bot_spec.rb b/plugins/discourse-ai/spec/lib/agents/bot_spec.rb
index 240a5bf0a70..28906349c62 100644
--- a/plugins/discourse-ai/spec/lib/agents/bot_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/bot_spec.rb
@@ -145,7 +145,7 @@ RSpec.describe DiscourseAi::Agents::Bot do
let(:agent_class) { agent_record.class_instance }
- it "stops the loop when token budget is exhausted" do
+ it "requests a final answer after the token budget is exhausted" do
tool_call =
DiscourseAi::Completions::ToolCall.new(id: "call_1", name: "categories", parameters: {})
@@ -165,7 +165,7 @@ RSpec.describe DiscourseAi::Agents::Bot do
call_count += 1
result = original.call(*args, **kwargs, &blk)
if (tracker = kwargs[:execution_context]&.token_usage_tracker)
- tracker.add_effective(request: 3000, response: 500)
+ tracker.add_effective(request: 5000, response: 1000)
end
result
end
@@ -173,15 +173,14 @@ RSpec.describe DiscourseAi::Agents::Bot do
bot.reply(context) { |_partial| }
end
- # first call: 3500 tokens (under 5000), tool runs
- # second call: 7000 total (over 5000), loop breaks after this call
+ # The first call exceeds the budget after its tool runs, then the second
+ # call provides the final answer with tools disabled.
expect(call_count).to eq(2)
end
- it "sets tool_choice to :none when 85% of budget is consumed" do
- # budget=10000, 85%=8500
- # first call adds 9000 tokens → crosses 85% but under 10000 → sets tool_choice=:none
- # second call sees tool_choice=:none in the prompt
+ it "requests a final answer when the token budget is consumed" do
+ # budget=10000, first call adds 10000 tokens → reaches the threshold
+ # and asks the second call to provide the final answer with tools disabled
big_budget_agent =
Fabricate(
:ai_agent,
@@ -197,6 +196,7 @@ RSpec.describe DiscourseAi::Agents::Bot do
responses = [tool_call, "Done"]
tool_choice_values = []
+ prompt_messages = []
DiscourseAi::Completions::Llm.with_prepared_responses(responses) do
bot = described_class.as(bot_user, agent: klass.new)
@@ -210,9 +210,10 @@ RSpec.describe DiscourseAi::Agents::Bot do
).and_wrap_original do |original, *args, **kwargs, &blk|
prompt_arg = args.first
tool_choice_values << prompt_arg.tool_choice
+ prompt_messages << prompt_arg.messages.map(&:dup)
result = original.call(*args, **kwargs, &blk)
if (tracker = kwargs[:execution_context]&.token_usage_tracker)
- tracker.add_effective(request: 8000, response: 1000)
+ tracker.add_effective(request: 8000, response: 2000)
end
result
end
@@ -220,8 +221,51 @@ RSpec.describe DiscourseAi::Agents::Bot do
bot.reply(context) { |_partial| }
end
- expect(tool_choice_values[0]).not_to eq(:none)
- expect(tool_choice_values[1]).to eq(:none)
+ expect(tool_choice_values.first).not_to eq(:none)
+ expect(tool_choice_values.last).to eq(:none)
+ expect(
+ prompt_messages.map do |messages|
+ messages.count do |message|
+ message[:content] == described_class::TOKEN_BUDGET_FINAL_ANSWER_HINT
+ end
+ end,
+ ).to eq([0, 1])
+ expect(prompt_messages.last.last).to eq(
+ type: :user,
+ content: described_class::TOKEN_BUDGET_FINAL_ANSWER_HINT,
+ )
+ end
+
+ it "allows a final response when the tracker starts at the token budget" do
+ tracker =
+ DiscourseAi::Completions::TokenUsageTracker.new(base_request: 5000, base_response: 0)
+ execution_context =
+ DiscourseAi::Completions::ExecutionContext.new(token_usage_tracker: tracker)
+ final_prompt = nil
+ call_count = 0
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["Final answer"]) do
+ agent_bot = described_class.as(bot_user, agent: agent_class.new)
+ context =
+ DiscourseAi::Agents::BotContext.new(messages: [{ type: :user, content: "Answer" }])
+
+ allow_any_instance_of(DiscourseAi::Completions::Llm).to receive(
+ :generate,
+ ).and_wrap_original do |original, *args, **kwargs, &blk|
+ final_prompt = args.first
+ call_count += 1
+ original.call(*args, **kwargs, &blk)
+ end
+
+ agent_bot.reply(context, execution_context:) { |_partial| }
+ end
+
+ expect(call_count).to eq(1)
+ expect(final_prompt.tool_choice).to eq(:none)
+ expect(final_prompt.messages.last).to eq(
+ type: :user,
+ content: described_class::TOKEN_BUDGET_FINAL_ANSWER_HINT,
+ )
end
it "defaults the turn budget to half the context window without max_turn_tokens" do
@@ -388,9 +432,9 @@ RSpec.describe DiscourseAi::Agents::Bot do
expect(main_generate_skip_trim_values).to eq([false])
end
- it "forces a final text-only call with budget hint when budget exhausted after tool execution" do
- # budget=2000, first call adds 3000 tokens → tool runs → budget exceeded
- # but prompt ends with :tool, so model gets one more tool_choice=:none call
+ it "forces a final text-only call after jumping past the token budget" do
+ # budget=2000, first call adds 3000 tokens
+ # The model gets one more tool_choice=:none call to provide the final answer.
small_budget_agent =
Fabricate(
:ai_agent,
@@ -422,7 +466,7 @@ RSpec.describe DiscourseAi::Agents::Bot do
call_count += 1
prompt_arg = args.first
tool_choice_values << prompt_arg.tool_choice
- prompt_messages << prompt_arg.messages.map { |m| m[:type] }
+ prompt_messages << prompt_arg.messages.map(&:dup)
result = original.call(*args, **kwargs, &blk)
if (tracker = kwargs[:execution_context]&.token_usage_tracker)
tracker.add_effective(request: 2500, response: 500)
@@ -436,8 +480,10 @@ RSpec.describe DiscourseAi::Agents::Bot do
expect(call_count).to eq(2)
expect(tool_choice_values[0]).not_to eq(:none)
expect(tool_choice_values[1]).to eq(:none)
- # the budget hint was injected as a :user message before the final call
- expect(prompt_messages[1]).to include(:user)
+ expect(prompt_messages.last.last).to eq(
+ type: :user,
+ content: described_class::TOKEN_BUDGET_FINAL_ANSWER_HINT,
+ )
end
it "keeps the caller execution context intact on error" do
@@ -515,12 +561,21 @@ RSpec.describe DiscourseAi::Agents::Bot do
raw_context << [user_message[:content], user_message[:id], "user"]
raw_context << [model_message[:content], nil, "model"]
end
+ hints = described_class::TRANSIENT_TOKEN_BUDGET_HINTS
+ hints.each do |hint|
+ messages << { type: :user, content: hint }
+ raw_context << [hint, nil, "user"]
+ end
prompt = DiscourseAi::Completions::Prompt.new(messages: messages, tools: [])
llm = bot.send(:llm)
allow(llm).to receive(:max_prompt_tokens).and_return(2000)
allow(llm).to receive(:tokenizer).and_return(DiscourseAi::Tokenizer::OpenAiTokenizer)
- allow(llm).to receive(:generate).and_return("Summary of the conversation.")
+ compression_prompt_messages = nil
+ allow(llm).to receive(:generate) do |compression_prompt|
+ compression_prompt_messages = compression_prompt.messages
+ "Summary of the conversation."
+ end
bot.send(:maybe_compress_context, prompt, llm, raw_context: raw_context)
@@ -531,6 +586,11 @@ RSpec.describe DiscourseAi::Agents::Bot do
["Understood, I have the context.", nil, "model", nil, nil],
)
expect(raw_context.flatten.join).not_to include("Message 0")
+ expect(compression_prompt_messages.map { |message| message[:content] }).not_to include(
+ *hints,
+ )
+ expect(prompt.messages.map { |message| message[:content] }).to include(*hints)
+ expect(raw_context.flatten).not_to include(*hints)
end
it "skips compression when under threshold" do
@@ -644,6 +704,28 @@ RSpec.describe DiscourseAi::Agents::Bot do
expect(prompt.messages[1][:content]).to include("<compressed_context>")
end
+ it "skips compression when removing a legacy hint makes the compression prompt invalid" do
+ bot = described_class.as(bot_user, agent: agent_class.new)
+ messages = [{ type: :system, content: "You are a bot" }]
+ 10.times do |index|
+ content =
+ if index == 3
+ described_class::LEGACY_BUDGET_EXHAUSTED_HINT
+ else
+ "Message #{index} " * 200
+ end
+ messages << { type: :user, content: content }
+ messages << { type: :model, content: "Response #{index} " * 200 }
+ end
+
+ prompt = DiscourseAi::Completions::Prompt.new(messages: messages, tools: [])
+ llm = bot.send(:llm)
+ allow(llm).to receive(:max_prompt_tokens).and_return(2000)
+ allow(llm).to receive(:tokenizer).and_return(DiscourseAi::Tokenizer::OpenAiTokenizer)
+
+ expect(bot.send(:maybe_compress_context, prompt, llm)).to eq(:skipped)
+ end
+
it "skips compression when summarization returns blank" do
bot = described_class.as(bot_user, agent: agent_class.new)
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
index c1eac65f331..9033767f78b 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
@@ -65,9 +65,21 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
id: "tool_1",
)
+ generated_requests = []
DiscourseAi::Completions::Llm.with_prepared_responses(
- [tool_call, "Final answer after budget pre-check."],
+ [tool_call, "Final answer after budget pre-check.", "Test title"],
) do
+ allow_any_instance_of(DiscourseAi::Completions::Llm).to receive(
+ :generate,
+ ).and_wrap_original do |original, *args, **kwargs, &blk|
+ prompt = args.first
+ generated_requests << {
+ tool_choice: prompt.tool_choice,
+ last_message: prompt.messages.last.dup,
+ }
+ original.call(*args, **kwargs, &blk)
+ end
+
session = build_session
events = collect_events(session)
@@ -97,6 +109,18 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
partials = resumed_events.select { |type, _| type == :partial }.map { |_, data| data }
expect(partials.join).to eq("Final answer after budget pre-check.")
+ finalization_request =
+ generated_requests.find do |request|
+ request[:last_message][:content] ==
+ DiscourseAi::Agents::Bot::TOKEN_BUDGET_FINAL_ANSWER_HINT
+ end
+ expect(finalization_request).to eq(
+ tool_choice: :none,
+ last_message: {
+ type: :user,
+ content: DiscourseAi::Agents::Bot::TOKEN_BUDGET_FINAL_ANSWER_HINT,
+ },
+ )
tool_events = resumed_events.select { |type, _| type == :tool_calls }
expect(tool_events).to be_empty