Skip to content

A platform for community discussion. Free, open, simple.

FIX: Track completion limits across resumed turns (#44581)

Sameeb08e090f84
diff --git a/plugins/discourse-ai/lib/agents/subagent_execution_state.rb b/plugins/discourse-ai/lib/agents/subagent_execution_state.rb
index 8df491be2af..56e155793a0 100644
--- a/plugins/discourse-ai/lib/agents/subagent_execution_state.rb
+++ b/plugins/discourse-ai/lib/agents/subagent_execution_state.rb
@@ -8,11 +8,11 @@ module DiscourseAi
attr_reader :execution_context, :root_token_budget
- def initialize(execution_context:, root_token_budget:)
+ def initialize(execution_context:, root_token_budget:, completion_count: 0)
@execution_context = execution_context
@root_token_budget = root_token_budget
@spawn_count = 0
- @completion_count = 0
+ @completion_count = completion_count
execution_context.work_budget ||=
DiscourseAi::Completions::TurnWorkBudget.new(limit: root_token_budget)
execution_context.work_execution_state = self
@@ -32,6 +32,10 @@ module DiscourseAi
end
end
+ def completion_count
+ @mutex.synchronize { @completion_count }
+ end
+
def completion_available?
@mutex.synchronize { @completion_count < MAX_COMPLETIONS - 1 }
end
diff --git a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
index 880fc3f6ec4..f2ae5d0f063 100644
--- a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
+++ b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
@@ -118,6 +118,7 @@ module DiscourseAi
"accumulated_response_tokens" => @accumulated_response_tokens,
"accumulated_tokens" => @accumulated_tokens,
"preparation_tokens" => @preparation_tokens,
+ "completion_count" => @completion_count,
),
)
raise
@@ -269,9 +270,8 @@ module DiscourseAi
@protected_user_index =
payload["protected_user_index"] ||
@prompt.messages.rindex { |entry| entry[:type] == :user }
- @preparation_elapsed_seconds = payload["preparation_elapsed_seconds"].to_f
- @preparation_calls = payload["preparation_calls"].to_i
- @preparation_spent_tokens = payload["preparation_spent_tokens"].to_i
+ # Resume rounds provide a lower bound when a payload lacks an execution count.
+ @completion_count = payload.fetch("completion_count", @round_count).to_i
@preparation_tokens = payload["preparation_tokens"].to_i
work = payload.fetch("work_budget")
@work_budget =
@@ -321,6 +321,12 @@ module DiscourseAi
token_usage_tracker: token_usage_tracker,
work_budget: @work_budget,
)
+ @execution_state =
+ DiscourseAi::Agents::SubagentExecutionState.new(
+ execution_context: execution_context,
+ root_token_budget: @work_budget.limit,
+ completion_count: @completion_count.to_i,
+ )
generate_options = {
user: @user,
temperature: @temperature,
@@ -333,9 +339,6 @@ module DiscourseAi
DiscourseAi::Completions::ContextPreparation.new(
llm,
threshold: resolve_agent_record&.compression_threshold,
- calls: @preparation_calls.to_i,
- spent_tokens: @preparation_spent_tokens.to_i,
- elapsed_seconds: @preparation_elapsed_seconds.to_f,
)
# Work admission is independent of history length and cache usage.
if @work_budget.remaining <= 0
@@ -475,39 +478,59 @@ module DiscourseAi
end
def generate_with_work_budget!(execution_context, options, final: false, &block)
- options =
- @work_budget.generation_options(
- options.merge(
- thinking_allowance:
- DiscourseAi::Completions::ThinkingConfig.response_allowance_for(
- llm_model: @bot.model,
- effort: options[:thinking_effort],
- ),
- ),
- maximum: @bot.model.max_output_tokens.presence,
- final: final,
- tools: @prompt.tools.present?,
- root: true,
- )
- prepare_prompt!(execution_context, options)
+ if !final && !@execution_state.completion_available?
+ request_completion_limit_final_answer!
+ final = true
+ end
+
+ generation_options = work_generation_options(options, final: final)
+ # Maintenance can use the last ordinary slot; reserve the reply only after it fits.
+ prepare_prompt!(execution_context, generation_options)
+ if !final && !@execution_state.reserve_completion
+ request_completion_limit_final_answer!
+ final = true
+ generation_options = work_generation_options(options, final: final)
+ size, capacity =
+ @bot.llm.prompt_capacity(
+ @prompt,
+ **generation_options.slice(
+ :max_tokens,
+ :thinking_effort,
+ :response_format,
+ :max_tokens_is_total,
+ ),
+ )
+ # Recheck the final hint/output allowance without another soft-threshold compression.
+ prepare_prompt!(execution_context, generation_options) if capacity <= 0 || size > capacity
+ end
+ if final && !@execution_state.reserve_root_final_completion
+ raise DiscourseAi::Completions::ContextPreparation::Error.new("completion_limit")
+ end
+
_, _, provider_output =
@bot.llm.prompt_capacity(
@prompt,
- **options.slice(:max_tokens, :thinking_effort, :max_tokens_is_total),
+ **generation_options.slice(
+ :max_tokens,
+ :thinking_effort,
+ :response_format,
+ :max_tokens_is_total,
+ ),
)
reservation =
@work_budget.reserve_generation_output(
provider_output,
- max_tokens: options[:max_tokens],
+ max_tokens: generation_options[:max_tokens],
final: final,
root: true,
)
if !reservation
raise DiscourseAi::Completions::ContextPreparation::Error.new("turn_budget_exhausted")
end
- @bot.llm.generate(@prompt, **options, work_generation_admitted: true, &block)
+ @bot.llm.generate(@prompt, **generation_options, work_generation_admitted: true, &block)
ensure
@work_budget.release_output(reservation)
+ @completion_count = @execution_state.completion_count
tracker = execution_context.token_usage_tracker
@accumulated_request_tokens = tracker.request
@accumulated_response_tokens = tracker.response
@@ -515,6 +538,30 @@ module DiscourseAi
@preparation_tokens = tracker.preparation_tokens
end
+ def work_generation_options(options, final:)
+ @work_budget.generation_options(
+ options.merge(
+ thinking_allowance:
+ DiscourseAi::Completions::ThinkingConfig.response_allowance_for(
+ llm_model: @bot.model,
+ effort: options[:thinking_effort],
+ ),
+ ),
+ maximum: @bot.model.max_output_tokens.presence,
+ final: final,
+ tools: @prompt.tools.present?,
+ root: true,
+ )
+ end
+
+ def request_completion_limit_final_answer!
+ @prompt.push(
+ type: :user,
+ content: DiscourseAi::Agents::Bot::COMPLETION_LIMIT_FINAL_ANSWER_HINT,
+ )
+ @prompt.tool_choice = :none
+ end
+
def prepare_prompt!(execution_context, options)
@history_snapshot.verify!
result =
@@ -522,6 +569,7 @@ module DiscourseAi
@prompt,
user: @user,
execution_context: execution_context,
+ subagent_execution_state: @execution_state,
protected_user_index: @protected_user_index,
**options.slice(:max_tokens, :thinking_effort, :response_format, :max_tokens_is_total),
feature_context: {
@@ -647,9 +695,7 @@ module DiscourseAi
history_snapshot: @history_snapshot.metadata,
evidence_scope: @history_snapshot.evidence_scope,
protected_user_index: @protected_user_index,
- preparation_elapsed_seconds: @preparation.elapsed_seconds,
- preparation_calls: @preparation.calls,
- preparation_spent_tokens: @preparation.spent_tokens,
+ completion_count: @completion_count,
preparation_tokens: @preparation_tokens,
current_user_id: @current_user&.id,
agent_id: @agent.id,
diff --git a/plugins/discourse-ai/lib/completions/context_preparation.rb b/plugins/discourse-ai/lib/completions/context_preparation.rb
index c578378a98e..2385c763a30 100644
--- a/plugins/discourse-ai/lib/completions/context_preparation.rb
+++ b/plugins/discourse-ai/lib/completions/context_preparation.rb
@@ -3,9 +3,6 @@
module DiscourseAi
module Completions
class ContextPreparation
- MAX_CALLS = 8
- MAX_SECONDS = 90
- MAX_ALLOWANCE_TOKENS = 1_000_000
RETAINED_ATTACHMENTS_ACK = "The preceding attachments are retained historical evidence."
HISTORY_DROPPED_NOTICE = <<~TEXT.squish
Part of the earlier conversation was omitted because it could not be summarized to fit
@@ -15,34 +12,6 @@ module DiscourseAi
PREVIOUS_SUMMARY_LABEL =
"\nSummary from an earlier checkpoint (it does not cover messages omitted after it):\n"
- class Budget
- attr_reader :calls, :spent_tokens, :elapsed_seconds
-
- def initialize(allowance:, calls: 0, spent_tokens: 0, elapsed_seconds: 0)
- @allowance = allowance
- @calls = calls
- @spent_tokens = spent_tokens
- @elapsed_seconds = elapsed_seconds
- @mutex = Mutex.new
- end
-
- def reserve(tokens)
- @mutex.synchronize do
- if @calls >= MAX_CALLS || @spent_tokens + tokens > @allowance ||
- @elapsed_seconds >= MAX_SECONDS
- return false
- end
- @calls += 1
- @spent_tokens += tokens
- true
- end
- end
-
- def record_elapsed(seconds)
- @mutex.synchronize { @elapsed_seconds += [seconds, 0].max }
- end
- end
-
class Error < StandardError
attr_reader :reason
attr_accessor :raw_context
@@ -71,29 +40,9 @@ module DiscourseAi
attr_reader :protected_user_index
- def calls
- @budget.calls
- end
-
- def spent_tokens
- @budget.spent_tokens
- end
-
- def elapsed_seconds
- @budget.elapsed_seconds
- end
-
- def initialize(llm, threshold: 80, calls: 0, spent_tokens: 0, elapsed_seconds: 0)
+ def initialize(llm, threshold: 80)
@llm = llm
@threshold = threshold || 80
- allowance = [[llm.max_prompt_tokens.to_i * 2, 64_000].max, MAX_ALLOWANCE_TOKENS].min
- @budget =
- Budget.new(
- allowance: allowance,
- calls: calls,
- spent_tokens: spent_tokens,
- elapsed_seconds: elapsed_seconds,
- )
end
def prepare!(
@@ -106,7 +55,6 @@ module DiscourseAi
protected_user_index: nil,
**options
)
- @budget = execution_context.share_preparation_budget(@budget) if execution_context
return :cancelled if cancel_manager&.cancelled?
prompt.skip_trim = true
size, capacity = @llm.prompt_capacity(prompt, **options)
@@ -320,7 +268,6 @@ module DiscourseAi
end
def summarize(source, **options)
- started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
evidence = @llm.context_evidence(source)
remaining = evidence.to_json
offset = 0
@@ -373,15 +320,11 @@ module DiscourseAi
compression_prompt = build_prompt.call(low)
remaining.slice!(0, low)
offset += low
- input_size, input_capacity, output_reserve = @llm.prompt_capacity(compression_prompt)
+ input_size, input_capacity = @llm.prompt_capacity(compression_prompt)
raise Error.new("summary_input_overflow") if input_size > input_capacity
- now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
- @budget.record_elapsed(now - started_at)
- started_at = now
return if options[:cancel_manager]&.cancelled?
- if [email protected](input_size + output_reserve)
- raise Error.new("preparation_budget_exhausted")
- end
+ # Maintenance shares the execution completion limit; latency and estimated
+ # token costs must not discard a usable summary or leave evidence uncovered.
if options[:subagent_execution_state] &&
!options[:subagent_execution_state].reserve_completion
raise Error.new("completion_limit")
@@ -396,17 +339,9 @@ module DiscourseAi
execution_context: options[:execution_context],
cancel_manager: options[:cancel_manager],
) { |partial| summary << partial if partial.is_a?(String) }
- now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
- @budget.record_elapsed(now - started_at)
- started_at = now
- raise Error.new("preparation_timeout") if elapsed_seconds >= MAX_SECONDS
raise Error.new("unusable_summary") if summary.blank?
end
summary
- ensure
- if started_at
- @budget.record_elapsed(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at)
- end
end
def transient_hint?(message)
diff --git a/plugins/discourse-ai/lib/completions/execution_context.rb b/plugins/discourse-ai/lib/completions/execution_context.rb
index 373d3f67653..1f9ec57924e 100644
--- a/plugins/discourse-ai/lib/completions/execution_context.rb
+++ b/plugins/discourse-ai/lib/completions/execution_context.rb
@@ -25,7 +25,6 @@ module DiscourseAi
@audit_logger = audit_logger
@structured_audit_logger = structured_audit_logger
@upload_skips = []
- @preparation_state = { mutex: Mutex.new, budget: nil }
end
def for_new_turn
@@ -36,10 +35,6 @@ module DiscourseAi
)
end
- def share_preparation_budget(budget)
- @preparation_state[:mutex].synchronize { @preparation_state[:budget] ||= budget }
- end
-
def settle_generation(tokens: nil, tokenizer:, complete: true)
return if generation_settled || !generated_output
exposed_tokens = generated_output.size(tokenizer)
diff --git a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
index 5211cc34d14..f9ed6ab37e9 100644
--- a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
@@ -58,44 +58,36 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
expect(request).to have_been_requested.once
end
- it "shares maintenance bounds across contexts forked before their first preparation" do
- stub_const(described_class, :MAX_CALLS, 2) do
- root =
- DiscourseAi::Completions::ExecutionContext.new(
- work_budget: DiscourseAi::Completions::TurnWorkBudget.new(limit: 4000, used: 200),
- )
- first = root.dup
- second = root.dup
- DiscourseAi::Completions::Llm.with_prepared_responses(
- ["Summary retains QUARTZ-OTTER-731"] * described_class::MAX_CALLS,
- ) do |canned|
- llm = model.to_llm
- described_class::MAX_CALLS.times do |index|
- preparation = described_class.new(llm, threshold: 5)
- expect(
- preparation.prepare!(
- history_prompt(repetitions: 200),
- user: user,
- execution_context: index.even? ? first : second,
- ),
- ).to eq(:compressed)
- expect(preparation.calls).to eq(index + 1)
- end
- # The prompt still fits hard capacity, so maintenance exhaustion keeps it intact.
- last = history_prompt(repetitions: 200)
- original = last.messages.deep_dup
+ it "shares execution completion limits across forked contexts without a preparation cap" do
+ tracker = DiscourseAi::Completions::TokenUsageTracker.new
+ root =
+ DiscourseAi::Completions::ExecutionContext.new(
+ token_usage_tracker: tracker,
+ work_budget: DiscourseAi::Completions::TurnWorkBudget.new(limit: 4000, used: 200),
+ )
+ state =
+ DiscourseAi::Agents::SubagentExecutionState.new(
+ execution_context: root,
+ root_token_budget: 4000,
+ )
+ contexts = [root.dup, root.dup]
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ Array.new(10, "Summary retains QUARTZ-OTTER-731"),
+ ) do |canned|
+ 10.times do |index|
expect(
- described_class.new(llm, threshold: 5).prepare!(
- last,
+ described_class.new(model.to_llm, threshold: 5).prepare!(
+ history_prompt(repetitions: 200),
user: user,
- execution_context: second,
+ execution_context: contexts[index % 2],
+ subagent_execution_state: state,
),
- ).to eq(:skipped)
- expect(last.messages).to eq(original)
- expect(canned.completions).to eq(described_class::MAX_CALLS)
+ ).to eq(:compressed)
end
- expect(root.work_budget.used).to eq(200)
+ expect(canned.completions).to eq(10)
end
+ expect(state.completion_count).to eq(10)
+ expect(root.work_budget.used).to eq(200)
end
it "compacts a small message count without summarizing the latest request" do
@@ -124,7 +116,12 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
end
it "keeps the intact prompt after a soft-threshold summarization failure" do
- ["", RuntimeError.new("summary failed"), "inflated " * 10_000].each do |response|
+ [
+ "",
+ RuntimeError.new("summary failed"),
+ Net::ReadTimeout.new,
+ "inflated " * 10_000,
+ ].each do |response|
prompt = history_prompt(repetitions: 600)
original = prompt.messages.deep_dup
DiscourseAi::Completions::Llm.with_prepared_responses([response]) do
@@ -296,15 +293,22 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
end
end
- it "preserves every fragment of source larger than the model window through bounded merges" do
- prompt = history_prompt(repetitions: 5000)
- original_result = prompt.messages[3][:content]
+ it "covers all evidence beyond eight chunks and the former estimated token allowance" do
+ prompt = history_prompt(repetitions: 20_000)
+ original_evidence = JSON.parse(model.to_llm.context_evidence(prompt.messages[1...-1]).to_json)
DiscourseAi::Completions::Llm.with_prepared_responses(
- Array.new(8, "Summary with QUARTZ-OTTER-731."),
+ Array.new(30, "Summary with QUARTZ-OTTER-731."),
) do |canned, _, prompts|
llm = DiscourseAi::Completions::Llm.proxy(model)
expect(described_class.new(llm).prepare!(prompt)).to eq(:compressed)
- expect(canned.completions).to be_between(2, 8)
+ expect(canned.completions).to be > 8
+ estimated_tokens =
+ prompts.sum do |request|
+ size, capacity, output_reserve = llm.prompt_capacity(request)
+ expect(size).to be <= capacity
+ size + output_reserve
+ end
+ expect(estimated_tokens).to be > 64_000
fragments =
prompts.map do |request|
request.messages.last[:content].split(
@@ -314,8 +318,9 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
end
evidence =
JSON.parse(fragments.map { |fragment| JSON.parse(fragment)["evidence_fragment"] }.join)
- expect(evidence.find { |message| message["type"] == "tool" }["content"]).to eq(
- original_result,
+ expect(evidence).to eq(original_evidence)
+ expect(prompt.messages[1][:content]).to eq(
+ "<compressed_context>Summary with QUARTZ-OTTER-731.</compressed_context>",
)
expect(prompts.map(&:skip_trim)).to all(eq(true))
end
@@ -342,22 +347,23 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
},
)
- expect(described_class.new(model.to_llm).prepare!(prompt, user: user)).to eq(:compressed)
+ tracker = DiscourseAi::Completions::TokenUsageTracker.new
+ execution = DiscourseAi::Completions::ExecutionContext.new(token_usage_tracker: tracker)
+ expect(
+ described_class.new(model.to_llm).prepare!(
+ prompt,
+ user: user,
+ execution_context: execution,
+ ),
+ ).to eq(:compressed)
logs = AiApiAuditLog.where(feature_name: "context_compression").order(:id).to_a
- expect(logs.size).to be_between(2, described_class::MAX_CALLS)
+ expect(logs.size).to be >= 2
expect(logs.map { |log| [log.topic_id, log.post_id] }).to all(eq([post.topic_id, post.id]))
expect(logs.last.prev_log_id).to eq(logs[-2].id)
- end
-
- it "bounds maintenance calls and drops hard-overflow history on exhaustion" do
- prompt = history_prompt(repetitions: 5000)
- DiscourseAi::Completions::Llm.with_prepared_responses(["summary"]) do |canned|
- llm = DiscourseAi::Completions::Llm.proxy(model)
- expect(described_class.new(llm, calls: 8).prepare!(prompt)).to eq(:compressed)
- expect_history_dropped(prompt)
- expect(canned.completions).to eq(0)
- end
+ expect(tracker.request).to eq(logs.size * 100)
+ expect(tracker.response).to eq(logs.size * 20)
+ expect(tracker.preparation_tokens).to eq(tracker.total)
end
it "keeps parallel batches, provider data and thinking intact in the protected tail" do
@@ -425,17 +431,20 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
end
end
- it "keeps applicable quota failures visible rather than treating them as a soft fallback" do
- prompt = history_prompt(repetitions: 600)
- original = prompt.messages.deep_dup
- DiscourseAi::Completions::Llm.with_prepared_responses(
- [LlmQuotaUsage::QuotaExceededError.new("quota exhausted")],
- ) do
- llm = DiscourseAi::Completions::Llm.proxy(model)
- expect {
- described_class.new(llm, threshold: 20).prepare!(prompt, user: user)
- }.to raise_error(LlmQuotaUsage::QuotaExceededError)
- expect(prompt.messages).to eq(original)
+ it "keeps quota and credit failures visible rather than treating them as a soft fallback" do
+ [
+ LlmQuotaUsage::QuotaExceededError.new("quota exhausted"),
+ LlmCreditAllocation::CreditLimitExceeded.new("credits exhausted"),
+ ].each do |error|
+ prompt = history_prompt(repetitions: 600)
+ original = prompt.messages.deep_dup
+ DiscourseAi::Completions::Llm.with_prepared_responses([error]) do
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+ expect {
+ described_class.new(llm, threshold: 20).prepare!(prompt, user: user)
+ }.to raise_error(error.class)
+ expect(prompt.messages).to eq(original)
+ end
end
end
@@ -547,21 +556,39 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
end
end
- it "leaves a late maintenance response unpublished after the preparation deadline" do
- prompt = history_prompt(repetitions: 600)
- original = prompt.messages.deep_dup
+ it "accepts a slow successful summary of current-turn evidence without earlier history to drop" do
+ result = "Completed action evidence " * 8000
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System",
+ messages: [
+ { type: :user, content: "Read and explain the result" },
+ { type: :tool_call, id: "read", name: "read", content: "{}" },
+ { type: :tool, id: "read", name: "read", content: result },
+ ],
+ )
now = 0
allow(Process).to receive(:clock_gettime).with(Process::CLOCK_MONOTONIC) { now }
- DiscourseAi::Completions::Llm.with_prepared_responses(["Late summary."]) do |canned|
- llm = DiscourseAi::Completions::Llm.proxy(model)
- allow(llm).to receive(:generate).and_wrap_original do |generate, *args, **options|
- result = generate.call(*args, **options)
- now = 91
- result
+ summary = "The completed read contains important evidence."
+ DiscourseAi::Completions::Llm.with_prepared_responses(Array.new(10, summary)) do |canned|
+ llm = model.to_llm
+ allow(llm).to receive(:generate).and_wrap_original do |generate, *args, **options, &block|
+ response = generate.call(*args, **options, &block)
+ now += 91
+ response
end
- expect(described_class.new(llm, threshold: 20).prepare!(prompt)).to eq(:skipped)
- expect(canned.completions).to eq(1)
- expect(prompt.messages).to eq(original)
+ expect(described_class.new(llm).prepare!(prompt)).to eq(:compressed)
+ expect(canned.completions).to be > 0
+ expect(prompt.messages.map { |message| message[:content] }).to eq(
+ [
+ "System",
+ "<compressed_context>#{summary}</compressed_context>",
+ DiscourseAi::Completions::PromptMessagesBuilder::COMPRESSED_CONTEXT_ACK,
+ "Read and explain the result",
+ ],
+ )
+ size, capacity = llm.prompt_capacity(prompt)
+ expect(size).to be <= capacity
end
end
@@ -570,8 +597,9 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
original = prompt.messages.deep_dup
cancel_manager = DiscourseAi::Completions::CancelManager.new
llm = DiscourseAi::Completions::Llm.proxy(model)
- allow(llm).to receive(:generate) do |_, **options|
+ allow(llm).to receive(:generate) do |_, **options, &block|
expect(options[:cancel_manager]).to eq(cancel_manager)
+ block.call("summary")
cancel_manager.cancel!
"summary"
end
@@ -581,32 +609,6 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
expect(prompt.messages).to eq(original)
end
- it "charges only maintenance time, not ordinary tool work, and shares the allowance across preparations" do
- now = 0.0
- allow(Process).to receive(:clock_gettime).with(Process::CLOCK_MONOTONIC) { now }
- execution = DiscourseAi::Completions::ExecutionContext.new
- DiscourseAi::Completions::Llm.with_prepared_responses(
- Array.new(8, "Concise evidence"),
- ) do |canned|
- llm = DiscourseAi::Completions::Llm.proxy(model)
- preparation = described_class.new(llm, threshold: 20, calls: 7)
- now = 120.0
- expect(
- preparation.prepare!(history_prompt(repetitions: 600), execution_context: execution),
- ).to eq(:compressed)
- expect(preparation.calls).to eq(8)
- expect(preparation.elapsed_seconds).to eq(0)
-
- child_preparation = described_class.new(llm, threshold: 20)
- prompt = history_prompt(repetitions: 4000)
- expect(child_preparation.prepare!(prompt, execution_context: execution)).to eq(:compressed)
- expect(child_preparation.calls).to eq(8)
- expect(child_preparation.spent_tokens).to eq(preparation.spent_tokens)
- expect_history_dropped(prompt)
- expect(canned.completions).to eq(1)
- end
- end
-
it "skips checkpoint-only maintenance when the protected current request is above the soft threshold" do
prompt =
DiscourseAi::Completions::Prompt.new(
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
index fa84b91175d..3573b359ccd 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
@@ -557,15 +557,11 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
expect(described_class.resume_state_exists?(token)).to eq(false)
end
- it "restores the cumulative maintenance time and call limit rather than refreshing them on resume" do
+ it "ignores retired preparation budget fields when resuming a legacy payload" do
llm.update!(max_prompt_tokens: 16_000)
+ ai_agent.update!(max_turn_tokens: 200_000)
call =
- DiscourseAi::Completions::ToolCall.new(
- name: "client_tool",
- parameters: {
- },
- id: "resume-budget",
- )
+ DiscourseAi::Completions::ToolCall.new(name: "client_tool", parameters: {}, id: "legacy")
token = nil
DiscourseAi::Completions::Llm.with_prepared_responses([call]) do
token =
@@ -573,26 +569,301 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
end
key = described_class.redis_key(token)
state = JSON.parse(Discourse.redis.get(key))
- expect(state["preparation_elapsed_seconds"]).to eq(0)
- state["preparation_calls"] = 8
- state["preparation_elapsed_seconds"] = 80
+ state.delete("completion_count")
+ state.merge!(
+ "preparation_calls" => 8,
+ "preparation_elapsed_seconds" => 1000,
+ "preparation_spent_tokens" => 10_000_000,
+ "preparation_tokens" => 123,
+ )
Discourse.redis.set(key, state.to_json)
- DiscourseAi::Completions::Llm.with_prepared_responses(["unused"]) do |_, _, _, options|
- expect {
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ ["First summary", "Complete summary", call],
+ ) do |_, _, _, options|
+ events =
collect_events(
build_session(
resume_token: token,
tool_results: [{ tool_call_id: call.id, content: "Oversized result " * 6000 }],
),
)
- }.to raise_error(
- DiscourseAi::Completions::ContextPreparation::Error,
- /preparation_budget_exhausted/,
+ expect(events.find { |type, _| type == :tool_calls }).to be_present
+ expect(options.map { |option| option[:feature_name] }).to eq(
+ %w[context_compression context_compression bot],
+ )
+ end
+ resumed_state = JSON.parse(Discourse.redis.get(key))
+ expect(resumed_state["completion_count"]).to eq(4)
+ expect(resumed_state["preparation_tokens"]).to eq(123)
+ expect(
+ resumed_state.keys &
+ %w[preparation_calls preparation_elapsed_seconds preparation_spent_tokens],
+ ).to eq([])
+ end
+
+ it "persists compaction completions across resumes and preserves the root final completion" do
+ llm.update!(max_prompt_tokens: 16_000)
+ ai_agent.update!(max_turn_tokens: 200_000)
+ call =
+ DiscourseAi::Completions::ToolCall.new(name: "client_tool", parameters: {}, id: "limited")
+ token = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses([call]) do
+ token =
+ collect_events(build_session).find { |type, _| type == :tool_calls }[1][:resume_token]
+ end
+ key = described_class.redis_key(token)
+ state = JSON.parse(Discourse.redis.get(key))
+ state["completion_count"] = 96
+ Discourse.redis.set(key, state.to_json)
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ ["First summary", "Complete summary", call],
+ ) do
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: "Oversized result " * 6000 }],
+ ),
+ )
+ end
+ expect(JSON.parse(Discourse.redis.get(key))["completion_count"]).to eq(99)
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ ["Final answer at the execution limit", "Test title"],
+ ) do |_, _, prompts|
+ events =
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: "Done" }],
+ ),
+ )
+ expect(events.select { |type, _| type == :partial }.map(&:last).join).to eq(
+ "Final answer at the execution limit",
+ )
+ expect(prompts.first.tool_choice).to eq(:none)
+ expect(prompts.first.messages[-2][:content]).to eq(
+ DiscourseAi::Agents::Bot::COMPLETION_LIMIT_FINAL_ANSWER_HINT,
+ )
+ expect(described_class.resume_state_exists?(token)).to eq(false)
+ end
+ end
+
+ it "uses the last ordinary slot for current-turn maintenance and the reserved slot for a text-only final" do
+ llm.update!(max_prompt_tokens: 16_000)
+ ai_agent.update!(max_turn_tokens: 200_000)
+ call =
+ DiscourseAi::Completions::ToolCall.new(name: "client_tool", parameters: {}, id: "boundary")
+ query = "request " * 11_000
+ token = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses([call]) do
+ token =
+ collect_events(build_session(query: query)).find { |type, _| type == :tool_calls }[1][
+ :resume_token
+ ]
+ end
+ key = described_class.redis_key(token)
+ state = JSON.parse(Discourse.redis.get(key))
+ expect(state["prompt"]["messages"].map { |message| message["type"] }).to eq(%w[system user])
+ state["completion_count"] = 98
+ Discourse.redis.set(key, state.to_json)
+ summary = "The completed caller action returned useful evidence."
+ final = "Final answer using the completed action"
+ result = "Completed evidence " * 2000
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [summary, final, "Test title"],
+ ) do |_, _, prompts, options|
+ events =
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: result }],
+ ),
+ )
+ expect(options.map { |option| option[:feature_name] }.first(2)).to eq(
+ %w[context_compression bot],
+ )
+ expect(prompts.first.messages.last[:content]).to include(result)
+ expect(prompts[1].messages[1][:content]).to eq(
+ "<compressed_context>#{summary}</compressed_context>",
+ )
+ expect(prompts[1].tool_choice).to eq(:none)
+ expect(prompts[1].messages.map { |message| message[:content] }).to include(
+ DiscourseAi::Agents::Bot::COMPLETION_LIMIT_FINAL_ANSWER_HINT,
)
- expect(options.none? { |option| option[:feature_name] == "context_compression" }).to eq(
- true,
+ expect(options[1]).to include(max_tokens: 4096, thinking_effort: "none")
+ size, capacity = llm.to_llm.prompt_capacity(prompts[1], **options[1])
+ expect(size).to be_between(capacity * 0.8, capacity)
+ expect(events.select { |type, _| type == :partial }.map(&:last).join).to eq(final)
+ expect(events.select { |type, _| type == :tool_calls }).to eq([])
+ end
+ expect(described_class.resume_state_exists?(token)).to eq(false)
+ end
+
+ it "rejects a final whose changed output allowance no longer leaves room for the current request" do
+ llm.update!(max_prompt_tokens: 16_000)
+ ai_agent.update!(max_turn_tokens: 200_000)
+ call =
+ DiscourseAi::Completions::ToolCall.new(name: "client_tool", parameters: {}, id: "capacity")
+ token = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses([call]) do
+ token =
+ collect_events(build_session(query: "request " * 12_000)).find do |type, _|
+ type == :tool_calls
+ end[
+ 1
+ ][
+ :resume_token
+ ]
+ end
+ key = described_class.redis_key(token)
+ state = JSON.parse(Discourse.redis.get(key))
+ state["completion_count"] = 98
+ Discourse.redis.set(key, state.to_json)
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ ["Completed caller evidence summarized."],
+ ) do |canned, _, _, options|
+ expect {
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: "Completed evidence " * 2000 }],
+ ),
+ )
+ }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /hard_overflow/)
+ expect(canned.completions).to eq(1)
+ expect(options.map { |option| option[:feature_name] }).to eq(["context_compression"])
+ end
+ expect(described_class.resume_state_exists?(token)).to eq(false)
+ end
+
+ %w[maintenance final_reply].each do |failure_stage|
+ it "persists consumed completions when #{failure_stage} fails after access is revoked" do
+ llm.update!(max_prompt_tokens: 16_000)
+ ai_agent.update!(max_turn_tokens: 200_000)
+ call =
+ DiscourseAi::Completions::ToolCall.new(
+ name: "client_tool",
+ parameters: {
+ },
+ id: "failed-final",
+ )
+ token = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses([call]) do
+ token =
+ collect_events(build_session(query: "request " * 11_000)).find do |type, _|
+ type == :tool_calls
+ end[
+ 1
+ ][
+ :resume_token
+ ]
+ end
+ key = described_class.redis_key(token)
+ state = JSON.parse(Discourse.redis.get(key))
+ state["completion_count"] = 98
+ Discourse.redis.set(key, state.to_json)
+ topic = Topic.find(state["topic_id"])
+ result = "Completed evidence " * 2000
+ allow_any_instance_of(DiscourseAi::Completions::Llm).to receive(
+ :generate,
+ ).and_wrap_original do |generate, *args, **options, &block|
+ if options[:feature_name] == "context_compression"
+ options[:user].update!(admin: false, moderator: false)
+ topic.topic_allowed_users.where(user_id: admin.id).delete_all
+ end
+ generate.call(*args, **options, &block)
+ end
+ responses = [LlmCreditAllocation::CreditLimitExceeded.new("credits exhausted")]
+ responses.unshift("Completed caller evidence summarized.") if failure_stage == "final_reply"
+ DiscourseAi::Completions::Llm.with_prepared_responses(responses) do
+ expect {
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: result }],
+ ),
+ )
+ }.to raise_error(LlmCreditAllocation::CreditLimitExceeded)
+ end
+ failed_state = JSON.parse(Discourse.redis.get(key))
+ completion_count = failure_stage == "maintenance" ? 99 : 100
+ expect(failed_state["completion_count"]).to eq(completion_count)
+ expect(failed_state["failed_tool_results"]).to eq(
+ [{ "tool_call_id" => call.id, "content" => result }],
)
+ expect {
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: result }],
+ ),
+ )
+ }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /history_changed/)
+ expect(JSON.parse(Discourse.redis.get(key))["completion_count"]).to eq(completion_count)
+ end
+ end
+
+ it "preserves completed results when the execution completion limit blocks required compaction" do
+ llm.update!(max_prompt_tokens: 16_000)
+ call =
+ DiscourseAi::Completions::ToolCall.new(name: "client_tool", parameters: {}, id: "completed")
+ token = nil
+ topic_id = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses([call]) do
+ events = collect_events(build_session)
+ token = events.find { |type, _| type == :tool_calls }[1][:resume_token]
+ topic_id = events.find { |type, _| type == :context }[1][:topic_id]
+ end
+ key = described_class.redis_key(token)
+ state = JSON.parse(Discourse.redis.get(key))
+ state["completion_count"] = 99
+ Discourse.redis.set(key, state.to_json)
+ result = "Completed evidence " * 6000
+ DiscourseAi::Completions::Llm.with_prepared_responses(["unused"]) do |canned|
+ expect {
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: result }],
+ ),
+ )
+ }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /completion_limit/)
+ expect(canned.completions).to eq(0)
+ end
+ reply = Post.where(topic_id: topic_id, user: ai_agent.user).last
+ expect(reply.post_custom_prompt.custom_prompt.to_s).to include(result, call.id)
+ expect(described_class.resume_state_exists?(token)).to eq(false)
+ end
+
+ it "rejects a resumed generation when the reserved final completion has already been consumed" do
+ call =
+ DiscourseAi::Completions::ToolCall.new(name: "client_tool", parameters: {}, id: "exhausted")
+ token = nil
+ topic_id = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses([call]) do
+ events = collect_events(build_session)
+ token = events.find { |type, _| type == :tool_calls }[1][:resume_token]
+ topic_id = events.find { |type, _| type == :context }[1][:topic_id]
end
+ key = described_class.redis_key(token)
+ state = JSON.parse(Discourse.redis.get(key))
+ state["completion_count"] = 100
+ Discourse.redis.set(key, state.to_json)
+ result = "Already completed"
+ DiscourseAi::Completions::Llm.with_prepared_responses(["unused"]) do |canned|
+ expect {
+ collect_events(
+ build_session(
+ resume_token: token,
+ tool_results: [{ tool_call_id: call.id, content: result }],
+ ),
+ )
+ }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /completion_limit/)
+ expect(canned.completions).to eq(0)
+ end
+ reply = Post.where(topic_id: topic_id, user: ai_agent.user).last
+ expect(reply.post_custom_prompt.custom_prompt.to_s).to include(result, call.id)
+ expect(described_class.resume_state_exists?(token)).to eq(false)
end
it "prepares initial tool-heavy history and persists the prepared prompt while calls are pending" do
@@ -630,7 +901,7 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
expect(prompts.first.messages.last[:content]).to include(source.raw, old_result)
expect(state["prompt"]["messages"][1]["content"]).to include("<compressed_context>")
expect(state["expected_tool_calls"].map { |call| call["id"] }).to eq(["next"])
- expect(state["preparation_calls"]).to eq(1)
+ expect(state["completion_count"]).to eq(2)
end
end