Skip to content

A platform for community discussion. Free, open, simple.

DEV: Report specific context summary failures (#44612)

Samb38163c50399
diff --git a/plugins/discourse-ai/lib/completions/context_preparation.rb b/plugins/discourse-ai/lib/completions/context_preparation.rb
index 2385c763a30..19b11f955fe 100644
--- a/plugins/discourse-ai/lib/completions/context_preparation.rb
+++ b/plugins/discourse-ai/lib/completions/context_preparation.rb
@@ -129,9 +129,8 @@ module DiscourseAi
]
candidate = copy_prompt(prompt, messages)
candidate_size, candidate_capacity = @llm.prompt_capacity(candidate, **options)
- if candidate_size >= size || candidate_size > candidate_capacity
- raise Error.new("unusable_summary")
- end
+ raise Error.new("summary_not_smaller") if candidate_size >= size
+ raise Error.new("summary_over_capacity") if candidate_size > candidate_capacity
return :cancelled if cancel_manager&.cancelled?
@protected_user_index = messages.length - tail.length
prompt.messages.replace(messages)
@@ -339,7 +338,7 @@ module DiscourseAi
execution_context: options[:execution_context],
cancel_manager: options[:cancel_manager],
) { |partial| summary << partial if partial.is_a?(String) }
- raise Error.new("unusable_summary") if summary.blank?
+ raise Error.new("empty_summary") if summary.blank?
end
summary
end
diff --git a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
index f9ed6ab37e9..cc736ec03e5 100644
--- a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
@@ -133,7 +133,59 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
end
end
- it "drops the oldest history when an unusable summary leaves the prompt over capacity" do
+ it "logs empty_summary when compression produces only whitespace" do
+ prompt = history_prompt(repetitions: 600)
+ original = prompt.messages.deep_dup
+ logger =
+ track_log_messages do
+ DiscourseAi::Completions::Llm.with_prepared_responses([" \n\t"]) do
+ expect(described_class.new(model.to_llm, threshold: 20).prepare!(prompt)).to eq(
+ :skipped,
+ )
+ end
+ end
+
+ expect(logger.warnings).to include("DiscourseAi: context_preparation reason=empty_summary")
+ expect(prompt.messages).to eq(original)
+ end
+
+ it "logs summary_not_smaller when compression expands the prompt" do
+ prompt = history_prompt(repetitions: 600)
+ original = prompt.messages.deep_dup
+ logger =
+ track_log_messages do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["inflated " * 10_000]) do
+ expect(described_class.new(model.to_llm, threshold: 20).prepare!(prompt)).to eq(
+ :skipped,
+ )
+ end
+ end
+
+ expect(logger.warnings).to include(
+ "DiscourseAi: context_preparation reason=summary_not_smaller",
+ )
+ expect(prompt.messages).to eq(original)
+ end
+
+ it "logs summary_over_capacity when a smaller summary still exceeds the answering capacity" do
+ prompt = history_prompt(repetitions: 600)
+ logger =
+ track_log_messages do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["inflated " * 2500]) do
+ expect(described_class.new(model.to_llm).prepare!(prompt, max_tokens: 14_000)).to eq(
+ :compressed,
+ )
+ end
+ end
+
+ expect(logger.warnings).to include(
+ "DiscourseAi: context_preparation reason=summary_over_capacity",
+ "DiscourseAi: context_preparation reason=history_dropped original_reason=summary_over_capacity dropped_history_messages=4",
+ )
+ expect_history_dropped(prompt)
+ end
+
+ it "drops the oldest history when an empty summary leaves the prompt over capacity" do
prompt = history_prompt(repetitions: 4000)
DiscourseAi::Completions::Llm.with_prepared_responses([""]) do |canned|
llm = DiscourseAi::Completions::Llm.proxy(model)
@@ -270,7 +322,7 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
expect { described_class.new(llm).prepare!(prompt) }.to raise_error(
described_class::Error,
- /unusable_summary/,
+ /empty_summary/,
)
expect(prompt.messages).to eq(original)
end
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb
index 6cb237258a7..bc3bac0f35a 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb
@@ -2546,7 +2546,7 @@ RSpec.describe DiscourseAi::AiBot::Playground do
[["Partial answer before the action", call], ""],
) { small_window_playground.reply_to(source, auto_set_title: false) }
reply = source.topic.posts.order(:post_number).last
- expect(reply.raw).to include("Partial answer before the action", "unusable_summary")
+ expect(reply.raw).to include("Partial answer before the action", "empty_summary")
entries = reply.post_custom_prompt.custom_prompt
expect(entries.find { |entry| entry[2] == "tool" }[1]).to eq(call.id)
expect(entries.to_s).to include("Important tool evidence")
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
index 3573b359ccd..ca5a2de3ccc 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
@@ -548,10 +548,10 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
tool_results: [{ tool_call_id: call.id, content: result }],
),
)
- }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /unusable_summary/)
+ }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /empty_summary/)
end
reply = Post.where(topic_id: topic_id, user: ai_agent.user).last
- expect(reply.raw).to include("Partial answer before the action", "unusable_summary")
+ expect(reply.raw).to include("Partial answer before the action", "empty_summary")
expect(reply.post_custom_prompt.custom_prompt.to_s).to include(result, call.id)
expect(reply.post_custom_prompt.custom_prompt.to_s).not_to include("<compressed_context>")
expect(described_class.resume_state_exists?(token)).to eq(false)