A platform for community discussion. Free, open, simple.
DEV: Report specific context summary failures (#44612)
Samb38163c50399
diff --git a/plugins/discourse-ai/lib/completions/context_preparation.rb b/plugins/discourse-ai/lib/completions/context_preparation.rbindex 2385c763a30..19b11f955fe 100644--- a/plugins/discourse-ai/lib/completions/context_preparation.rb+++ b/plugins/discourse-ai/lib/completions/context_preparation.rb@@ -129,9 +129,8 @@ module DiscourseAi]candidate = copy_prompt(prompt, messages)candidate_size, candidate_capacity = @llm.prompt_capacity(candidate, **options)- if candidate_size >= size || candidate_size > candidate_capacity- raise Error.new("unusable_summary")- end+ raise Error.new("summary_not_smaller") if candidate_size >= size+ raise Error.new("summary_over_capacity") if candidate_size > candidate_capacityreturn :cancelled if cancel_manager&.cancelled?@protected_user_index = messages.length - tail.lengthprompt.messages.replace(messages)@@ -339,7 +338,7 @@ module DiscourseAiexecution_context: options[:execution_context],cancel_manager: options[:cancel_manager],) { |partial| summary << partial if partial.is_a?(String) }- raise Error.new("unusable_summary") if summary.blank?+ raise Error.new("empty_summary") if summary.blank?endsummaryenddiff --git a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rbindex f9ed6ab37e9..cc736ec03e5 100644--- a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb@@ -133,7 +133,59 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation doendend- it "drops the oldest history when an unusable summary leaves the prompt over capacity" do+ it "logs empty_summary when compression produces only whitespace" do+ prompt = history_prompt(repetitions: 600)+ original = prompt.messages.deep_dup+ logger =+ track_log_messages do+ DiscourseAi::Completions::Llm.with_prepared_responses([" \n\t"]) do+ expect(described_class.new(model.to_llm, threshold: 20).prepare!(prompt)).to eq(+ :skipped,+ )+ end+ end++ expect(logger.warnings).to include("DiscourseAi: context_preparation reason=empty_summary")+ expect(prompt.messages).to eq(original)+ end++ it "logs summary_not_smaller when compression expands the prompt" do+ prompt = history_prompt(repetitions: 600)+ original = prompt.messages.deep_dup+ logger =+ track_log_messages do+ DiscourseAi::Completions::Llm.with_prepared_responses(["inflated " * 10_000]) do+ expect(described_class.new(model.to_llm, threshold: 20).prepare!(prompt)).to eq(+ :skipped,+ )+ end+ end++ expect(logger.warnings).to include(+ "DiscourseAi: context_preparation reason=summary_not_smaller",+ )+ expect(prompt.messages).to eq(original)+ end++ it "logs summary_over_capacity when a smaller summary still exceeds the answering capacity" do+ prompt = history_prompt(repetitions: 600)+ logger =+ track_log_messages do+ DiscourseAi::Completions::Llm.with_prepared_responses(["inflated " * 2500]) do+ expect(described_class.new(model.to_llm).prepare!(prompt, max_tokens: 14_000)).to eq(+ :compressed,+ )+ end+ end++ expect(logger.warnings).to include(+ "DiscourseAi: context_preparation reason=summary_over_capacity",+ "DiscourseAi: context_preparation reason=history_dropped original_reason=summary_over_capacity dropped_history_messages=4",+ )+ expect_history_dropped(prompt)+ end++ it "drops the oldest history when an empty summary leaves the prompt over capacity" doprompt = history_prompt(repetitions: 4000)DiscourseAi::Completions::Llm.with_prepared_responses([""]) do |canned|llm = DiscourseAi::Completions::Llm.proxy(model)@@ -270,7 +322,7 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation doexpect { described_class.new(llm).prepare!(prompt) }.to raise_error(described_class::Error,- /unusable_summary/,+ /empty_summary/,)expect(prompt.messages).to eq(original)enddiff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rbindex 6cb237258a7..bc3bac0f35a 100644--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb@@ -2546,7 +2546,7 @@ RSpec.describe DiscourseAi::AiBot::Playground do[["Partial answer before the action", call], ""],) { small_window_playground.reply_to(source, auto_set_title: false) }reply = source.topic.posts.order(:post_number).last- expect(reply.raw).to include("Partial answer before the action", "unusable_summary")+ expect(reply.raw).to include("Partial answer before the action", "empty_summary")entries = reply.post_custom_prompt.custom_promptexpect(entries.find { |entry| entry[2] == "tool" }[1]).to eq(call.id)expect(entries.to_s).to include("Important tool evidence")diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rbindex 3573b359ccd..ca5a2de3ccc 100644--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb@@ -548,10 +548,10 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession dotool_results: [{ tool_call_id: call.id, content: result }],),)- }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /unusable_summary/)+ }.to raise_error(DiscourseAi::Completions::ContextPreparation::Error, /empty_summary/)endreply = Post.where(topic_id: topic_id, user: ai_agent.user).last- expect(reply.raw).to include("Partial answer before the action", "unusable_summary")+ expect(reply.raw).to include("Partial answer before the action", "empty_summary")expect(reply.post_custom_prompt.custom_prompt.to_s).to include(result, call.id)expect(reply.post_custom_prompt.custom_prompt.to_s).not_to include("<compressed_context>")expect(described_class.resume_state_exists?(token)).to eq(false)