Skip to content

A platform for community discussion. Free, open, simple.

FIX: Normalize multi-block LLM responses (#42459)

Sam61842fbc8f5e
diff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rb
index 3ac46c73aa6..661aef7db60 100644
--- a/plugins/discourse-ai/lib/agents/bot.rb
+++ b/plugins/discourse-ai/lib/agents/bot.rb
@@ -261,13 +261,8 @@ module DiscourseAi
if !tool_found
ongoing_chain = false
- text = result
-
# we must strip out thinking and other types of blocks
- if result.is_a?(Array)
- text = +""
- result.each { |item| text << item if item.is_a?(String) }
- end
+ text = DiscourseAi::Completions::Llm.text_from_response(result)
raw_context << [text, bot_user&.username]
end
@@ -612,7 +607,7 @@ module DiscourseAi
return :skipped
end
- summary = summary.is_a?(Array) ? summary.select { |s| s.is_a?(String) }.join : summary
+ summary = DiscourseAi::Completions::Llm.text_from_response(summary)
return :skipped if summary.blank?
summary_tokens = tokenizer.size(summary)
diff --git a/plugins/discourse-ai/lib/agents/tool_runner/llm.rb b/plugins/discourse-ai/lib/agents/tool_runner/llm.rb
index 9c70b63fc5f..fa9d9710ca8 100644
--- a/plugins/discourse-ai/lib/agents/tool_runner/llm.rb
+++ b/plugins/discourse-ai/lib/agents/tool_runner/llm.rb
@@ -24,17 +24,19 @@ module DiscourseAi
if response_format && !response_format.is_a?(Hash)
raise ::Discourse::InvalidParameters.new("response_format must be a hash")
end
- @llm.generate(
- convert_js_prompt_to_ruby(prompt),
- user: llm_user,
- feature_name: "custom_tool_#{tool.name}",
- response_format: response_format,
- temperature: options["temperature"],
- top_p: options["top_p"],
- max_tokens: options["max_tokens"],
- thinking_effort: options["thinking_effort"],
- stop_sequences: options["stop_sequences"],
- )
+ response =
+ @llm.generate(
+ convert_js_prompt_to_ruby(prompt),
+ user: llm_user,
+ feature_name: "custom_tool_#{tool.name}",
+ response_format: response_format,
+ temperature: options["temperature"],
+ top_p: options["top_p"],
+ max_tokens: options["max_tokens"],
+ thinking_effort: options["thinking_effort"],
+ stop_sequences: options["stop_sequences"],
+ )
+ DiscourseAi::Completions::Llm.text_from_response(response)
end
end,
)
diff --git a/plugins/discourse-ai/lib/agents/tools/summarize.rb b/plugins/discourse-ai/lib/agents/tools/summarize.rb
index a59d8180dd1..dc7c53044c5 100644
--- a/plugins/discourse-ai/lib/agents/tools/summarize.rb
+++ b/plugins/discourse-ai/lib/agents/tools/summarize.rb
@@ -132,7 +132,14 @@ module DiscourseAi
prompt = section_prompt(topic, section, guidance)
summary =
- llm.generate(prompt, max_tokens: 400, user: bot_user, feature_name: "summarize_tool")
+ DiscourseAi::Completions::Llm.text_from_response(
+ llm.generate(
+ prompt,
+ max_tokens: 400,
+ user: bot_user,
+ feature_name: "summarize_tool",
+ ),
+ )
summaries << summary
end
@@ -147,12 +154,14 @@ module DiscourseAi
"concatenated the disjoint summaries, creating a cohesive narrative:\n#{summaries.join("\n")}}",
}
- llm.generate(
- concatenation_prompt,
- max_tokens: 500,
- user: bot_user,
- feature_name: "summarize_tool",
- )
+ result =
+ llm.generate(
+ concatenation_prompt,
+ max_tokens: 500,
+ user: bot_user,
+ feature_name: "summarize_tool",
+ )
+ DiscourseAi::Completions::Llm.text_from_response(result)
else
summaries.first
end
diff --git a/plugins/discourse-ai/lib/ai_bot/playground.rb b/plugins/discourse-ai/lib/ai_bot/playground.rb
index dc9e93b50a6..1f598e6c4c3 100644
--- a/plugins/discourse-ai/lib/ai_bot/playground.rb
+++ b/plugins/discourse-ai/lib/ai_bot/playground.rb
@@ -292,12 +292,10 @@ module DiscourseAi
)
new_title =
- bot
- .llm
- .generate(title_prompt, user: user, feature_name: "bot_title")
- .strip
- .split("\n")
- .last
+ DiscourseAi::Completions::Llm.text_from_response(
+ bot.llm.generate(title_prompt, user: user, feature_name: "bot_title"),
+ )
+ new_title = new_title.strip.split("\n").last
PostRevisor.new(post.topic.first_post, post.topic).revise!(
bot.bot_user,
diff --git a/plugins/discourse-ai/lib/completions/llm.rb b/plugins/discourse-ai/lib/completions/llm.rb
index f551a0a80a3..10a25f26242 100644
--- a/plugins/discourse-ai/lib/completions/llm.rb
+++ b/plugins/discourse-ai/lib/completions/llm.rb
@@ -109,6 +109,14 @@ module DiscourseAi
@prompts
end
+ def text_from_response(response)
+ if response.is_a?(Array)
+ response.select { |content| content.is_a?(String) }.join
+ else
+ response
+ end
+ end
+
def proxy(model)
llm_model =
if model.is_a?(LlmModel)
diff --git a/plugins/discourse-ai/lib/utils/image_to_text.rb b/plugins/discourse-ai/lib/utils/image_to_text.rb
index 04e710a506d..7c08f8be0f6 100644
--- a/plugins/discourse-ai/lib/utils/image_to_text.rb
+++ b/plugins/discourse-ai/lib/utils/image_to_text.rb
@@ -147,7 +147,7 @@ class DiscourseAi::Utils::ImageToText
end
prompt = DiscourseAi::Completions::Prompt.new(system_message, messages: messages)
result = llm.generate(prompt, user: Discourse.system_user, execution_context:)
- extract_chunks(result)
+ extract_chunks(DiscourseAi::Completions::Llm.text_from_response(result))
end
def extract_text_with_tesseract(page)
diff --git a/plugins/discourse-ai/spec/lib/agents/tool_runner_spec.rb b/plugins/discourse-ai/spec/lib/agents/tool_runner_spec.rb
index cbd0fce74cd..0199ee4326a 100644
--- a/plugins/discourse-ai/spec/lib/agents/tool_runner_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tool_runner_spec.rb
@@ -28,6 +28,21 @@ RSpec.describe DiscourseAi::Agents::ToolRunner do
expect(result).to eq({ "result" => "ok" })
end
+ it "combines multiple LLM text blocks before returning them to scripts" do
+ tool.update!(script: <<~JS)
+ function invoke() {
+ return { result: llm.generate("Generate a greeting") };
+ }
+ JS
+ result = nil
+ DiscourseAi::Completions::Llm.with_prepared_responses([["hello ", "world"]]) do
+ runner = described_class.new(parameters: {}, llm: llm, bot_user: bot_user, tool: tool)
+ result = runner.invoke
+ end
+
+ expect(result["result"]).to eq("hello world")
+ end
+
it "exposes discourse.baseUrl" do
tool.update!(script: "function invoke() { return { baseUrl: discourse.baseUrl }; }")
runner = described_class.new(parameters: {}, llm: llm, bot_user: bot_user, tool: tool)
diff --git a/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb
index 37a575514b9..20eff796c9a 100644
--- a/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb
@@ -32,6 +32,22 @@ RSpec.describe DiscourseAi::Agents::Tools::Summarize do
end
end
+ it "combines multiple LLM text blocks into the custom response" do
+ post = Fabricate(:post)
+
+ DiscourseAi::Completions::Llm.with_prepared_responses([["summary ", "stuff"]]) do
+ summarization =
+ described_class.new(
+ { topic_id: post.topic_id, guidance: "why did it happen?" },
+ bot_user: bot_user,
+ llm: llm,
+ )
+ summarization.invoke(&progress_blk)
+
+ expect(summarization.custom_raw).to eq("summary stuff")
+ end
+ end
+
it "protects hidden data" do
category = Fabricate(:category)
category.set_permissions({})
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb
index b9ccddb0a5c..3927da01be0 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/playground_spec.rb
@@ -69,6 +69,16 @@ RSpec.describe DiscourseAi::AiBot::Playground do
AiAgent.agent_cache.flush!
end
+ describe "#title_playground" do
+ it "updates the topic title when the LLM returns multiple text blocks" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [["A concise ", "conversation title"]],
+ ) { playground.title_playground(second_post, user) }
+
+ expect(pm.reload.title).to eq("A concise conversation title")
+ end
+ end
+
describe "is_bot_user_id?" do
it "properly detects ALL bots as bot users" do
agent = Fabricate(:ai_agent, enabled: false)
diff --git a/plugins/discourse-ai/spec/lib/utils/image_to_text_spec.rb b/plugins/discourse-ai/spec/lib/utils/image_to_text_spec.rb
new file mode 100644
index 00000000000..008a77d4dba
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/utils/image_to_text_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Utils::ImageToText do
+ fab!(:llm_model)
+ fab!(:upload)
+
+ before { enable_current_plugin }
+
+ describe "#extract_text" do
+ it "combines multiple LLM text blocks before extracting chunks" do
+ extractor =
+ described_class.new(
+ upload: upload,
+ llm_model: llm_model,
+ user: Discourse.system_user,
+ guidance_text: "Extracted text",
+ )
+ chunks = []
+
+ DiscourseAi::Completions::Llm.with_prepared_responses([["<chunk>First ", "block</chunk>"]]) do
+ extractor.extract_text(retries: 1) { |chunk| chunks << chunk }
+ end
+
+ expect(chunks).to eq(["First block"])
+ end
+ end
+end