Skip to content

A platform for community discussion. Free, open, simple.

FIX: Stream responses in reports and summaries (#44571)

Sam2e76b8f86645
diff --git a/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb b/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb
index f85c432ed0b..1392b5b6d93 100644
--- a/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb
+++ b/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb
@@ -38,21 +38,29 @@ module DiscourseAi
DiscourseAi::Completions::Llm.proxy(
LlmModel.find(agent.class.default_llm_id.presence || SiteSetting.ai_default_llm_model),
)
- if llm.tokenizer.size(prompt.messages.to_json) + 8000 > llm.max_prompt_tokens
+ input_size, input_capacity =
+ llm.prompt_capacity(prompt, thinking_effort: agent.thinking_effort)
+ if input_capacity <= 0 || input_size > input_capacity
raise InvalidAnalysis, "Snapshot exceeds model context"
end
prompt.skip_trim = true
- response =
- llm.generate(
- prompt,
- user: @report.requested_by,
- feature_name: "ask_ai_report",
- temperature: agent.temperature,
- top_p: agent.top_p,
- thinking_effort: agent.thinking_effort,
- response_format: response_format(agent),
- max_response_tokens: 8000,
- )
+ response = +""
+ llm.generate(
+ prompt,
+ user: @report.requested_by,
+ feature_name: "ask_ai_report",
+ temperature: agent.temperature,
+ top_p: agent.top_p,
+ thinking_effort: agent.thinking_effort,
+ response_format: response_format(agent),
+ ) do |partial|
+ if partial.is_a?(DiscourseAi::Completions::StructuredOutput)
+ # The structured callback retains prefill and excludes provider reasoning.
+ response = partial
+ elsif partial.is_a?(String) && response.is_a?(String)
+ response << partial
+ end
+ end
result = JSON.parse(DiscourseAi::Completions::Llm.text_from_response(response))
ids = asks.map(&:first)
validate!(result, ids)
diff --git a/plugins/discourse-ai/lib/agents/tools/summarize.rb b/plugins/discourse-ai/lib/agents/tools/summarize.rb
index a77db2e22e5..6f682b4be48 100644
--- a/plugins/discourse-ai/lib/agents/tools/summarize.rb
+++ b/plugins/discourse-ai/lib/agents/tools/summarize.rb
@@ -131,16 +131,14 @@ module DiscourseAi
prompt = section_prompt(topic, section, guidance)
- summary =
- DiscourseAi::Completions::Llm.text_from_response(
- llm.generate(
- prompt,
- max_response_tokens: 400,
- user: bot_user,
- feature_name: "summarize_tool",
- execution_context: context.execution_context,
- ),
- )
+ summary = +""
+ llm.generate(
+ prompt,
+ max_response_tokens: 400,
+ user: bot_user,
+ feature_name: "summarize_tool",
+ execution_context: context.execution_context,
+ ) { |partial| summary << partial if partial.is_a?(String) }
summaries << summary
end
@@ -149,21 +147,27 @@ module DiscourseAi
progress << "."
progress_blk.call(progress)
- concatenation_prompt = {
- insts: "You are a helpful bot",
- input:
- "concatenated the disjoint summaries, creating a cohesive narrative:\n#{summaries.join("\n")}}",
- }
-
- result =
- llm.generate(
- concatenation_prompt,
- max_response_tokens: 500,
- user: bot_user,
- feature_name: "summarize_tool",
- execution_context: context.execution_context,
+ concatenation_prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "You are a helpful bot",
+ messages: [
+ {
+ type: :user,
+ content:
+ "Concatenate the disjoint summaries, creating a cohesive narrative:\n#{summaries.join("\n")}",
+ },
+ ],
)
- DiscourseAi::Completions::Llm.text_from_response(result)
+
+ result = +""
+ llm.generate(
+ concatenation_prompt,
+ max_response_tokens: 500,
+ user: bot_user,
+ feature_name: "summarize_tool",
+ execution_context: context.execution_context,
+ ) { |partial| result << partial if partial.is_a?(String) }
+ result
else
summaries.first
end
diff --git a/plugins/discourse-ai/lib/completions/context_preparation.rb b/plugins/discourse-ai/lib/completions/context_preparation.rb
index 9cf8e90d4e3..c578378a98e 100644
--- a/plugins/discourse-ai/lib/completions/context_preparation.rb
+++ b/plugins/discourse-ai/lib/completions/context_preparation.rb
@@ -387,20 +387,19 @@ module DiscourseAi
raise Error.new("completion_limit")
end
# Uncapped: thinking tokens count against max_tokens and could leave no summary.
- response =
- @llm.generate(
- compression_prompt,
- user: options[:user],
- feature_name: "context_compression",
- feature_context: options[:feature_context],
- execution_context: options[:execution_context],
- cancel_manager: options[:cancel_manager],
- )
+ summary = +""
+ @llm.generate(
+ compression_prompt,
+ user: options[:user],
+ feature_name: "context_compression",
+ feature_context: options[:feature_context],
+ execution_context: options[:execution_context],
+ cancel_manager: options[:cancel_manager],
+ ) { |partial| summary << partial if partial.is_a?(String) }
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@budget.record_elapsed(now - started_at)
started_at = now
raise Error.new("preparation_timeout") if elapsed_seconds >= MAX_SECONDS
- summary = Llm.text_from_response(response)
raise Error.new("unusable_summary") if summary.blank?
end
summary
diff --git a/plugins/discourse-ai/spec/lib/admin_dashboard/ask_ai_report_generator_spec.rb b/plugins/discourse-ai/spec/lib/admin_dashboard/ask_ai_report_generator_spec.rb
new file mode 100644
index 00000000000..7afb1ae8cf5
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/admin_dashboard/ask_ai_report_generator_spec.rb
@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::AdminDashboard::AskAiReportGenerator do
+ fab!(:admin)
+ fab!(:llm_model)
+
+ let(:ask) do
+ AskAiLog.create!(user: admin, query: "How do I configure email?", asked_at: Time.current)
+ end
+ let(:other_ask) do
+ AskAiLog.create!(user: admin, query: "How do I restore a backup?", asked_at: Time.current)
+ end
+ let(:report) do
+ AskAiReport.new(
+ requested_by: admin,
+ start_date: Date.current,
+ end_date: Date.current,
+ selected_ask_ids: [ask.id, other_ask.id],
+ reported_ask_count: 2,
+ total_ask_count: 2,
+ )
+ end
+ let(:output) do
+ {
+ "summary" => "Members ask about email setup.",
+ "subjects" => [
+ { "name" => "Email", "description" => "Setting up email.", "ask_ids" => [ask.id] },
+ ],
+ "insights" => [
+ {
+ "title" => "Email setup guidance",
+ "observation" => "A question asks how to configure email.",
+ "suggested_action" => "Review the email setup guide.",
+ "ask_ids" => [ask.id],
+ },
+ ],
+ }
+ end
+
+ before do
+ enable_current_plugin
+ allow(SiteSetting).to receive(:ai_default_llm_model).and_return(llm_model.id)
+ end
+
+ def stub_open_ai_stream(*partials)
+ body =
+ partials
+ .map { |partial| "data: #{{ choices: [{ delta: { content: partial } }] }.to_json}\n\n" }
+ .join
+ body << "data: [DONE]\n\n"
+
+ stub_request(:post, llm_model.url)
+ .with { |request| JSON.parse(request.body)["stream"] == true }
+ .to_return(status: 200, body:, headers: { "Content-Type" => "text/event-stream" })
+ end
+
+ describe "#generate" do
+ it "streams structured JSON and validates the complete report before filling missing subjects" do
+ output["subjects"][0]["ask_ids"] << other_ask.id + 1
+ json = output.to_json
+ request = stub_open_ai_stream(json.first(35), json[35...-1], json.last)
+
+ result = described_class.new(report).generate
+
+ expect(request).to have_been_requested.once
+ expect(result).to eq(
+ output.merge(
+ "subjects" => [
+ { "name" => "Email", "description" => "Setting up email.", "ask_ids" => [ask.id] },
+ {
+ "name" => I18n.t("discourse_ai.ask_ai_reports.ungrouped_name"),
+ "description" => I18n.t("discourse_ai.ask_ai_reports.ungrouped_description"),
+ "ask_ids" => [other_ask.id],
+ },
+ ],
+ "examples" => {
+ ask.id => ask.query,
+ other_ask.id => other_ask.query,
+ },
+ ),
+ )
+ end
+
+ it "uses the model capacity instead of a fixed 8000-token reservation" do
+ llm_model.update!(max_prompt_tokens: 6000, max_output_tokens: 1000)
+ request = stub_open_ai_stream(output.to_json)
+
+ result = described_class.new(report).generate
+
+ expect(result).to include(output.slice("summary", "insights"))
+ expect(request).to have_been_requested.once
+ end
+
+ it "rejects malformed JSON instead of accepting a partial structured response" do
+ request = stub_open_ai_stream(output.to_json[0...-1])
+
+ expect { described_class.new(report).generate }.to raise_error(JSON::ParserError)
+ expect(request).to have_been_requested.once
+ end
+
+ it "rejects invalid report fields after streaming completes" do
+ output["summary"] = ""
+ request = stub_open_ai_stream(output.to_json)
+
+ expect { described_class.new(report).generate }.to raise_error(
+ described_class::InvalidAnalysis,
+ )
+ expect(request).to have_been_requested.once
+ end
+
+ it "rejects insight evidence outside the selected asks after streaming completes" do
+ output["insights"][0]["ask_ids"] = [other_ask.id + 1]
+ request = stub_open_ai_stream(output.to_json)
+
+ expect { described_class.new(report).generate }.to raise_error(
+ described_class::InvalidAnalysis,
+ "Insight evidence must reference 1 to 3 distinct supplied IDs",
+ )
+ expect(request).to have_been_requested.once
+ end
+
+ it "collects string deltas beyond 8000 tokens when the agent has no structured properties" do
+ llm_model.update!(max_prompt_tokens: 32_000, max_output_tokens: 16_000)
+ agent = Fabricate(:ai_agent, default_llm_id: llm_model.id, response_format: [])
+ SiteSetting.ai_ask_ai_report_agent = agent.id
+ json = output.to_json[0...-1] + " \n" * 18_000 + "}"
+ expect(llm_model.to_llm.tokenizer.size(json)).to be > 8000
+ request = stub_open_ai_stream(json.first(35), json[35..])
+
+ result = described_class.new(report).generate
+
+ expect(request).to have_been_requested.once
+ expect(result).to include(output.slice("summary", "insights"))
+ expect(result["subjects"].first).to eq(output["subjects"].first)
+ end
+
+ it "retains the opening brace supplied by structured output prefilling" do
+ model =
+ Fabricate(:anthropic_model, provider_params: { disable_native_structured_output: true })
+ allow(SiteSetting).to receive(:ai_default_llm_model).and_return(model.id)
+ json = output.to_json[1..]
+ events = [
+ {
+ type: "message_start",
+ message: {
+ id: "msg_report",
+ type: "message",
+ role: "assistant",
+ content: [],
+ model: model.name,
+ usage: {
+ input_tokens: 25,
+ output_tokens: 1,
+ },
+ },
+ },
+ { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
+ {
+ type: "content_block_delta",
+ index: 0,
+ delta: {
+ type: "text_delta",
+ text: json.first(35),
+ },
+ },
+ { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: json[35..] } },
+ { type: "content_block_stop", index: 0 },
+ {
+ type: "message_delta",
+ delta: {
+ stop_reason: "end_turn",
+ stop_sequence: nil,
+ },
+ usage: {
+ output_tokens: 100,
+ },
+ },
+ { type: "message_stop" },
+ ]
+ body = events.map { |event| "event: #{event[:type]}\ndata: #{event.to_json}\n\n" }.join
+ request =
+ stub_request(:post, model.url)
+ .with do |http_request|
+ payload = JSON.parse(http_request.body)
+ payload["stream"] == true &&
+ payload["messages"].last == { "role" => "assistant", "content" => "{" }
+ end
+ .to_return(status: 200, body:, headers: { "Content-Type" => "text/event-stream" })
+
+ result = described_class.new(report).generate
+
+ expect(request).to have_been_requested.once
+ expect(result).to include(output.slice("summary", "insights"))
+ expect(result["subjects"].first).to eq(output["subjects"].first)
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb
index c7644e330d9..07bdd3779d7 100644
--- a/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb
@@ -14,6 +14,39 @@ RSpec.describe DiscourseAi::Agents::Tools::Summarize do
let(:summary) { "summary stuff" }
describe "#process" do
+ it "streams section summaries and their combined summary" do
+ post = Fabricate(:post, raw: "evidence " * 3200)
+ responses =
+ [["First ", "section"], ["Second ", "section"], ["Combined ", "summary"]].map do |deltas|
+ body =
+ deltas
+ .map { |delta| "data: #{{ choices: [{ delta: { content: delta } }] }.to_json}\n\n" }
+ .join
+ { status: 200, body: body + "data: [DONE]\n\n" }
+ end
+ request =
+ stub_request(:post, llm_model.url)
+ .with { |req| JSON.parse(req.body)["stream"] == true }
+ .to_return(*responses)
+ summarization =
+ described_class.new(
+ { topic_id: post.topic_id, guidance: "What happened?" },
+ bot_user: bot_user,
+ llm: llm,
+ )
+
+ info = summarization.invoke(&progress_blk)
+
+ expect(info).to eq("Topic summarized")
+ expect(summarization.custom_raw).to eq("Combined summary")
+ expect(request).to have_been_requested.times(3)
+ expect(WebMock).to have_requested(:post, llm_model.url)
+ .with { |req|
+ JSON.parse(req.body)["messages"].last["content"].include?("First section\nSecond section")
+ }
+ .once
+ end
+
it "can generate correct info" do
post = Fabricate(:post)
@@ -32,10 +65,11 @@ RSpec.describe DiscourseAi::Agents::Tools::Summarize do
end
end
- it "combines multiple LLM text blocks into the custom response" do
+ it "combines streamed text blocks without including reasoning" do
post = Fabricate(:post)
- DiscourseAi::Completions::Llm.with_prepared_responses([["summary ", "stuff"]]) do
+ thinking = DiscourseAi::Completions::Thinking.new(message: "Private reasoning")
+ DiscourseAi::Completions::Llm.with_prepared_responses([[thinking, "summary ", "stuff"]]) do
summarization =
described_class.new(
{ topic_id: post.topic_id, guidance: "why did it happen?" },
diff --git a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
index 64ba2df982b..5211cc34d14 100644
--- a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb
@@ -36,6 +36,28 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
end
describe "#prepare!" do
+ it "streams and buffers the compression summary" do
+ summary = "The codeword is QUARTZ-OTTER-731."
+ deltas = ["The codeword is ", "QUARTZ-OTTER-731."]
+ body =
+ deltas
+ .map { |delta| "data: #{{ choices: [{ delta: { content: delta } }] }.to_json}\n\n" }
+ .join
+ request =
+ stub_request(:post, model.url)
+ .with { |req| JSON.parse(req.body)["stream"] == true }
+ .to_return(status: 200, body: body + "data: [DONE]\n\n")
+ prompt = history_prompt
+
+ result = described_class.new(model.to_llm, threshold: 50).prepare!(prompt, user: user)
+
+ expect(result).to eq(:compressed)
+ expect(prompt.messages[1][:content]).to eq(
+ "<compressed_context>#{summary}</compressed_context>",
+ )
+ expect(request).to have_been_requested.once
+ end
+
it "shares maintenance bounds across contexts forked before their first preparation" do
stub_const(described_class, :MAX_CALLS, 2) do
root =
@@ -79,8 +101,9 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
it "compacts a small message count without summarizing the latest request" do
prompt = history_prompt
original = prompt.messages.deep_dup
+ thinking = DiscourseAi::Completions::Thinking.new(message: "Private reasoning")
DiscourseAi::Completions::Llm.with_prepared_responses(
- ["The codeword is QUARTZ-OTTER-731."],
+ [[thinking, "The codeword is ", "QUARTZ-OTTER-731."]],
) do |_, _, prompts, options|
llm = DiscourseAi::Completions::Llm.proxy(model)
result = described_class.new(llm, threshold: 50).prepare!(prompt, user: user)
@@ -88,6 +111,9 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
expect(result).to eq(:compressed)
expect(prompt.messages.map { |message| message[:type] }).to eq(%i[system user model user])
expect(prompt.messages.last).to eq(original.last)
+ expect(prompt.messages[1][:content]).to eq(
+ "<compressed_context>The codeword is QUARTZ-OTTER-731.</compressed_context>",
+ )
expect(prompts.first.skip_trim).to eq(true)
expect(options.first).to include(user: user, feature_name: "context_compression")
expect(options.first[:max_tokens]).to be_nil
@@ -300,18 +326,21 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation do
prompt = history_prompt(repetitions: 5000)
prompt.topic_id = post.topic_id
prompt.post_id = post.id
- stub_request(:post, model.url).to_return(
- body: {
- choices: [{ message: { content: "Summary retains QUARTZ-OTTER-731." } }],
- usage: {
- prompt_tokens: 100,
- completion_tokens: 20,
- },
- }.to_json,
- headers: {
- "Content-Type" => "application/json",
+ response = {
+ choices: [{ delta: { content: "Summary retains QUARTZ-OTTER-731." } }],
+ usage: {
+ prompt_tokens: 100,
+ completion_tokens: 20,
},
- )
+ }
+ stub_request(:post, model.url)
+ .with { |request| JSON.parse(request.body)["stream"] == true }
+ .to_return(
+ body: "data: #{response.to_json}\n\ndata: [DONE]\n\n",
+ headers: {
+ "Content-Type" => "text/event-stream",
+ },
+ )
expect(described_class.new(model.to_llm).prepare!(prompt, user: user)).to eq(:compressed)