A platform for community discussion. Free, open, simple.
FIX: Stream responses in reports and summaries (#44571)
Sam2e76b8f86645
diff --git a/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb b/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rbindex f85c432ed0b..1392b5b6d93 100644--- a/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb+++ b/plugins/discourse-ai/lib/admin_dashboard/ask_ai_report_generator.rb@@ -38,21 +38,29 @@ module DiscourseAiDiscourseAi::Completions::Llm.proxy(LlmModel.find(agent.class.default_llm_id.presence || SiteSetting.ai_default_llm_model),)- if llm.tokenizer.size(prompt.messages.to_json) + 8000 > llm.max_prompt_tokens+ input_size, input_capacity =+ llm.prompt_capacity(prompt, thinking_effort: agent.thinking_effort)+ if input_capacity <= 0 || input_size > input_capacityraise InvalidAnalysis, "Snapshot exceeds model context"endprompt.skip_trim = true- response =- llm.generate(- prompt,- user: @report.requested_by,- feature_name: "ask_ai_report",- temperature: agent.temperature,- top_p: agent.top_p,- thinking_effort: agent.thinking_effort,- response_format: response_format(agent),- max_response_tokens: 8000,- )+ response = +""+ llm.generate(+ prompt,+ user: @report.requested_by,+ feature_name: "ask_ai_report",+ temperature: agent.temperature,+ top_p: agent.top_p,+ thinking_effort: agent.thinking_effort,+ response_format: response_format(agent),+ ) do |partial|+ if partial.is_a?(DiscourseAi::Completions::StructuredOutput)+ # The structured callback retains prefill and excludes provider reasoning.+ response = partial+ elsif partial.is_a?(String) && response.is_a?(String)+ response << partial+ end+ endresult = JSON.parse(DiscourseAi::Completions::Llm.text_from_response(response))ids = asks.map(&:first)validate!(result, ids)diff --git a/plugins/discourse-ai/lib/agents/tools/summarize.rb b/plugins/discourse-ai/lib/agents/tools/summarize.rbindex a77db2e22e5..6f682b4be48 100644--- a/plugins/discourse-ai/lib/agents/tools/summarize.rb+++ b/plugins/discourse-ai/lib/agents/tools/summarize.rb@@ -131,16 +131,14 @@ module DiscourseAiprompt = section_prompt(topic, section, guidance)- summary =- DiscourseAi::Completions::Llm.text_from_response(- llm.generate(- prompt,- max_response_tokens: 400,- user: bot_user,- feature_name: "summarize_tool",- execution_context: context.execution_context,- ),- )+ summary = +""+ llm.generate(+ prompt,+ max_response_tokens: 400,+ user: bot_user,+ feature_name: "summarize_tool",+ execution_context: context.execution_context,+ ) { |partial| summary << partial if partial.is_a?(String) }summaries << summaryend@@ -149,21 +147,27 @@ module DiscourseAiprogress << "."progress_blk.call(progress)- concatenation_prompt = {- insts: "You are a helpful bot",- input:- "concatenated the disjoint summaries, creating a cohesive narrative:\n#{summaries.join("\n")}}",- }-- result =- llm.generate(- concatenation_prompt,- max_response_tokens: 500,- user: bot_user,- feature_name: "summarize_tool",- execution_context: context.execution_context,+ concatenation_prompt =+ DiscourseAi::Completions::Prompt.new(+ "You are a helpful bot",+ messages: [+ {+ type: :user,+ content:+ "Concatenate the disjoint summaries, creating a cohesive narrative:\n#{summaries.join("\n")}",+ },+ ],)- DiscourseAi::Completions::Llm.text_from_response(result)++ result = +""+ llm.generate(+ concatenation_prompt,+ max_response_tokens: 500,+ user: bot_user,+ feature_name: "summarize_tool",+ execution_context: context.execution_context,+ ) { |partial| result << partial if partial.is_a?(String) }+ resultelsesummaries.firstenddiff --git a/plugins/discourse-ai/lib/completions/context_preparation.rb b/plugins/discourse-ai/lib/completions/context_preparation.rbindex 9cf8e90d4e3..c578378a98e 100644--- a/plugins/discourse-ai/lib/completions/context_preparation.rb+++ b/plugins/discourse-ai/lib/completions/context_preparation.rb@@ -387,20 +387,19 @@ module DiscourseAiraise Error.new("completion_limit")end# Uncapped: thinking tokens count against max_tokens and could leave no summary.- response =- @llm.generate(- compression_prompt,- user: options[:user],- feature_name: "context_compression",- feature_context: options[:feature_context],- execution_context: options[:execution_context],- cancel_manager: options[:cancel_manager],- )+ summary = +""+ @llm.generate(+ compression_prompt,+ user: options[:user],+ feature_name: "context_compression",+ feature_context: options[:feature_context],+ execution_context: options[:execution_context],+ cancel_manager: options[:cancel_manager],+ ) { |partial| summary << partial if partial.is_a?(String) }now = Process.clock_gettime(Process::CLOCK_MONOTONIC)@budget.record_elapsed(now - started_at)started_at = nowraise Error.new("preparation_timeout") if elapsed_seconds >= MAX_SECONDS- summary = Llm.text_from_response(response)raise Error.new("unusable_summary") if summary.blank?endsummarydiff --git a/plugins/discourse-ai/spec/lib/admin_dashboard/ask_ai_report_generator_spec.rb b/plugins/discourse-ai/spec/lib/admin_dashboard/ask_ai_report_generator_spec.rbnew file mode 100644index 00000000000..7afb1ae8cf5--- /dev/null+++ b/plugins/discourse-ai/spec/lib/admin_dashboard/ask_ai_report_generator_spec.rb@@ -0,0 +1,197 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::AdminDashboard::AskAiReportGenerator do+ fab!(:admin)+ fab!(:llm_model)++ let(:ask) do+ AskAiLog.create!(user: admin, query: "How do I configure email?", asked_at: Time.current)+ end+ let(:other_ask) do+ AskAiLog.create!(user: admin, query: "How do I restore a backup?", asked_at: Time.current)+ end+ let(:report) do+ AskAiReport.new(+ requested_by: admin,+ start_date: Date.current,+ end_date: Date.current,+ selected_ask_ids: [ask.id, other_ask.id],+ reported_ask_count: 2,+ total_ask_count: 2,+ )+ end+ let(:output) do+ {+ "summary" => "Members ask about email setup.",+ "subjects" => [+ { "name" => "Email", "description" => "Setting up email.", "ask_ids" => [ask.id] },+ ],+ "insights" => [+ {+ "title" => "Email setup guidance",+ "observation" => "A question asks how to configure email.",+ "suggested_action" => "Review the email setup guide.",+ "ask_ids" => [ask.id],+ },+ ],+ }+ end++ before do+ enable_current_plugin+ allow(SiteSetting).to receive(:ai_default_llm_model).and_return(llm_model.id)+ end++ def stub_open_ai_stream(*partials)+ body =+ partials+ .map { |partial| "data: #{{ choices: [{ delta: { content: partial } }] }.to_json}\n\n" }+ .join+ body << "data: [DONE]\n\n"++ stub_request(:post, llm_model.url)+ .with { |request| JSON.parse(request.body)["stream"] == true }+ .to_return(status: 200, body:, headers: { "Content-Type" => "text/event-stream" })+ end++ describe "#generate" do+ it "streams structured JSON and validates the complete report before filling missing subjects" do+ output["subjects"][0]["ask_ids"] << other_ask.id + 1+ json = output.to_json+ request = stub_open_ai_stream(json.first(35), json[35...-1], json.last)++ result = described_class.new(report).generate++ expect(request).to have_been_requested.once+ expect(result).to eq(+ output.merge(+ "subjects" => [+ { "name" => "Email", "description" => "Setting up email.", "ask_ids" => [ask.id] },+ {+ "name" => I18n.t("discourse_ai.ask_ai_reports.ungrouped_name"),+ "description" => I18n.t("discourse_ai.ask_ai_reports.ungrouped_description"),+ "ask_ids" => [other_ask.id],+ },+ ],+ "examples" => {+ ask.id => ask.query,+ other_ask.id => other_ask.query,+ },+ ),+ )+ end++ it "uses the model capacity instead of a fixed 8000-token reservation" do+ llm_model.update!(max_prompt_tokens: 6000, max_output_tokens: 1000)+ request = stub_open_ai_stream(output.to_json)++ result = described_class.new(report).generate++ expect(result).to include(output.slice("summary", "insights"))+ expect(request).to have_been_requested.once+ end++ it "rejects malformed JSON instead of accepting a partial structured response" do+ request = stub_open_ai_stream(output.to_json[0...-1])++ expect { described_class.new(report).generate }.to raise_error(JSON::ParserError)+ expect(request).to have_been_requested.once+ end++ it "rejects invalid report fields after streaming completes" do+ output["summary"] = ""+ request = stub_open_ai_stream(output.to_json)++ expect { described_class.new(report).generate }.to raise_error(+ described_class::InvalidAnalysis,+ )+ expect(request).to have_been_requested.once+ end++ it "rejects insight evidence outside the selected asks after streaming completes" do+ output["insights"][0]["ask_ids"] = [other_ask.id + 1]+ request = stub_open_ai_stream(output.to_json)++ expect { described_class.new(report).generate }.to raise_error(+ described_class::InvalidAnalysis,+ "Insight evidence must reference 1 to 3 distinct supplied IDs",+ )+ expect(request).to have_been_requested.once+ end++ it "collects string deltas beyond 8000 tokens when the agent has no structured properties" do+ llm_model.update!(max_prompt_tokens: 32_000, max_output_tokens: 16_000)+ agent = Fabricate(:ai_agent, default_llm_id: llm_model.id, response_format: [])+ SiteSetting.ai_ask_ai_report_agent = agent.id+ json = output.to_json[0...-1] + " \n" * 18_000 + "}"+ expect(llm_model.to_llm.tokenizer.size(json)).to be > 8000+ request = stub_open_ai_stream(json.first(35), json[35..])++ result = described_class.new(report).generate++ expect(request).to have_been_requested.once+ expect(result).to include(output.slice("summary", "insights"))+ expect(result["subjects"].first).to eq(output["subjects"].first)+ end++ it "retains the opening brace supplied by structured output prefilling" do+ model =+ Fabricate(:anthropic_model, provider_params: { disable_native_structured_output: true })+ allow(SiteSetting).to receive(:ai_default_llm_model).and_return(model.id)+ json = output.to_json[1..]+ events = [+ {+ type: "message_start",+ message: {+ id: "msg_report",+ type: "message",+ role: "assistant",+ content: [],+ model: model.name,+ usage: {+ input_tokens: 25,+ output_tokens: 1,+ },+ },+ },+ { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },+ {+ type: "content_block_delta",+ index: 0,+ delta: {+ type: "text_delta",+ text: json.first(35),+ },+ },+ { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: json[35..] } },+ { type: "content_block_stop", index: 0 },+ {+ type: "message_delta",+ delta: {+ stop_reason: "end_turn",+ stop_sequence: nil,+ },+ usage: {+ output_tokens: 100,+ },+ },+ { type: "message_stop" },+ ]+ body = events.map { |event| "event: #{event[:type]}\ndata: #{event.to_json}\n\n" }.join+ request =+ stub_request(:post, model.url)+ .with do |http_request|+ payload = JSON.parse(http_request.body)+ payload["stream"] == true &&+ payload["messages"].last == { "role" => "assistant", "content" => "{" }+ end+ .to_return(status: 200, body:, headers: { "Content-Type" => "text/event-stream" })++ result = described_class.new(report).generate++ expect(request).to have_been_requested.once+ expect(result).to include(output.slice("summary", "insights"))+ expect(result["subjects"].first).to eq(output["subjects"].first)+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rbindex c7644e330d9..07bdd3779d7 100644--- a/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/tools/summarize_spec.rb@@ -14,6 +14,39 @@ RSpec.describe DiscourseAi::Agents::Tools::Summarize dolet(:summary) { "summary stuff" }describe "#process" do+ it "streams section summaries and their combined summary" do+ post = Fabricate(:post, raw: "evidence " * 3200)+ responses =+ [["First ", "section"], ["Second ", "section"], ["Combined ", "summary"]].map do |deltas|+ body =+ deltas+ .map { |delta| "data: #{{ choices: [{ delta: { content: delta } }] }.to_json}\n\n" }+ .join+ { status: 200, body: body + "data: [DONE]\n\n" }+ end+ request =+ stub_request(:post, llm_model.url)+ .with { |req| JSON.parse(req.body)["stream"] == true }+ .to_return(*responses)+ summarization =+ described_class.new(+ { topic_id: post.topic_id, guidance: "What happened?" },+ bot_user: bot_user,+ llm: llm,+ )++ info = summarization.invoke(&progress_blk)++ expect(info).to eq("Topic summarized")+ expect(summarization.custom_raw).to eq("Combined summary")+ expect(request).to have_been_requested.times(3)+ expect(WebMock).to have_requested(:post, llm_model.url)+ .with { |req|+ JSON.parse(req.body)["messages"].last["content"].include?("First section\nSecond section")+ }+ .once+ end+it "can generate correct info" dopost = Fabricate(:post)@@ -32,10 +65,11 @@ RSpec.describe DiscourseAi::Agents::Tools::Summarize doendend- it "combines multiple LLM text blocks into the custom response" do+ it "combines streamed text blocks without including reasoning" dopost = Fabricate(:post)- DiscourseAi::Completions::Llm.with_prepared_responses([["summary ", "stuff"]]) do+ thinking = DiscourseAi::Completions::Thinking.new(message: "Private reasoning")+ DiscourseAi::Completions::Llm.with_prepared_responses([[thinking, "summary ", "stuff"]]) dosummarization =described_class.new({ topic_id: post.topic_id, guidance: "why did it happen?" },diff --git a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rbindex 64ba2df982b..5211cc34d14 100644--- a/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/context_preparation_spec.rb@@ -36,6 +36,28 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation doenddescribe "#prepare!" do+ it "streams and buffers the compression summary" do+ summary = "The codeword is QUARTZ-OTTER-731."+ deltas = ["The codeword is ", "QUARTZ-OTTER-731."]+ body =+ deltas+ .map { |delta| "data: #{{ choices: [{ delta: { content: delta } }] }.to_json}\n\n" }+ .join+ request =+ stub_request(:post, model.url)+ .with { |req| JSON.parse(req.body)["stream"] == true }+ .to_return(status: 200, body: body + "data: [DONE]\n\n")+ prompt = history_prompt++ result = described_class.new(model.to_llm, threshold: 50).prepare!(prompt, user: user)++ expect(result).to eq(:compressed)+ expect(prompt.messages[1][:content]).to eq(+ "<compressed_context>#{summary}</compressed_context>",+ )+ expect(request).to have_been_requested.once+ end+it "shares maintenance bounds across contexts forked before their first preparation" dostub_const(described_class, :MAX_CALLS, 2) doroot =@@ -79,8 +101,9 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation doit "compacts a small message count without summarizing the latest request" doprompt = history_promptoriginal = prompt.messages.deep_dup+ thinking = DiscourseAi::Completions::Thinking.new(message: "Private reasoning")DiscourseAi::Completions::Llm.with_prepared_responses(- ["The codeword is QUARTZ-OTTER-731."],+ [[thinking, "The codeword is ", "QUARTZ-OTTER-731."]],) do |_, _, prompts, options|llm = DiscourseAi::Completions::Llm.proxy(model)result = described_class.new(llm, threshold: 50).prepare!(prompt, user: user)@@ -88,6 +111,9 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation doexpect(result).to eq(:compressed)expect(prompt.messages.map { |message| message[:type] }).to eq(%i[system user model user])expect(prompt.messages.last).to eq(original.last)+ expect(prompt.messages[1][:content]).to eq(+ "<compressed_context>The codeword is QUARTZ-OTTER-731.</compressed_context>",+ )expect(prompts.first.skip_trim).to eq(true)expect(options.first).to include(user: user, feature_name: "context_compression")expect(options.first[:max_tokens]).to be_nil@@ -300,18 +326,21 @@ RSpec.describe DiscourseAi::Completions::ContextPreparation doprompt = history_prompt(repetitions: 5000)prompt.topic_id = post.topic_idprompt.post_id = post.id- stub_request(:post, model.url).to_return(- body: {- choices: [{ message: { content: "Summary retains QUARTZ-OTTER-731." } }],- usage: {- prompt_tokens: 100,- completion_tokens: 20,- },- }.to_json,- headers: {- "Content-Type" => "application/json",+ response = {+ choices: [{ delta: { content: "Summary retains QUARTZ-OTTER-731." } }],+ usage: {+ prompt_tokens: 100,+ completion_tokens: 20,},- )+ }+ stub_request(:post, model.url)+ .with { |request| JSON.parse(request.body)["stream"] == true }+ .to_return(+ body: "data: #{response.to_json}\n\ndata: [DONE]\n\n",+ headers: {+ "Content-Type" => "text/event-stream",+ },+ )expect(described_class.new(model.to_llm).prepare!(prompt, user: user)).to eq(:compressed)