A platform for community discussion. Free, open, simple.
FIX: Preserve whitespace-only chunks when streaming structured output (#42524)
Rafael dos Santos Silva67f8c017e03b
diff --git a/plugins/discourse-ai/lib/ai_helper/assistant.rb b/plugins/discourse-ai/lib/ai_helper/assistant.rbindex b86c3674cb7..d4c5a7a1e61 100644--- a/plugins/discourse-ai/lib/ai_helper/assistant.rb+++ b/plugins/discourse-ai/lib/ai_helper/assistant.rb@@ -148,8 +148,7 @@ module DiscourseAistructured_output = partialif schema_type == "string"- helper_chunk = partial.read_buffered_property(schema_key)- if helper_chunk.present?+ partial.read_buffered_property_chunk(schema_key) do |helper_chunk|helper_response << helper_chunkblock.call(helper_chunk) if blockenddiff --git a/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rb b/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rbindex 7c1f46c227c..2c880f4186b 100644--- a/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rb+++ b/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rb@@ -45,8 +45,7 @@ module DiscourseAiresult = +""bot.reply(context) do |partial, _, type|if type == :structured_output- title = partial.read_buffered_property(:title)- result << title if title.present?+ partial.read_buffered_property_chunk(:title) { |title| result << title }elsif type.blank?result << partialenddiff --git a/plugins/discourse-ai/lib/automation/report_runner.rb b/plugins/discourse-ai/lib/automation/report_runner.rbindex c5c621fa40a..10611a011b7 100644--- a/plugins/discourse-ai/lib/automation/report_runner.rb+++ b/plugins/discourse-ai/lib/automation/report_runner.rb@@ -153,10 +153,12 @@ module DiscourseAibuffer_blk =Proc.new do |partial, _, type|if type == :structured_output- read_chunk = partial.read_buffered_property(json_summary_schema_key["key"]&.to_sym)-- print read_chunk if Rails.env.development? && @debug_mode- result << read_chunk if read_chunk.present?+ partial.read_buffered_property_chunk(+ json_summary_schema_key["key"]&.to_sym,+ ) do |read_chunk|+ print read_chunk if Rails.env.development? && @debug_mode+ result << read_chunk+ endelsif type.blank?# Assume response is a regular completion.print partial if Rails.env.development? && @debug_modediff --git a/plugins/discourse-ai/lib/completions/structured_output.rb b/plugins/discourse-ai/lib/completions/structured_output.rbindex 8405663f382..1f98ca16f5f 100644--- a/plugins/discourse-ai/lib/completions/structured_output.rb+++ b/plugins/discourse-ai/lib/completions/structured_output.rb@@ -85,6 +85,15 @@ module DiscourseAiendend+ # Yields the unread chunk of a string property, if any. Reading consumes+ # the buffer, so unlike a presence check at the call site this never+ # drops whitespace-only chunks (e.g. a "\n\n" delta between paragraphs).+ def read_buffered_property_chunk(prop_name)+ chunk = read_buffered_property(prop_name)+ yield chunk if !chunk.nil? && !chunk.empty?+ chunk+ end+def notify_progress(key, value)key_sym = key.to_symreturn if !@property_names.include?(key_sym)diff --git a/plugins/discourse-ai/lib/summarization/fold_content.rb b/plugins/discourse-ai/lib/summarization/fold_content.rbindex e5e2a4c794e..f96a913a547 100644--- a/plugins/discourse-ai/lib/summarization/fold_content.rb+++ b/plugins/discourse-ai/lib/summarization/fold_content.rb@@ -195,10 +195,9 @@ module DiscourseAiendelsif type == :structured_outputjson_summary_schema_key = bot.agent.response_format&.first.to_h- partial_summary =- partial.read_buffered_property(json_summary_schema_key["key"]&.to_sym)-- if !partial_summary.nil? && !partial_summary.empty?+ partial.read_buffered_property_chunk(+ json_summary_schema_key["key"]&.to_sym,+ ) do |partial_summary|summary << partial_summaryon_partial_blk.call(partial_summary) if on_partial_blkenddiff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rbindex 6684ae2aeea..9b1b48eb06e 100644--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb@@ -28,6 +28,25 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput dobefore { enable_current_plugin }+ describe "#read_buffered_property_chunk" do+ it "yields non-empty chunks, including whitespace-only ones" do+ yielded = []++ structured_output << "{\"message\": \"First."+ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }++ structured_output << "\\n\\n"+ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }++ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }++ structured_output << "Second.\"}"+ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }++ expect(yielded).to eq(["First.", "\n\n", "Second."])+ end+ end+describe "Parsing structured output on the fly" doit "acts as a buffer for an streamed JSON" dochunks = [diff --git a/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rbindex 2d109d5246b..a9dd20f98c7 100644--- a/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb@@ -398,5 +398,43 @@ RSpec.describe DiscourseAi::AiHelper::Assistant doexpect(response[:suggestions]).to eq(["The solitary horse", "A horse lost in time"])endend++ context "when using a prompt that returns a string" do+ let(:mode) { described_class::PROOFREAD }++ it "preserves whitespace-only deltas streamed between paragraphs" do+ streaming_assistant = described_class.new(helper_llm: Fabricate(:llm_model))++ # Mirrors backends that emit paragraph breaks (an escaped "\n\n") or a+ # bare space as their own delta; reading the buffered property consumes+ # it, so dropping these chunks loses the whitespace permanently.+ deltas = [+ "{\"output\":\"",+ "First paragraph.",+ "\\n\\n",+ "Do a",+ " ",+ "2-pass on every message.",+ "\"}",+ ]++ body =+ deltas+ .map { |delta| { choices: [{ index: 0, delta: { content: delta } }] } }+ .push({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })+ .map { |event| "data: #{event.to_json}\n\n" }+ .join+ body << "data: [DONE]\n\n"++ stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(+ status: 200,+ body: body,+ )++ response = streaming_assistant.generate_and_send_prompt(mode, english_text, user)++ expect(response[:suggestions]).to eq(["First paragraph.\n\nDo a 2-pass on every message."])+ end+ endendend