Skip to content

A platform for community discussion. Free, open, simple.

FIX: Preserve whitespace-only chunks when streaming structured output (#42524)

Rafael dos Santos Silva67f8c017e03b
diff --git a/plugins/discourse-ai/lib/ai_helper/assistant.rb b/plugins/discourse-ai/lib/ai_helper/assistant.rb
index b86c3674cb7..d4c5a7a1e61 100644
--- a/plugins/discourse-ai/lib/ai_helper/assistant.rb
+++ b/plugins/discourse-ai/lib/ai_helper/assistant.rb
@@ -148,8 +148,7 @@ module DiscourseAi
structured_output = partial
if schema_type == "string"
- helper_chunk = partial.read_buffered_property(schema_key)
- if helper_chunk.present?
+ partial.read_buffered_property_chunk(schema_key) do |helper_chunk|
helper_response << helper_chunk
block.call(helper_chunk) if block
end
diff --git a/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rb b/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rb
index 7c1f46c227c..2c880f4186b 100644
--- a/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rb
+++ b/plugins/discourse-ai/lib/ai_helper/chat_thread_titler.rb
@@ -45,8 +45,7 @@ module DiscourseAi
result = +""
bot.reply(context) do |partial, _, type|
if type == :structured_output
- title = partial.read_buffered_property(:title)
- result << title if title.present?
+ partial.read_buffered_property_chunk(:title) { |title| result << title }
elsif type.blank?
result << partial
end
diff --git a/plugins/discourse-ai/lib/automation/report_runner.rb b/plugins/discourse-ai/lib/automation/report_runner.rb
index c5c621fa40a..10611a011b7 100644
--- a/plugins/discourse-ai/lib/automation/report_runner.rb
+++ b/plugins/discourse-ai/lib/automation/report_runner.rb
@@ -153,10 +153,12 @@ module DiscourseAi
buffer_blk =
Proc.new do |partial, _, type|
if type == :structured_output
- read_chunk = partial.read_buffered_property(json_summary_schema_key["key"]&.to_sym)
-
- print read_chunk if Rails.env.development? && @debug_mode
- result << read_chunk if read_chunk.present?
+ partial.read_buffered_property_chunk(
+ json_summary_schema_key["key"]&.to_sym,
+ ) do |read_chunk|
+ print read_chunk if Rails.env.development? && @debug_mode
+ result << read_chunk
+ end
elsif type.blank?
# Assume response is a regular completion.
print partial if Rails.env.development? && @debug_mode
diff --git a/plugins/discourse-ai/lib/completions/structured_output.rb b/plugins/discourse-ai/lib/completions/structured_output.rb
index 8405663f382..1f98ca16f5f 100644
--- a/plugins/discourse-ai/lib/completions/structured_output.rb
+++ b/plugins/discourse-ai/lib/completions/structured_output.rb
@@ -85,6 +85,15 @@ module DiscourseAi
end
end
+ # Yields the unread chunk of a string property, if any. Reading consumes
+ # the buffer, so unlike a presence check at the call site this never
+ # drops whitespace-only chunks (e.g. a "\n\n" delta between paragraphs).
+ def read_buffered_property_chunk(prop_name)
+ chunk = read_buffered_property(prop_name)
+ yield chunk if !chunk.nil? && !chunk.empty?
+ chunk
+ end
+
def notify_progress(key, value)
key_sym = key.to_sym
return if !@property_names.include?(key_sym)
diff --git a/plugins/discourse-ai/lib/summarization/fold_content.rb b/plugins/discourse-ai/lib/summarization/fold_content.rb
index e5e2a4c794e..f96a913a547 100644
--- a/plugins/discourse-ai/lib/summarization/fold_content.rb
+++ b/plugins/discourse-ai/lib/summarization/fold_content.rb
@@ -195,10 +195,9 @@ module DiscourseAi
end
elsif type == :structured_output
json_summary_schema_key = bot.agent.response_format&.first.to_h
- partial_summary =
- partial.read_buffered_property(json_summary_schema_key["key"]&.to_sym)
-
- if !partial_summary.nil? && !partial_summary.empty?
+ partial.read_buffered_property_chunk(
+ json_summary_schema_key["key"]&.to_sym,
+ ) do |partial_summary|
summary << partial_summary
on_partial_blk.call(partial_summary) if on_partial_blk
end
diff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
index 6684ae2aeea..9b1b48eb06e 100644
--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
@@ -28,6 +28,25 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
before { enable_current_plugin }
+ describe "#read_buffered_property_chunk" do
+ it "yields non-empty chunks, including whitespace-only ones" do
+ yielded = []
+
+ structured_output << "{\"message\": \"First."
+ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }
+
+ structured_output << "\\n\\n"
+ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }
+
+ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }
+
+ structured_output << "Second.\"}"
+ structured_output.read_buffered_property_chunk(:message) { |chunk| yielded << chunk }
+
+ expect(yielded).to eq(["First.", "\n\n", "Second."])
+ end
+ end
+
describe "Parsing structured output on the fly" do
it "acts as a buffer for an streamed JSON" do
chunks = [
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb
index 2d109d5246b..a9dd20f98c7 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb
@@ -398,5 +398,43 @@ RSpec.describe DiscourseAi::AiHelper::Assistant do
expect(response[:suggestions]).to eq(["The solitary horse", "A horse lost in time"])
end
end
+
+ context "when using a prompt that returns a string" do
+ let(:mode) { described_class::PROOFREAD }
+
+ it "preserves whitespace-only deltas streamed between paragraphs" do
+ streaming_assistant = described_class.new(helper_llm: Fabricate(:llm_model))
+
+ # Mirrors backends that emit paragraph breaks (an escaped "\n\n") or a
+ # bare space as their own delta; reading the buffered property consumes
+ # it, so dropping these chunks loses the whitespace permanently.
+ deltas = [
+ "{\"output\":\"",
+ "First paragraph.",
+ "\\n\\n",
+ "Do a",
+ " ",
+ "2-pass on every message.",
+ "\"}",
+ ]
+
+ body =
+ deltas
+ .map { |delta| { choices: [{ index: 0, delta: { content: delta } }] } }
+ .push({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })
+ .map { |event| "data: #{event.to_json}\n\n" }
+ .join
+ body << "data: [DONE]\n\n"
+
+ stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
+ status: 200,
+ body: body,
+ )
+
+ response = streaming_assistant.generate_and_send_prompt(mode, english_text, user)
+
+ expect(response[:suggestions]).to eq(["First paragraph.\n\nDo a 2-pass on every message."])
+ end
+ end
end
end