Skip to content

A platform for community discussion. Free, open, simple.

FIX: AI helper title suggestions erroring on partially streamed JSON arrays (#42442)

Rafael dos Santos Silva2f5c074d663b
diff --git a/plugins/discourse-ai/lib/agents/titles_generator.rb b/plugins/discourse-ai/lib/agents/titles_generator.rb
index a66d7526d16..e110835d582 100644
--- a/plugins/discourse-ai/lib/agents/titles_generator.rb
+++ b/plugins/discourse-ai/lib/agents/titles_generator.rb
@@ -21,15 +21,7 @@ module DiscourseAi
The title suggestions should be returned in a JSON array, under the `output` key, like this:
- {
- "output": [
- "suggeested title #1",
- "suggeested title #2",
- "suggeested title #3",
- "suggeested title #4",
- "suggeested title #5"
- ]
- }
+ {"output": ["suggested title #1", "suggested title #2", "suggested title #3", "suggested title #4", "suggested title #5"]}
Return only the JSON
PROMPT
@@ -43,17 +35,15 @@ module DiscourseAi
[
[
"<input>In the labyrinth of time, a solitary horse, etched in gold by the setting sun, embarked on an infinite journey.</input>",
- <<~OUTPUT,
{
- "output": [
+ output: [
"The solitary horse",
"The horse etched in gold",
"A horse's infinite journey",
"A horse lost in time",
- "A horse's last rid"
- ]
- }
- OUTPUT
+ "A horse's last ride",
+ ],
+ }.to_json,
],
]
end
diff --git a/plugins/discourse-ai/lib/ai_helper/assistant.rb b/plugins/discourse-ai/lib/ai_helper/assistant.rb
index 0fd3394a8ff..b86c3674cb7 100644
--- a/plugins/discourse-ai/lib/ai_helper/assistant.rb
+++ b/plugins/discourse-ai/lib/ai_helper/assistant.rb
@@ -134,37 +134,26 @@ module DiscourseAi
)
context = attach_user_context(context, user, force_default_locale: force_default_locale)
- bad_json = false
json_summary_schema_key = bot.agent.response_format&.first.to_h
schema_key = json_summary_schema_key["key"]&.to_sym
schema_type = json_summary_schema_key["type"]
- if schema_type == "array"
- helper_response = []
- else
- helper_response = +""
- end
+ structured_output = nil
+ helper_response = +""
buffer_blk =
Proc.new do |partial, _, type|
if type == :structured_output && schema_type
- helper_chunk = partial.read_buffered_property(schema_key)
- next if helper_chunk.nil? || helper_chunk.empty?
-
- if schema_type == "array"
- if helper_chunk.is_a?(Array)
- helper_chunk.each do |item|
- helper_response << item if helper_response.exclude?(item)
- end
+ structured_output = partial
+
+ if schema_type == "string"
+ helper_chunk = partial.read_buffered_property(schema_key)
+ if helper_chunk.present?
+ helper_response << helper_chunk
+ block.call(helper_chunk) if block
end
- elsif schema_type == "string"
- helper_response << helper_chunk
- else
- helper_response = helper_chunk
end
-
- block.call(helper_chunk) if block && !bad_json
elsif type.blank?
# Assume response is a regular completion.
helper_response << partial
@@ -174,6 +163,16 @@ module DiscourseAi
bot.reply(context, &buffer_blk)
+ if schema_type && schema_type != "string"
+ # Non-string properties aren't streamed to callers; a single read at
+ # the end avoids transient placeholders the partial-JSON parser emits
+ # for incomplete elements (e.g. a trailing nil while an array is
+ # mid-stream).
+ helper_response = structured_output&.read_buffered_property(schema_key)
+ helper_response = helper_response.compact if helper_response.is_a?(Array)
+ helper_response = [] if schema_type == "array" && helper_response.nil?
+ end
+
helper_response
end
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb
index 1ae8a60dfa5..2d109d5246b 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_helper/assistant_spec.rb
@@ -342,6 +342,61 @@ RSpec.describe DiscourseAi::AiHelper::Assistant do
expect(response[:suggestions]).to contain_exactly(*expected)
end
+
+ it "returns clean suggestions when the model streams the JSON across many partial chunks" do
+ streaming_assistant = described_class.new(helper_llm: Fabricate(:llm_model))
+
+ # Mirrors backends that emit a title's closing `",` and the following
+ # whitespace as separate deltas, leaving the buffered JSON with a
+ # dangling comma between chunks.
+ deltas = [
+ "{",
+ "\n",
+ " ",
+ " \"",
+ "output",
+ "\":",
+ " [",
+ "\n",
+ " ",
+ " \"",
+ "The",
+ " solitary",
+ " horse",
+ "\",",
+ "\n",
+ " ",
+ " \"",
+ "A",
+ " horse",
+ " lost",
+ " in",
+ " time",
+ "\"",
+ "\n",
+ " ",
+ " ]",
+ "\n",
+ "}",
+ ]
+
+ body =
+ deltas
+ .map { |delta| { choices: [{ index: 0, delta: { content: delta } }] } }
+ .push({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })
+ .map { |event| "data: #{event.to_json}\n\n" }
+ .join
+ body << "data: [DONE]\n\n"
+
+ stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
+ status: 200,
+ body: body,
+ )
+
+ response = streaming_assistant.generate_and_send_prompt(mode, english_text, user)
+
+ expect(response[:suggestions]).to eq(["The solitary horse", "A horse lost in time"])
+ end
end
end
end