Skip to content

A platform for community discussion. Free, open, simple.

FIX: Preserve whitespace in streamed JSON (#42339)

Samc6d0e5409d20
diff --git a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
index ec97acffd83..0cf98387873 100644
--- a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
+++ b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
@@ -11,6 +11,7 @@ module DiscourseAi
@completer = JsonCompleter.new
@broken = false
@last_notified = {}
+ @control_character_escaper = DiscourseAi::Utils::JsonControlCharacterEscaper.new
end
def broken?
@@ -25,9 +26,7 @@ module DiscourseAi
return
end
- @escaped_buffer << DiscourseAi::Utils::BestEffortJsonParser.escape_control_characters(
- raw_json,
- )
+ @escaped_buffer << @control_character_escaper.escape(raw_json)
parsed =
begin
diff --git a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb
index d552b920063..cc6bff422c3 100644
--- a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb
+++ b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb
@@ -17,13 +17,6 @@ module DiscourseAi
cast_value(value, schema_type)
end
- # Some providers deliver JSON whose string values had their control
- # characters unescaped by an outer JSON parse (real newlines inside
- # string values). Re-escaping them restores a parseable document.
- def escape_control_characters(text)
- text.gsub(/[\x00-\x1F]/) { |char| format("\\u%04x", char.ord) }
- end
-
private
# Parse attempts, in order:
@@ -41,7 +34,7 @@ module DiscourseAi
def best_effort_parse(cleaned, key)
attempts = [
-> { JsonCompleter.parse(cleaned) },
- -> { JsonCompleter.parse(escape_control_characters(cleaned)) },
+ -> { JsonCompleter.parse(JsonControlCharacterEscaper.escape(cleaned)) },
-> { parse_from_key_object(cleaned, key) },
-> { SmarterJSON.process_one(cleaned) },
]
@@ -73,7 +66,7 @@ module DiscourseAi
start = cleaned.rindex("{", key_idx)
return if start.nil? || start.zero?
- JsonCompleter.parse(escape_control_characters(cleaned[start..]))
+ JsonCompleter.parse(JsonControlCharacterEscaper.escape(cleaned[start..]))
end
def remove_markdown_fences(text)
diff --git a/plugins/discourse-ai/lib/utils/json_control_character_escaper.rb b/plugins/discourse-ai/lib/utils/json_control_character_escaper.rb
new file mode 100644
index 00000000000..0c0786efcb9
--- /dev/null
+++ b/plugins/discourse-ai/lib/utils/json_control_character_escaper.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module DiscourseAi
+ module Utils
+ # Some providers return JSON with control characters unescaped inside string values.
+ # Formatting whitespace outside strings must remain unchanged when repairing those values.
+ # Instances retain lexical state and must be scoped to one JSON document.
+ class JsonControlCharacterEscaper
+ SPECIAL_CHARACTERS = /["\\\x00-\x1F]/
+ private_constant :SPECIAL_CHARACTERS
+
+ def self.escape(json)
+ new.escape(json)
+ end
+
+ def initialize
+ @inside_string = false
+ @escaping_character = false
+ end
+
+ def escape(json)
+ scanner = StringScanner.new(json)
+ escaped = +""
+
+ while (segment = scanner.scan_until(SPECIAL_CHARACTERS))
+ character = segment.slice!(-1)
+ if !segment.empty?
+ escaped << segment
+ @escaping_character = false
+ end
+ append_special_character(escaped, character)
+ end
+
+ if !scanner.rest.empty?
+ escaped << scanner.rest
+ @escaping_character = false
+ end
+
+ escaped
+ end
+
+ private
+
+ def append_special_character(escaped, character)
+ if !@inside_string
+ escaped << character
+ @inside_string = true if character == '"'
+ elsif character.ord < 0x20
+ escaped << format("\\u%04x", character.ord)
+ @escaping_character = false
+ else
+ escaped << character
+ if @escaping_character
+ @escaping_character = false
+ elsif character == "\\"
+ @escaping_character = true
+ elsif character == '"'
+ @inside_string = false
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
index 8ffc129de15..6684ae2aeea 100644
--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
@@ -73,6 +73,15 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
expect(structured_output.read_buffered_property(:status)).to eq("")
end
+ it "streams pretty-printed JSON" do
+ pretty_json = JSON.pretty_generate(message: 'Hello, "world"!')
+ pretty_json.each_char.each_slice(4) { |chunk| structured_output << chunk.join }
+ structured_output.finish
+
+ expect(structured_output.broken?).to eq(false)
+ expect(structured_output.read_buffered_property(:message)).to eq('Hello, "world"!')
+ end
+
it "supports array types" do
chunks = [
+"{ \"",
diff --git a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb
index 069b7b636a0..e2cc200af66 100644
--- a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb
+++ b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb
@@ -165,11 +165,13 @@ RSpec.describe DiscourseAi::Utils::BestEffortJsonParser do
# regression specs for https://meta.discourse.org/t/407251
it "extracts the complete value when string values contain real newlines" do
content = "First line.\n\nA \"quoted\" word 😊 and everything after it."
- input = { output: content }.to_json.gsub("\\n", "\n")
+ inputs = [{ output: content }.to_json, JSON.pretty_generate(output: content)]
- result = described_class.extract_key(input, "string", :output)
+ inputs.each do |input|
+ result = described_class.extract_key(input.gsub("\\n", "\n"), "string", :output)
- expect(result).to eq(content)
+ expect(result).to eq(content)
+ end
end
it "completes truncated JSON instead of scraping garbage" do
diff --git a/plugins/discourse-ai/spec/lib/utils/json_control_character_escaper_spec.rb b/plugins/discourse-ai/spec/lib/utils/json_control_character_escaper_spec.rb
new file mode 100644
index 00000000000..ff00208370f
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/utils/json_control_character_escaper_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Utils::JsonControlCharacterEscaper do
+ before { enable_current_plugin }
+
+ describe ".escape" do
+ it "escapes controls in strings while preserving formatting whitespace" do
+ json = "{\n \"message\": \"First 😊 line\nSecond line\"\n}"
+
+ escaped = described_class.escape(json)
+
+ expect(escaped).to eq("{\n \"message\": \"First 😊 line\\u000aSecond line\"\n}")
+ end
+ end
+
+ describe "#escape" do
+ it "clears pending escape state when the next chunk contains ordinary characters" do
+ chunks = ['{"a":"x' + "\\", "nyz 😊", '","b":"p', "\nq\"}"]
+ escaper = described_class.new
+
+ escaped = chunks.map { |chunk| escaper.escape(chunk) }.join
+
+ expect(JSON.parse(escaped)).to eq("a" => "x\nyz 😊", "b" => "p\nq")
+ end
+
+ it "retains string and escape state across chunks" do
+ json = "{\n \"message\": \"A \\\"quoted\\\" value\nnext\"\n}"
+ split_at = json.index('\\"')
+ chunks = [json[..split_at], json[(split_at + 1)..]]
+ escaper = described_class.new
+
+ escaped = chunks.map { |chunk| escaper.escape(chunk) }.join
+
+ expect(JSON.parse(escaped)).to eq("message" => "A \"quoted\" value\nnext")
+ end
+
+ it "handles split backslashes and a control character at the start of a chunk" do
+ chunks = ['{"message":"path' + "\\", "\\", "\tend\"\n}"]
+ escaper = described_class.new
+
+ escaped = chunks.map { |chunk| escaper.escape(chunk) }.join
+
+ expect(JSON.parse(escaped)).to eq("message" => "path\\\tend")
+ end
+ end
+end