A platform for community discussion. Free, open, simple.
FIX: Preserve whitespace in streamed JSON (#42339)
Samc6d0e5409d20
diff --git a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rbindex ec97acffd83..0cf98387873 100644--- a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb+++ b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb@@ -11,6 +11,7 @@ module DiscourseAi@completer = JsonCompleter.new@broken = false@last_notified = {}+ @control_character_escaper = DiscourseAi::Utils::JsonControlCharacterEscaper.newenddef broken?@@ -25,9 +26,7 @@ module DiscourseAireturnend- @escaped_buffer << DiscourseAi::Utils::BestEffortJsonParser.escape_control_characters(- raw_json,- )+ @escaped_buffer << @control_character_escaper.escape(raw_json)parsed =begindiff --git a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rbindex d552b920063..cc6bff422c3 100644--- a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb+++ b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb@@ -17,13 +17,6 @@ module DiscourseAicast_value(value, schema_type)end- # Some providers deliver JSON whose string values had their control- # characters unescaped by an outer JSON parse (real newlines inside- # string values). Re-escaping them restores a parseable document.- def escape_control_characters(text)- text.gsub(/[\x00-\x1F]/) { |char| format("\\u%04x", char.ord) }- end-private# Parse attempts, in order:@@ -41,7 +34,7 @@ module DiscourseAidef best_effort_parse(cleaned, key)attempts = [-> { JsonCompleter.parse(cleaned) },- -> { JsonCompleter.parse(escape_control_characters(cleaned)) },+ -> { JsonCompleter.parse(JsonControlCharacterEscaper.escape(cleaned)) },-> { parse_from_key_object(cleaned, key) },-> { SmarterJSON.process_one(cleaned) },]@@ -73,7 +66,7 @@ module DiscourseAistart = cleaned.rindex("{", key_idx)return if start.nil? || start.zero?- JsonCompleter.parse(escape_control_characters(cleaned[start..]))+ JsonCompleter.parse(JsonControlCharacterEscaper.escape(cleaned[start..]))enddef remove_markdown_fences(text)diff --git a/plugins/discourse-ai/lib/utils/json_control_character_escaper.rb b/plugins/discourse-ai/lib/utils/json_control_character_escaper.rbnew file mode 100644index 00000000000..0c0786efcb9--- /dev/null+++ b/plugins/discourse-ai/lib/utils/json_control_character_escaper.rb@@ -0,0 +1,66 @@+# frozen_string_literal: true++require "strscan"++module DiscourseAi+ module Utils+ # Some providers return JSON with control characters unescaped inside string values.+ # Formatting whitespace outside strings must remain unchanged when repairing those values.+ # Instances retain lexical state and must be scoped to one JSON document.+ class JsonControlCharacterEscaper+ SPECIAL_CHARACTERS = /["\\\x00-\x1F]/+ private_constant :SPECIAL_CHARACTERS++ def self.escape(json)+ new.escape(json)+ end++ def initialize+ @inside_string = false+ @escaping_character = false+ end++ def escape(json)+ scanner = StringScanner.new(json)+ escaped = +""++ while (segment = scanner.scan_until(SPECIAL_CHARACTERS))+ character = segment.slice!(-1)+ if !segment.empty?+ escaped << segment+ @escaping_character = false+ end+ append_special_character(escaped, character)+ end++ if !scanner.rest.empty?+ escaped << scanner.rest+ @escaping_character = false+ end++ escaped+ end++ private++ def append_special_character(escaped, character)+ if !@inside_string+ escaped << character+ @inside_string = true if character == '"'+ elsif character.ord < 0x20+ escaped << format("\\u%04x", character.ord)+ @escaping_character = false+ else+ escaped << character+ if @escaping_character+ @escaping_character = false+ elsif character == "\\"+ @escaping_character = true+ elsif character == '"'+ @inside_string = false+ end+ end+ end+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rbindex 8ffc129de15..6684ae2aeea 100644--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb@@ -73,6 +73,15 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput doexpect(structured_output.read_buffered_property(:status)).to eq("")end+ it "streams pretty-printed JSON" do+ pretty_json = JSON.pretty_generate(message: 'Hello, "world"!')+ pretty_json.each_char.each_slice(4) { |chunk| structured_output << chunk.join }+ structured_output.finish++ expect(structured_output.broken?).to eq(false)+ expect(structured_output.read_buffered_property(:message)).to eq('Hello, "world"!')+ end+it "supports array types" dochunks = [+"{ \"",diff --git a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rbindex 069b7b636a0..e2cc200af66 100644--- a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb+++ b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb@@ -165,11 +165,13 @@ RSpec.describe DiscourseAi::Utils::BestEffortJsonParser do# regression specs for https://meta.discourse.org/t/407251it "extracts the complete value when string values contain real newlines" docontent = "First line.\n\nA \"quoted\" word π and everything after it."- input = { output: content }.to_json.gsub("\\n", "\n")+ inputs = [{ output: content }.to_json, JSON.pretty_generate(output: content)]- result = described_class.extract_key(input, "string", :output)+ inputs.each do |input|+ result = described_class.extract_key(input.gsub("\\n", "\n"), "string", :output)- expect(result).to eq(content)+ expect(result).to eq(content)+ endendit "completes truncated JSON instead of scraping garbage" dodiff --git a/plugins/discourse-ai/spec/lib/utils/json_control_character_escaper_spec.rb b/plugins/discourse-ai/spec/lib/utils/json_control_character_escaper_spec.rbnew file mode 100644index 00000000000..ff00208370f--- /dev/null+++ b/plugins/discourse-ai/spec/lib/utils/json_control_character_escaper_spec.rb@@ -0,0 +1,46 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Utils::JsonControlCharacterEscaper do+ before { enable_current_plugin }++ describe ".escape" do+ it "escapes controls in strings while preserving formatting whitespace" do+ json = "{\n \"message\": \"First π line\nSecond line\"\n}"++ escaped = described_class.escape(json)++ expect(escaped).to eq("{\n \"message\": \"First π line\\u000aSecond line\"\n}")+ end+ end++ describe "#escape" do+ it "clears pending escape state when the next chunk contains ordinary characters" do+ chunks = ['{"a":"x' + "\\", "nyz π", '","b":"p', "\nq\"}"]+ escaper = described_class.new++ escaped = chunks.map { |chunk| escaper.escape(chunk) }.join++ expect(JSON.parse(escaped)).to eq("a" => "x\nyz π", "b" => "p\nq")+ end++ it "retains string and escape state across chunks" do+ json = "{\n \"message\": \"A \\\"quoted\\\" value\nnext\"\n}"+ split_at = json.index('\\"')+ chunks = [json[..split_at], json[(split_at + 1)..]]+ escaper = described_class.new++ escaped = chunks.map { |chunk| escaper.escape(chunk) }.join++ expect(JSON.parse(escaped)).to eq("message" => "A \"quoted\" value\nnext")+ end++ it "handles split backslashes and a control character at the start of a chunk" do+ chunks = ['{"message":"path' + "\\", "\\", "\tend\"\n}"]+ escaper = described_class.new++ escaped = chunks.map { |chunk| escaper.escape(chunk) }.join++ expect(JSON.parse(escaped)).to eq("message" => "path\\\tend")+ end+ end+end