A platform for community discussion. Free, open, simple.
FIX: Silent truncation and corruption of AI structured output streams (#41716)
Rafael dos Santos Silva87775447c04a
diff --git a/Gemfile b/Gemfileindex 2df98e77f04..0cb16063f2c 100644--- a/Gemfile+++ b/Gemfile@@ -305,6 +305,8 @@ gem "octokit", require: false# for discourse-aigem "tokenizers", require: falsegem "tiktoken_ruby", require: false+gem "smarter_json", require: false+gem "json_completer", require: falsegem "discourse_ai-tokenizers", require: falsegem "ed25519" # TODO: remove this as existing ssl gem should handle thisgem "Ascii85", require: falsediff --git a/Gemfile.lock b/Gemfile.lockindex d46cace862e..f29f9722096 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -305,6 +305,7 @@ GEMjson-schema (6.2.0)addressable (~> 2.8)bigdecimal (>= 3.1, < 5)+ json_completer (1.2.0)json_schemer (2.5.0)bigdecimalhana (~> 1.3)@@ -720,6 +721,8 @@ GEMsimplecov-html (0.13.2)simplecov_json_formatter (0.1.4)simpleidn (0.2.3)+ smarter_json (1.2.6)+ bigdecimalsnaky_hash (2.0.4)hashie (>= 0.1.0, < 6)version_gem (>= 1.1.8, < 3)@@ -869,6 +872,7 @@ DEPENDENCIESinflectioniso8601json+ json_completerjson_schemerlandlockliquid (= 5.12.0)@@ -958,6 +962,7 @@ DEPENDENCIESshoulda-matcherssidekiq (>= 7.3.10)simplecov+ smarter_jsonsqlite3sshkeystackprof@@ -1095,6 +1100,7 @@ CHECKSUMSjmespath (1.6.2) sha256=238d774a58723d6c090494c8879b5e9918c19485f7e840f2c1c7532cf84ebcb1json (2.19.9) sha256=9b9025b7cdddafa38d316eca0b2358488e42d417045c1b90d216a9fefe46b79ajson-schema (6.2.0) sha256=e8bff46ed845a22c1ab2bd0d7eccf831c01fe23bb3920caa4c74db4306813666+ json_completer (1.2.0) sha256=4665749172634eccb208c562eb59ea81dd7a612a1fa9a36df441ac473ff177b1json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396jwt (2.10.1) sha256=e6424ae1d813f63e761a04d6284e10e7ec531d6f701917fadcd0d9b2deaf1cc5landlock (0.2) sha256=9bb86d7d12b16485b278ba575d49f55b730089c7a3d0f45a769fe4d7384d5e55@@ -1275,6 +1281,7 @@ CHECKSUMSsimplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428simpleidn (0.2.3) sha256=08ce96f03fa1605286be22651ba0fc9c0b2d6272c9b27a260bc88be05b0d2c29+ smarter_json (1.2.6) sha256=6638a2d23954e26d268212b4c8f5b9871d31b72f3d169371cca1082c1a722b45snaky_hash (2.0.4) sha256=2b12758c57defa6796341a1620f84b1a23737421d8d7e2575d0550b53cc4fecesqlite3 (2.9.5-aarch64-linux-gnu) sha256=78075b6337d3d182c6d2b4691049ed45cd220826160c9ea18946bf6a1de200dcsqlite3 (2.9.5-aarch64-linux-musl) sha256=18c801185deb4adc01ddb281e8f672a39e3d1729979ca91e39439cd3eac0402ddiff --git a/plugins/discourse-ai/lib/completions/json_streaming_parser.rb b/plugins/discourse-ai/lib/completions/json_streaming_parser.rbdeleted file mode 100644index f3288ab6bd8..00000000000--- a/plugins/discourse-ai/lib/completions/json_streaming_parser.rb+++ /dev/null@@ -1,668 +0,0 @@-# frozen_string_literal: true--# This code is copied from the MIT licensed json-stream-# see: https://github.com/dgraham/json-stream-#-# It was copied to avoid the dependency and allow us to make some small changes-# particularly we need better access to internal state when parsing--module DiscourseAi- module Completions- # Raised on any invalid JSON text.- ParserError = Class.new(RuntimeError)-- # A streaming JSON parser that generates SAX-like events for state changes.- # Use the json gem for small documents. Use this for huge documents that- # won't fit in memory.- #- # Examples- #- # parser = JSON::Stream::Parser.new- # parser.key { |key| puts key }- # parser.value { |value| puts value }- # parser << '{"answer":'- # parser << ' 42}'- class JsonStreamingParser- # our changes:- attr_reader :state, :buf, :pos-- # A character buffer that expects a UTF-8 encoded stream of bytes.- # This handles truncated multi-byte characters properly so we can just- # feed it binary data and receive a properly formatted UTF-8 String as- # output.- #- # More UTF-8 parsing details are available at:- #- # http://en.wikipedia.org/wiki/UTF-8- # http://tools.ietf.org/html/rfc3629#section-3- class Buffer- def initialize- @state = :start- @buffer = []- @need = 0- end-- # Fill the buffer with a String of binary UTF-8 encoded bytes. Returns- # as much of the data in a UTF-8 String as we have. Truncated multi-byte- # characters are saved in the buffer until the next call to this method- # where we expect to receive the rest of the multi-byte character.- #- # data - The partial binary encoded String data.- #- # Raises JSON::Stream::ParserError if the UTF-8 byte sequence is malformed.- #- # Returns a UTF-8 encoded String.- def <<(data)- data = data.dup if data.frozen?- # Avoid state machine for complete UTF-8.- if @buffer.empty?- data.force_encoding(Encoding::UTF_8)- return data if data.valid_encoding?- end-- bytes = []- data.each_byte do |byte|- case @state- when :start- if byte < 128- bytes << byte- elsif byte >= 192- @state = :multi_byte- @buffer << byte- @need =- case- when byte >= 240- 4- when byte >= 224- 3- when byte >= 192- 2- end- else- error("Expected start of multi-byte or single byte char")- end- when :multi_byte- if byte > 127 && byte < 192- @buffer << byte- if @buffer.size == @need- bytes += @buffer.slice!(0, @buffer.size)- @state = :start- end- else- error("Expected continuation byte")- end- end- end-- # Build UTF-8 encoded string from completed codepoints.- bytes- .pack("C*")- .force_encoding(Encoding::UTF_8)- .tap { |text| error("Invalid UTF-8 byte sequence") unless text.valid_encoding? }- end-- # Determine if the buffer contains partial UTF-8 continuation bytes that- # are waiting on subsequent completion bytes before a full codepoint is- # formed.- #- # Examples- #- # bytes = "é".bytes- #- # buffer << bytes[0]- # buffer.empty?- # # => false- #- # buffer << bytes[1]- # buffer.empty?- # # => true- #- # Returns true if the buffer is empty.- def empty?- @buffer.empty?- end-- private-- def error(message)- raise ParserError, message- end- end-- BUF_SIZE = 4096- CONTROL = /[\x00-\x1F]/- WS = /[ \n\t\r]/- HEX = /[0-9a-fA-F]/- DIGIT = /[0-9]/- DIGIT_1_9 = /[1-9]/- DIGIT_END = /\d$/- TRUE_RE = /[rue]/- FALSE_RE = /[alse]/- NULL_RE = /[ul]/- TRUE_KEYWORD = "true"- FALSE_KEYWORD = "false"- NULL_KEYWORD = "null"- LEFT_BRACE = "{"- RIGHT_BRACE = "}"- LEFT_BRACKET = "["- RIGHT_BRACKET = "]"- BACKSLASH = '\\'- SLASH = "/"- QUOTE = '"'- COMMA = ","- COLON = ":"- ZERO = "0"- MINUS = "-"- PLUS = "+"- POINT = "."- EXPONENT = /[eE]/- B, F, N, R, T, U = %w[b f n r t u]-- # Create a new parser with an optional initialization block where- # we can register event callbacks.- #- # Examples- #- # parser = JSON::Stream::Parser.new do- # start_document { puts "start document" }- # end_document { puts "end document" }- # start_object { puts "start object" }- # end_object { puts "end object" }- # start_array { puts "start array" }- # end_array { puts "end array" }- # key { |k| puts "key: #{k}" }- # value { |v| puts "value: #{v}" }- # end- def initialize(&block)- @state = :start_document- @utf8 = Buffer.new- @listeners = {- start_document: [],- end_document: [],- start_object: [],- end_object: [],- start_array: [],- end_array: [],- key: [],- value: [],- }-- # Track parse stack.- @stack = []- @unicode = +""- @buf = +""- @pos = -1-- # Register any observers in the block.- instance_eval(&block) if block_given?- end-- def start_document(&block)- @listeners[:start_document] << block- end-- def end_document(&block)- @listeners[:end_document] << block- end-- def start_object(&block)- @listeners[:start_object] << block- end-- def end_object(&block)- @listeners[:end_object] << block- end-- def start_array(&block)- @listeners[:start_array] << block- end-- def end_array(&block)- @listeners[:end_array] << block- end-- def key(&block)- @listeners[:key] << block- end-- def value(&block)- @listeners[:value] << block- end-- # Pass data into the parser to advance the state machine and- # generate callback events. This is well suited for an EventMachine- # receive_data loop.- #- # data - The String of partial JSON data to parse.- #- # Raises a JSON::Stream::ParserError if the JSON data is malformed.- #- # Returns nothing.- def <<(data)- (@utf8 << data).each_char do |ch|- @pos += 1- case @state- when :start_document- start_value(ch)- when :start_object- case ch- when QUOTE- @state = :start_string- @stack.push(:key)- when RIGHT_BRACE- end_container(:object)- when WS- # ignore- else- error("Expected object key start")- end- when :start_string- case ch- when QUOTE- if @stack.pop == :string- end_value(@buf)- else # :key- @state = :end_key- notify(:key, @buf)- end- @buf = +""- when BACKSLASH- @state = :start_escape- when CONTROL- error("Control characters must be escaped")- else- @buf << ch- end- when :start_escape- case ch- when QUOTE, BACKSLASH, SLASH- @buf << ch- @state = :start_string- when B- @buf << "\b"- @state = :start_string- when F- @buf << "\f"- @state = :start_string- when N- @buf << "\n"- @state = :start_string- when R- @buf << "\r"- @state = :start_string- when T- @buf << "\t"- @state = :start_string- when U- @state = :unicode_escape- else- error("Expected escaped character")- end- when :unicode_escape- case ch- when HEX- @unicode << ch- if @unicode.size == 4- codepoint = @unicode.slice!(0, 4).hex- if codepoint >= 0xD800 && codepoint <= 0xDBFF- error("Expected low surrogate pair half") if @stack[-1].is_a?(Integer)- @state = :start_surrogate_pair- @stack.push(codepoint)- elsif codepoint >= 0xDC00 && codepoint <= 0xDFFF- high = @stack.pop- error("Expected high surrogate pair half") unless high.is_a?(Integer)- pair = ((high - 0xD800) * 0x400) + (codepoint - 0xDC00) + 0x10000- @buf << pair- @state = :start_string- else- @buf << codepoint- @state = :start_string- end- end- else- error("Expected unicode escape hex digit")- end- when :start_surrogate_pair- case ch- when BACKSLASH- @state = :start_surrogate_pair_u- else- error("Expected low surrogate pair half")- end- when :start_surrogate_pair_u- case ch- when U- @state = :unicode_escape- else- error("Expected low surrogate pair half")- end- when :start_negative_number- case ch- when ZERO- @state = :start_zero- @buf << ch- when DIGIT_1_9- @state = :start_int- @buf << ch- else- error("Expected 0-9 digit")- end- when :start_zero- case ch- when POINT- @state = :start_float- @buf << ch- when EXPONENT- @state = :start_exponent- @buf << ch- else- end_value(@buf.to_i)- @buf = +""- @pos -= 1- redo- end- when :start_float- case ch- when DIGIT- @state = :in_float- @buf << ch- else- error("Expected 0-9 digit")- end- when :in_float- case ch- when DIGIT- @buf << ch- when EXPONENT- @state = :start_exponent- @buf << ch- else- end_value(@buf.to_f)- @buf = +""- @pos -= 1- redo- end- when :start_exponent- case ch- when MINUS, PLUS, DIGIT- @state = :in_exponent- @buf << ch- else- error("Expected +, -, or 0-9 digit")- end- when :in_exponent- case ch- when DIGIT- @buf << ch- else- error("Expected 0-9 digit") unless @buf =~ DIGIT_END- end_value(@buf.to_f)- @buf = +""- @pos -= 1- redo- end- when :start_int- case ch- when DIGIT- @buf << ch- when POINT- @state = :start_float- @buf << ch- when EXPONENT- @state = :start_exponent- @buf << ch- else- end_value(@buf.to_i)- @buf = +""- @pos -= 1- redo- end- when :start_true- keyword(TRUE_KEYWORD, true, TRUE_RE, ch)- when :start_false- keyword(FALSE_KEYWORD, false, FALSE_RE, ch)- when :start_null- keyword(NULL_KEYWORD, nil, NULL_RE, ch)- when :end_key- case ch- when COLON- @state = :key_sep- when WS- # ignore- else- error("Expected colon key separator")- end- when :key_sep- start_value(ch)- when :start_array- case ch- when RIGHT_BRACKET- end_container(:array)- when WS- # ignore- else- start_value(ch)- end- when :end_value- case ch- when COMMA- @state = :value_sep- when RIGHT_BRACE- end_container(:object)- when RIGHT_BRACKET- end_container(:array)- when WS- # ignore- else- error("Expected comma or object or array close")- end- when :value_sep- if @stack[-1] == :object- case ch- when QUOTE- @state = :start_string- @stack.push(:key)- when WS- # ignore- else- error("Expected object key start")- end- else- start_value(ch)- end- when :end_document- error("Unexpected data") unless ch =~ WS- end- end- end-- # Drain any remaining buffered characters into the parser to complete- # the parsing of the document.- #- # This is only required when parsing a document containing a single- # numeric value, integer or float. The parser has no other way to- # detect when it should no longer expect additional characters with- # which to complete the parse, so it must be signaled by a call to- # this method.- #- # If you're parsing more typical object or array documents, there's no- # need to call `finish` because the parse will complete when the final- # closing `]` or `}` character is scanned.- #- # Raises a JSON::Stream::ParserError if the JSON data is malformed.- #- # Returns nothing.- def finish- # Partial multi-byte character waiting for completion bytes.- error("Unexpected end-of-file") unless @utf8.empty?-- # Partial array, object, or string.- error("Unexpected end-of-file") unless @stack.empty?-- case @state- when :end_document- # done, do nothing- when :in_float- end_value(@buf.to_f)- when :in_exponent- error("Unexpected end-of-file") unless @buf =~ DIGIT_END- end_value(@buf.to_f)- when :start_zero- end_value(@buf.to_i)- when :start_int- end_value(@buf.to_i)- else- error("Unexpected end-of-file")- end- end-- private-- # Invoke all registered observer procs for the event type.- #- # type - The Symbol listener name.- # args - The argument list to pass into the observer procs.- #- # Examples- #- # # broadcast events for {"answer": 42}- # notify(:start_object)- # notify(:key, "answer")- # notify(:value, 42)- # notify(:end_object)- #- # Returns nothing.- def notify(type, *args)- @listeners[type].each { |block| block.call(*args) }- end-- # Complete an object or array container value type.- #- # type - The Symbol, :object or :array, of the expected type.- #- # Raises a JSON::Stream::ParserError if the expected container type- # was not completed.- #- # Returns nothing.- def end_container(type)- @state = :end_value- if @stack.pop == type- case type- when :object- notify(:end_object)- when :array- notify(:end_array)- end- else- error("Expected end of #{type}")- end- notify_end_document if @stack.empty?- end-- # Broadcast an `end_document` event to observers after a complete JSON- # value document (object, array, number, string, true, false, null) has- # been parsed from the text. This is the final event sent to observers- # and signals the parse has finished.- #- # Returns nothing.- def notify_end_document- @state = :end_document- notify(:end_document)- end-- # Parse one of the three allowed keywords: true, false, null.- #- # word - The String keyword ('true', 'false', 'null').- # value - The Ruby value (true, false, nil).- # re - The Regexp of allowed keyword characters.- # ch - The current String character being parsed.- #- # Raises a JSON::Stream::ParserError if the character does not belong- # in the expected keyword.- #- # Returns nothing.- def keyword(word, value, re, ch)- if ch =~ re- @buf << ch- else- error("Expected #{word} keyword")- end-- if @buf.size == word.size- if @buf == word- @buf = +""- end_value(value)- else- error("Expected #{word} keyword")- end- end- end-- # Process the first character of one of the seven possible JSON- # values: object, array, string, true, false, null, number.- #- # ch - The current character String.- #- # Raises a JSON::Stream::ParserError if the character does not signal- # the start of a value.- #- # Returns nothing.- def start_value(ch)- case ch- when LEFT_BRACE- notify(:start_document) if @stack.empty?- @state = :start_object- @stack.push(:object)- notify(:start_object)- when LEFT_BRACKET- notify(:start_document) if @stack.empty?- @state = :start_array- @stack.push(:array)- notify(:start_array)- when QUOTE- @state = :start_string- @stack.push(:string)- when T- @state = :start_true- @buf << ch- when F- @state = :start_false- @buf << ch- when N- @state = :start_null- @buf << ch- when MINUS- @state = :start_negative_number- @buf << ch- when ZERO- @state = :start_zero- @buf << ch- when DIGIT_1_9- @state = :start_int- @buf << ch- when WS- # ignore- else- error("Expected value")- end- end-- # Advance the state machine and notify `value` observers that a- # string, number or keyword (true, false, null) value was parsed.- #- # value - The object to broadcast to observers.- #- # Returns nothing.- def end_value(value)- @state = :end_value- notify(:start_document) if @stack.empty?- notify(:value, value)- notify_end_document if @stack.empty?- end-- def error(message)- raise ParserError, "#{message}: char #{@pos}"- end- end- end-enddiff --git a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rbindex b4de6424fa6..ec97acffd83 100644--- a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb+++ b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb@@ -3,55 +3,14 @@module DiscourseAimodule Completionsclass JsonStreamingTracker- attr_reader :current_key, :current_value, :stream_consumer+ attr_reader :stream_consumerdef initialize(stream_consumer)@stream_consumer = stream_consumer- @current_key = nil- @current_value = nil- @tracking_array = false+ @escaped_buffer = +""+ @completer = JsonCompleter.new@broken = false- @parser = DiscourseAi::Completions::JsonStreamingParser.new-- @parser.key do |k|- next if @broken- if @tracking_array && @current_key.present?- mark_broken!- next- end-- @current_key = k- @current_value = nil- end-- @parser.value do |value|- next if @broken- if @current_key- if @tracking_array- @current_value ||= []- @current_value << value- stream_consumer.notify_progress(@current_key, @current_value)- else- stream_consumer.notify_progress(@current_key, value)- @current_key = nil- end- end- end-- @parser.start_array do- next if @broken- next if !@current_key-- @tracking_array = true- @current_value = []- end-- @parser.end_array do- next if @broken- @tracking_array = false- @current_key = nil- @current_value = nil- end+ @last_notified = {}enddef broken?@@ -59,58 +18,40 @@ module DiscourseAienddef <<(raw_json)- # llm could send broken json- # in that case just deal with it later- # don't streamreturn if @broken- begin- pre_append_buffer = @parser.buf.dup- @parser << raw_json- rescue DiscourseAi::Completions::ParserError- # Note: We're parsing JSON content that was itself embedded as a string inside another JSON object.- # During the outer JSON.parse, any escaped control characters (like "\\n") are unescaped to real characters ("\n"),- # which corrupts the inner JSON structure when passed to the parser here.- # To handle this, we retry parsing with the string JSON-escaped again (`.dump[1..-2]`) if the first attempt fails.- try_escape_and_parse(raw_json, pre_append_buffer)- return if @broken+ if !raw_json.is_a?(String)+ @broken = true+ returnend- if @parser.state == :start_string && @current_key- buffered = @tracking_array ? [@parser.buf] : @parser.buf- # this is is worth notifying- stream_consumer.notify_progress(@current_key, buffered)- end+ @escaped_buffer << DiscourseAi::Utils::BestEffortJsonParser.escape_control_characters(+ raw_json,+ )- @current_key = nil if @parser.state == :end_value+ parsed =+ begin+ @completer.parse(@escaped_buffer)+ rescue JsonCompleter::ParseError+ @broken = true+ return+ end++ notify_changes(parsed) if parsed.is_a?(Hash)endprivate- def mark_broken!- @broken = true- @tracking_array = false- @current_key = nil- @current_value = nil- end+ def notify_changes(parsed)+ parsed.each do |key, value|+ next if value.nil?+ next if @last_notified[key] == value- def try_escape_and_parse(raw_json, pre_append_buffer)- if !raw_json.is_a?(String)- @broken = true- return+ # the completer mutates parsed containers in place between calls, so+ # compare against a snapshot+ @last_notified[key] = value.deep_dup+ stream_consumer.notify_progress(key, value)end- # Escape the string as JSON and remove surrounding quotes- escaped_json = raw_json.dump[1..-2]-- # Assume we could have already processed some of the chunk which was stored in the parser's buffer.- already_processed_chunk = @parser.buf- last_seen_char_idx = already_processed_chunk.length - pre_append_buffer.length-- escaped_json = escaped_json[last_seen_char_idx..]-- @parser << escaped_json- rescue DiscourseAi::Completions::ParserError- @broken = trueendendenddiff --git a/plugins/discourse-ai/lib/completions/structured_output.rb b/plugins/discourse-ai/lib/completions/structured_output.rbindex 1f726899adc..8405663f382 100644--- a/plugins/discourse-ai/lib/completions/structured_output.rb+++ b/plugins/discourse-ai/lib/completions/structured_output.rb@@ -58,6 +58,7 @@ module DiscourseAiif @partial_json_tracker.broken?if @donereturn nil if @type_map[prop_name.to_sym].nil?+ log_broken_streamreturn(DiscourseAi::Utils::BestEffortJsonParser.extract_key(@raw_response,@@ -90,6 +91,17 @@ module DiscourseAi@tracked[key_sym] = valueend++ private++ def log_broken_stream+ return if @broken_logged+ @broken_logged = true+ Rails.logger.warn(+ "Discourse AI: structured output response was not valid JSON, " \+ "falling back to best-effort parsing (#{@raw_response.bytesize} bytes)",+ )+ endendendenddiff --git a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rbindex c2f2f2bf2e4..a050706575d 100644--- a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb+++ b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb@@ -1,7 +1,5 @@# frozen_string_literal: true-require "json"-module DiscourseAimodule Utilsclass BestEffortJsonParser@@ -10,108 +8,62 @@ module DiscourseAireturn helper_response unless helper_response.is_a?(String)schema_type = schema_type.to_sym- schema_key = schema_key&.to_sym+ key = schema_key.to_scleaned = remove_markdown_fences(helper_response.strip)- parsed =- try_parse(cleaned) || try_parse(fix_common_issues(cleaned)) ||- manual_extract(cleaned, schema_key, schema_type)-- value = parsed.is_a?(Hash) ? parsed[schema_key.to_s] : parsed+ parsed = best_effort_parse(cleaned, key)+ value = parsed.is_a?(Hash) ? parsed[key] : parsedcast_value(value, schema_type)end- private-- def remove_markdown_fences(text)- return text unless text.match?(/^```(?:json)?\s*\n/i)-- text.gsub(/^```(?:json)?\s*\n/i, "").gsub(/\n```\s*$/, "")- end-- def fix_common_issues(text)- text.gsub(/(\w+):/, '"\1":').gsub(/'/, "\"")- end-- def try_parse(text)- JSON.parse(text)- rescue JSON::ParserError- nil- end-- def manual_extract(text, key, schema_type)- return default_for(schema_type) unless key-- case schema_type- when :object- extract_object(text, key.to_s)- when :array, :string- extract_scalar(text, key.to_s, schema_type)- else- default_for(schema_type)- end+ # Some providers deliver JSON whose string values had their control+ # characters unescaped by an outer JSON parse (real newlines inside+ # string values). Re-escaping them restores a parseable document.+ def escape_control_characters(text)+ text.gsub(/[\x00-\x1F]/) { |char| format("\\u%04x", char.ord) }end- def extract_scalar(text, key, schema_type)- patterns =- if schema_type == :array- [- /"#{key}"\s*:\s*\[([^\]]+)\]/,- /'#{key}'\s*:\s*\[([^\]]+)\]/,- /#{key}\s*:\s*\[([^\]]+)\]/,- ]- else- [- /"#{key}"\s*:\s*"([^"]+)"/,- /'#{key}'\s*:\s*'([^']+)'/,- /#{key}\s*:\s*"([^"]+)"/,- /#{key}\s*:\s*'([^']+)'/,- ]- end-- patterns.each do |pattern|- match = text.match(pattern)- next unless match+ private- value = match[1]- return schema_type == :array ? parse_array(value) : value+ # Parse attempts, in order:+ # 1. JsonCompleter: strict JSON that tolerates truncation, completing+ # unclosed structures and strings.+ # 2. JsonCompleter over a control-character-escaped copy: recovers+ # values containing real newlines/tabs.+ # 3. SmarterJSON: lenient parsing for single quotes, unquoted keys,+ # prose-wrapped JSON, and other LLM formatting quirks.+ #+ # A candidate wins if it holds the schema key; a parseable document+ # missing the key is kept only as a last resort.+ def best_effort_parse(cleaned, key)+ attempts = [+ -> { JsonCompleter.parse(cleaned) },+ -> { JsonCompleter.parse(escape_control_characters(cleaned)) },+ -> { SmarterJSON.process_one(cleaned) },+ ]++ fallback = nil+ attempts.each do |attempt|+ candidate =+ begin+ attempt.call+ rescue StandardError+ nil+ end++ next if candidate.nil?+ return candidate if !candidate.is_a?(Hash) || candidate.key?(key)+ fallback ||= candidateend- default_for(schema_type)+ fallbackend- def parse_array(value)- JSON.parse("[#{value}]")- rescue JSON::ParserError- value.split(",").map { |item| item.strip.gsub(/^['"]|['"]$/, "") }- end-- def extract_object(text, key)- pattern = /("#{key}"|'#{key}'|#{key})\s*:\s*\{/- match = text.match(pattern) or return {}-- start = match.end(0) - 1- return {} unless text[start] == "{"-- end_pos = find_matching_brace(text, start)- return {} unless end_pos-- obj_str = text[start..end_pos]- try_parse(obj_str) || try_parse(fix_common_issues(obj_str)) || {}- end-- def find_matching_brace(text, start_pos)- brace_count = 0+ def remove_markdown_fences(text)+ return text unless text.match?(/^```(?:json)?\s*\n/i)- text[start_pos..-1].each_char.with_index do |char, idx|- brace_count += 1 if char == "{"- if char == "}"- brace_count -= 1- return start_pos + idx if brace_count.zero?- end- end- nil+ text.gsub(/^```(?:json)?\s*\n/i, "").gsub(/\n```\s*$/, "")enddef cast_value(value, schema_type)@@ -123,14 +75,13 @@ module DiscourseAiwhen :booleanreturn value if [true, false, nil].include?(value)value.to_s.downcase == "true"+ when :integer, :number+ return value if value.is_a?(Numeric)+ Integer(value.to_s, exception: false) || Float(value.to_s, exception: false)elsevalue.to_sendend-- def default_for(schema_type)- schema_type == :array ? [] : schema_type == :object ? {} : ""- endendendenddiff --git a/plugins/discourse-ai/plugin.rb b/plugins/discourse-ai/plugin.rbindex 2bd6de3cb24..304a92edc83 100644--- a/plugins/discourse-ai/plugin.rb+++ b/plugins/discourse-ai/plugin.rb@@ -11,6 +11,8 @@ require "tokenizers"require "tiktoken_ruby"require "discourse_ai/tokenizers"require "ed25519"+require "smarter_json"+require "json_completer"enabled_site_setting :discourse_ai_enableddiff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rbindex 3766d35244d..cc545b4f7a7 100644--- a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb@@ -1024,7 +1024,7 @@ TEXTendend- expect(partials.compact.length).to eq(128)+ expect(partials.compact.length).to eq(134)params =partialsdiff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rbindex 8b45aab8eec..8ffc129de15 100644--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb@@ -53,10 +53,10 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput dostructured_output << chunks[3]expect(structured_output.read_buffered_property(:bool)).to eq(true)- # Waiting for number to be fully buffered.+ # Numbers stream too: mid-stream reads return the value buffered so far.structured_output << chunks[4]expect(structured_output.read_buffered_property(:bool)).to eq(true)- expect(structured_output.read_buffered_property(:number)).to be_nil+ expect(structured_output.read_buffered_property(:number)).to eq(4)structured_output << chunks[5]expect(structured_output.read_buffered_property(:number)).to eq(42)@@ -102,7 +102,9 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput dostructured_output << chunks[6]structured_output << chunks[7]- expect(structured_output.read_buffered_property(:list)).to eq(["Hello! I am a chunk"])+ # A trailing comma opens a slot for the next element, which reads as a+ # nil placeholder until its value arrives.+ expect(structured_output.read_buffered_property(:list)).to eq(["Hello! I am a chunk", nil])structured_output << chunks[8]expect(structured_output.read_buffered_property(:list)).to eq(@@ -128,14 +130,16 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput dodescribed_class.new({ ratings: { type: "array", items: { type: "object" } } })end- it "falls back to full parsing without raising" do+ it "streams partial objects and completes them at the end" dochunks = [+"{\"ratings\":[{\"candidate\":\"alpha\",\"rating\":9",+"},{\"candidate\":\"bravo\",\"rating\":6}]}",]- expect { structured_output << chunks[0] }.not_to raise_error- expect(structured_output.read_buffered_property(:ratings)).to eq(nil)+ structured_output << chunks[0]+ expect(structured_output.read_buffered_property(:ratings)).to eq(+ [{ "candidate" => "alpha", "rating" => 9 }],+ )structured_output << chunks[1]structured_output.finish@@ -147,6 +151,37 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput doendend+ describe "recovering from streams with unescaped control characters" do+ # regression specs for https://meta.discourse.org/t/407251+ it "streams the full content instead of truncating at escaped quotes" do+ content = "Erste Zeile.\n\nZweite Zeile mit einem \"Zitat\" und 😊 Emoji."+ raw = { message: content }.to_json.gsub("\\n", "\n")++ streamed = +""+ raw+ .each_char+ .each_slice(13) do |chunk|+ structured_output << chunk.join+ streamed << (structured_output.read_buffered_property(:message) || "")+ end+ structured_output.finish+ streamed << (structured_output.read_buffered_property(:message) || "")++ expect(structured_output.broken?).to eq(false)+ expect(streamed).to eq(content)+ end++ it "recovers the full content at finish when the response is also fenced" do+ content = "A \"quoted\" word\nand a second line"+ raw = "```json\n#{{ message: content }.to_json.gsub("\\n", "\n")}\n```"++ structured_output << raw+ structured_output.finish++ expect(structured_output.read_buffered_property(:message)).to eq(content)+ end+ end+describe "dealing with non-JSON responses" doit "treat it as plain text once we determined it's invalid JSON" dochunks = [+"I'm not", +"a", +"JSON :)"]diff --git a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rbindex 002d85865bc..db6d2f4bdad 100644--- a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb+++ b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb@@ -161,6 +161,30 @@ RSpec.describe DiscourseAi::Utils::BestEffortJsonParser doendend+ context "with responses where JSON streaming broke" do+ # regression specs for https://meta.discourse.org/t/407251+ it "extracts the complete value when string values contain real newlines" do+ content = "First line.\n\nA \"quoted\" word 😊 and everything after it."+ input = { output: content }.to_json.gsub("\\n", "\n")++ result = described_class.extract_key(input, "string", :output)++ expect(result).to eq(content)+ end++ it "completes truncated JSON instead of scraping garbage" do+ input = "{\"output\": \"A translation that got cut off midway thro"++ result = described_class.extract_key(input, "string", :output)++ expect(result).to eq("A translation that got cut off midway thro")+ end++ it "casts values to numbers for integer schemas" do+ expect(described_class.extract_key('{"output": 42}', "integer", :output)).to eq(42)+ end+ end+context "when very broken JSON is entered" doit "returns empty string when no valid JSON can be extracted for string type" doinput = "This is just plain text with no JSON"