Skip to content

A platform for community discussion. Free, open, simple.

FIX: Silent truncation and corruption of AI structured output streams (#41716)

Rafael dos Santos Silva87775447c04a
diff --git a/Gemfile b/Gemfile
index 2df98e77f04..0cb16063f2c 100644
--- a/Gemfile
+++ b/Gemfile
@@ -305,6 +305,8 @@ gem "octokit", require: false
# for discourse-ai
gem "tokenizers", require: false
gem "tiktoken_ruby", require: false
+gem "smarter_json", require: false
+gem "json_completer", require: false
gem "discourse_ai-tokenizers", require: false
gem "ed25519" # TODO: remove this as existing ssl gem should handle this
gem "Ascii85", require: false
diff --git a/Gemfile.lock b/Gemfile.lock
index d46cace862e..f29f9722096 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -305,6 +305,7 @@ GEM
json-schema (6.2.0)
addressable (~> 2.8)
bigdecimal (>= 3.1, < 5)
+ json_completer (1.2.0)
json_schemer (2.5.0)
bigdecimal
hana (~> 1.3)
@@ -720,6 +721,8 @@ GEM
simplecov-html (0.13.2)
simplecov_json_formatter (0.1.4)
simpleidn (0.2.3)
+ smarter_json (1.2.6)
+ bigdecimal
snaky_hash (2.0.4)
hashie (>= 0.1.0, < 6)
version_gem (>= 1.1.8, < 3)
@@ -869,6 +872,7 @@ DEPENDENCIES
inflection
iso8601
json
+ json_completer
json_schemer
landlock
liquid (= 5.12.0)
@@ -958,6 +962,7 @@ DEPENDENCIES
shoulda-matchers
sidekiq (>= 7.3.10)
simplecov
+ smarter_json
sqlite3
sshkey
stackprof
@@ -1095,6 +1100,7 @@ CHECKSUMS
jmespath (1.6.2) sha256=238d774a58723d6c090494c8879b5e9918c19485f7e840f2c1c7532cf84ebcb1
json (2.19.9) sha256=9b9025b7cdddafa38d316eca0b2358488e42d417045c1b90d216a9fefe46b79a
json-schema (6.2.0) sha256=e8bff46ed845a22c1ab2bd0d7eccf831c01fe23bb3920caa4c74db4306813666
+ json_completer (1.2.0) sha256=4665749172634eccb208c562eb59ea81dd7a612a1fa9a36df441ac473ff177b1
json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396
jwt (2.10.1) sha256=e6424ae1d813f63e761a04d6284e10e7ec531d6f701917fadcd0d9b2deaf1cc5
landlock (0.2) sha256=9bb86d7d12b16485b278ba575d49f55b730089c7a3d0f45a769fe4d7384d5e55
@@ -1275,6 +1281,7 @@ CHECKSUMS
simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
simpleidn (0.2.3) sha256=08ce96f03fa1605286be22651ba0fc9c0b2d6272c9b27a260bc88be05b0d2c29
+ smarter_json (1.2.6) sha256=6638a2d23954e26d268212b4c8f5b9871d31b72f3d169371cca1082c1a722b45
snaky_hash (2.0.4) sha256=2b12758c57defa6796341a1620f84b1a23737421d8d7e2575d0550b53cc4fece
sqlite3 (2.9.5-aarch64-linux-gnu) sha256=78075b6337d3d182c6d2b4691049ed45cd220826160c9ea18946bf6a1de200dc
sqlite3 (2.9.5-aarch64-linux-musl) sha256=18c801185deb4adc01ddb281e8f672a39e3d1729979ca91e39439cd3eac0402d
diff --git a/plugins/discourse-ai/lib/completions/json_streaming_parser.rb b/plugins/discourse-ai/lib/completions/json_streaming_parser.rb
deleted file mode 100644
index f3288ab6bd8..00000000000
--- a/plugins/discourse-ai/lib/completions/json_streaming_parser.rb
+++ /dev/null
@@ -1,668 +0,0 @@
-# frozen_string_literal: true
-
-# This code is copied from the MIT licensed json-stream
-# see: https://github.com/dgraham/json-stream
-#
-# It was copied to avoid the dependency and allow us to make some small changes
-# particularly we need better access to internal state when parsing
-
-module DiscourseAi
- module Completions
- # Raised on any invalid JSON text.
- ParserError = Class.new(RuntimeError)
-
- # A streaming JSON parser that generates SAX-like events for state changes.
- # Use the json gem for small documents. Use this for huge documents that
- # won't fit in memory.
- #
- # Examples
- #
- # parser = JSON::Stream::Parser.new
- # parser.key { |key| puts key }
- # parser.value { |value| puts value }
- # parser << '{"answer":'
- # parser << ' 42}'
- class JsonStreamingParser
- # our changes:
- attr_reader :state, :buf, :pos
-
- # A character buffer that expects a UTF-8 encoded stream of bytes.
- # This handles truncated multi-byte characters properly so we can just
- # feed it binary data and receive a properly formatted UTF-8 String as
- # output.
- #
- # More UTF-8 parsing details are available at:
- #
- # http://en.wikipedia.org/wiki/UTF-8
- # http://tools.ietf.org/html/rfc3629#section-3
- class Buffer
- def initialize
- @state = :start
- @buffer = []
- @need = 0
- end
-
- # Fill the buffer with a String of binary UTF-8 encoded bytes. Returns
- # as much of the data in a UTF-8 String as we have. Truncated multi-byte
- # characters are saved in the buffer until the next call to this method
- # where we expect to receive the rest of the multi-byte character.
- #
- # data - The partial binary encoded String data.
- #
- # Raises JSON::Stream::ParserError if the UTF-8 byte sequence is malformed.
- #
- # Returns a UTF-8 encoded String.
- def <<(data)
- data = data.dup if data.frozen?
- # Avoid state machine for complete UTF-8.
- if @buffer.empty?
- data.force_encoding(Encoding::UTF_8)
- return data if data.valid_encoding?
- end
-
- bytes = []
- data.each_byte do |byte|
- case @state
- when :start
- if byte < 128
- bytes << byte
- elsif byte >= 192
- @state = :multi_byte
- @buffer << byte
- @need =
- case
- when byte >= 240
- 4
- when byte >= 224
- 3
- when byte >= 192
- 2
- end
- else
- error("Expected start of multi-byte or single byte char")
- end
- when :multi_byte
- if byte > 127 && byte < 192
- @buffer << byte
- if @buffer.size == @need
- bytes += @buffer.slice!(0, @buffer.size)
- @state = :start
- end
- else
- error("Expected continuation byte")
- end
- end
- end
-
- # Build UTF-8 encoded string from completed codepoints.
- bytes
- .pack("C*")
- .force_encoding(Encoding::UTF_8)
- .tap { |text| error("Invalid UTF-8 byte sequence") unless text.valid_encoding? }
- end
-
- # Determine if the buffer contains partial UTF-8 continuation bytes that
- # are waiting on subsequent completion bytes before a full codepoint is
- # formed.
- #
- # Examples
- #
- # bytes = "é".bytes
- #
- # buffer << bytes[0]
- # buffer.empty?
- # # => false
- #
- # buffer << bytes[1]
- # buffer.empty?
- # # => true
- #
- # Returns true if the buffer is empty.
- def empty?
- @buffer.empty?
- end
-
- private
-
- def error(message)
- raise ParserError, message
- end
- end
-
- BUF_SIZE = 4096
- CONTROL = /[\x00-\x1F]/
- WS = /[ \n\t\r]/
- HEX = /[0-9a-fA-F]/
- DIGIT = /[0-9]/
- DIGIT_1_9 = /[1-9]/
- DIGIT_END = /\d$/
- TRUE_RE = /[rue]/
- FALSE_RE = /[alse]/
- NULL_RE = /[ul]/
- TRUE_KEYWORD = "true"
- FALSE_KEYWORD = "false"
- NULL_KEYWORD = "null"
- LEFT_BRACE = "{"
- RIGHT_BRACE = "}"
- LEFT_BRACKET = "["
- RIGHT_BRACKET = "]"
- BACKSLASH = '\\'
- SLASH = "/"
- QUOTE = '"'
- COMMA = ","
- COLON = ":"
- ZERO = "0"
- MINUS = "-"
- PLUS = "+"
- POINT = "."
- EXPONENT = /[eE]/
- B, F, N, R, T, U = %w[b f n r t u]
-
- # Create a new parser with an optional initialization block where
- # we can register event callbacks.
- #
- # Examples
- #
- # parser = JSON::Stream::Parser.new do
- # start_document { puts "start document" }
- # end_document { puts "end document" }
- # start_object { puts "start object" }
- # end_object { puts "end object" }
- # start_array { puts "start array" }
- # end_array { puts "end array" }
- # key { |k| puts "key: #{k}" }
- # value { |v| puts "value: #{v}" }
- # end
- def initialize(&block)
- @state = :start_document
- @utf8 = Buffer.new
- @listeners = {
- start_document: [],
- end_document: [],
- start_object: [],
- end_object: [],
- start_array: [],
- end_array: [],
- key: [],
- value: [],
- }
-
- # Track parse stack.
- @stack = []
- @unicode = +""
- @buf = +""
- @pos = -1
-
- # Register any observers in the block.
- instance_eval(&block) if block_given?
- end
-
- def start_document(&block)
- @listeners[:start_document] << block
- end
-
- def end_document(&block)
- @listeners[:end_document] << block
- end
-
- def start_object(&block)
- @listeners[:start_object] << block
- end
-
- def end_object(&block)
- @listeners[:end_object] << block
- end
-
- def start_array(&block)
- @listeners[:start_array] << block
- end
-
- def end_array(&block)
- @listeners[:end_array] << block
- end
-
- def key(&block)
- @listeners[:key] << block
- end
-
- def value(&block)
- @listeners[:value] << block
- end
-
- # Pass data into the parser to advance the state machine and
- # generate callback events. This is well suited for an EventMachine
- # receive_data loop.
- #
- # data - The String of partial JSON data to parse.
- #
- # Raises a JSON::Stream::ParserError if the JSON data is malformed.
- #
- # Returns nothing.
- def <<(data)
- (@utf8 << data).each_char do |ch|
- @pos += 1
- case @state
- when :start_document
- start_value(ch)
- when :start_object
- case ch
- when QUOTE
- @state = :start_string
- @stack.push(:key)
- when RIGHT_BRACE
- end_container(:object)
- when WS
- # ignore
- else
- error("Expected object key start")
- end
- when :start_string
- case ch
- when QUOTE
- if @stack.pop == :string
- end_value(@buf)
- else # :key
- @state = :end_key
- notify(:key, @buf)
- end
- @buf = +""
- when BACKSLASH
- @state = :start_escape
- when CONTROL
- error("Control characters must be escaped")
- else
- @buf << ch
- end
- when :start_escape
- case ch
- when QUOTE, BACKSLASH, SLASH
- @buf << ch
- @state = :start_string
- when B
- @buf << "\b"
- @state = :start_string
- when F
- @buf << "\f"
- @state = :start_string
- when N
- @buf << "\n"
- @state = :start_string
- when R
- @buf << "\r"
- @state = :start_string
- when T
- @buf << "\t"
- @state = :start_string
- when U
- @state = :unicode_escape
- else
- error("Expected escaped character")
- end
- when :unicode_escape
- case ch
- when HEX
- @unicode << ch
- if @unicode.size == 4
- codepoint = @unicode.slice!(0, 4).hex
- if codepoint >= 0xD800 && codepoint <= 0xDBFF
- error("Expected low surrogate pair half") if @stack[-1].is_a?(Integer)
- @state = :start_surrogate_pair
- @stack.push(codepoint)
- elsif codepoint >= 0xDC00 && codepoint <= 0xDFFF
- high = @stack.pop
- error("Expected high surrogate pair half") unless high.is_a?(Integer)
- pair = ((high - 0xD800) * 0x400) + (codepoint - 0xDC00) + 0x10000
- @buf << pair
- @state = :start_string
- else
- @buf << codepoint
- @state = :start_string
- end
- end
- else
- error("Expected unicode escape hex digit")
- end
- when :start_surrogate_pair
- case ch
- when BACKSLASH
- @state = :start_surrogate_pair_u
- else
- error("Expected low surrogate pair half")
- end
- when :start_surrogate_pair_u
- case ch
- when U
- @state = :unicode_escape
- else
- error("Expected low surrogate pair half")
- end
- when :start_negative_number
- case ch
- when ZERO
- @state = :start_zero
- @buf << ch
- when DIGIT_1_9
- @state = :start_int
- @buf << ch
- else
- error("Expected 0-9 digit")
- end
- when :start_zero
- case ch
- when POINT
- @state = :start_float
- @buf << ch
- when EXPONENT
- @state = :start_exponent
- @buf << ch
- else
- end_value(@buf.to_i)
- @buf = +""
- @pos -= 1
- redo
- end
- when :start_float
- case ch
- when DIGIT
- @state = :in_float
- @buf << ch
- else
- error("Expected 0-9 digit")
- end
- when :in_float
- case ch
- when DIGIT
- @buf << ch
- when EXPONENT
- @state = :start_exponent
- @buf << ch
- else
- end_value(@buf.to_f)
- @buf = +""
- @pos -= 1
- redo
- end
- when :start_exponent
- case ch
- when MINUS, PLUS, DIGIT
- @state = :in_exponent
- @buf << ch
- else
- error("Expected +, -, or 0-9 digit")
- end
- when :in_exponent
- case ch
- when DIGIT
- @buf << ch
- else
- error("Expected 0-9 digit") unless @buf =~ DIGIT_END
- end_value(@buf.to_f)
- @buf = +""
- @pos -= 1
- redo
- end
- when :start_int
- case ch
- when DIGIT
- @buf << ch
- when POINT
- @state = :start_float
- @buf << ch
- when EXPONENT
- @state = :start_exponent
- @buf << ch
- else
- end_value(@buf.to_i)
- @buf = +""
- @pos -= 1
- redo
- end
- when :start_true
- keyword(TRUE_KEYWORD, true, TRUE_RE, ch)
- when :start_false
- keyword(FALSE_KEYWORD, false, FALSE_RE, ch)
- when :start_null
- keyword(NULL_KEYWORD, nil, NULL_RE, ch)
- when :end_key
- case ch
- when COLON
- @state = :key_sep
- when WS
- # ignore
- else
- error("Expected colon key separator")
- end
- when :key_sep
- start_value(ch)
- when :start_array
- case ch
- when RIGHT_BRACKET
- end_container(:array)
- when WS
- # ignore
- else
- start_value(ch)
- end
- when :end_value
- case ch
- when COMMA
- @state = :value_sep
- when RIGHT_BRACE
- end_container(:object)
- when RIGHT_BRACKET
- end_container(:array)
- when WS
- # ignore
- else
- error("Expected comma or object or array close")
- end
- when :value_sep
- if @stack[-1] == :object
- case ch
- when QUOTE
- @state = :start_string
- @stack.push(:key)
- when WS
- # ignore
- else
- error("Expected object key start")
- end
- else
- start_value(ch)
- end
- when :end_document
- error("Unexpected data") unless ch =~ WS
- end
- end
- end
-
- # Drain any remaining buffered characters into the parser to complete
- # the parsing of the document.
- #
- # This is only required when parsing a document containing a single
- # numeric value, integer or float. The parser has no other way to
- # detect when it should no longer expect additional characters with
- # which to complete the parse, so it must be signaled by a call to
- # this method.
- #
- # If you're parsing more typical object or array documents, there's no
- # need to call `finish` because the parse will complete when the final
- # closing `]` or `}` character is scanned.
- #
- # Raises a JSON::Stream::ParserError if the JSON data is malformed.
- #
- # Returns nothing.
- def finish
- # Partial multi-byte character waiting for completion bytes.
- error("Unexpected end-of-file") unless @utf8.empty?
-
- # Partial array, object, or string.
- error("Unexpected end-of-file") unless @stack.empty?
-
- case @state
- when :end_document
- # done, do nothing
- when :in_float
- end_value(@buf.to_f)
- when :in_exponent
- error("Unexpected end-of-file") unless @buf =~ DIGIT_END
- end_value(@buf.to_f)
- when :start_zero
- end_value(@buf.to_i)
- when :start_int
- end_value(@buf.to_i)
- else
- error("Unexpected end-of-file")
- end
- end
-
- private
-
- # Invoke all registered observer procs for the event type.
- #
- # type - The Symbol listener name.
- # args - The argument list to pass into the observer procs.
- #
- # Examples
- #
- # # broadcast events for {"answer": 42}
- # notify(:start_object)
- # notify(:key, "answer")
- # notify(:value, 42)
- # notify(:end_object)
- #
- # Returns nothing.
- def notify(type, *args)
- @listeners[type].each { |block| block.call(*args) }
- end
-
- # Complete an object or array container value type.
- #
- # type - The Symbol, :object or :array, of the expected type.
- #
- # Raises a JSON::Stream::ParserError if the expected container type
- # was not completed.
- #
- # Returns nothing.
- def end_container(type)
- @state = :end_value
- if @stack.pop == type
- case type
- when :object
- notify(:end_object)
- when :array
- notify(:end_array)
- end
- else
- error("Expected end of #{type}")
- end
- notify_end_document if @stack.empty?
- end
-
- # Broadcast an `end_document` event to observers after a complete JSON
- # value document (object, array, number, string, true, false, null) has
- # been parsed from the text. This is the final event sent to observers
- # and signals the parse has finished.
- #
- # Returns nothing.
- def notify_end_document
- @state = :end_document
- notify(:end_document)
- end
-
- # Parse one of the three allowed keywords: true, false, null.
- #
- # word - The String keyword ('true', 'false', 'null').
- # value - The Ruby value (true, false, nil).
- # re - The Regexp of allowed keyword characters.
- # ch - The current String character being parsed.
- #
- # Raises a JSON::Stream::ParserError if the character does not belong
- # in the expected keyword.
- #
- # Returns nothing.
- def keyword(word, value, re, ch)
- if ch =~ re
- @buf << ch
- else
- error("Expected #{word} keyword")
- end
-
- if @buf.size == word.size
- if @buf == word
- @buf = +""
- end_value(value)
- else
- error("Expected #{word} keyword")
- end
- end
- end
-
- # Process the first character of one of the seven possible JSON
- # values: object, array, string, true, false, null, number.
- #
- # ch - The current character String.
- #
- # Raises a JSON::Stream::ParserError if the character does not signal
- # the start of a value.
- #
- # Returns nothing.
- def start_value(ch)
- case ch
- when LEFT_BRACE
- notify(:start_document) if @stack.empty?
- @state = :start_object
- @stack.push(:object)
- notify(:start_object)
- when LEFT_BRACKET
- notify(:start_document) if @stack.empty?
- @state = :start_array
- @stack.push(:array)
- notify(:start_array)
- when QUOTE
- @state = :start_string
- @stack.push(:string)
- when T
- @state = :start_true
- @buf << ch
- when F
- @state = :start_false
- @buf << ch
- when N
- @state = :start_null
- @buf << ch
- when MINUS
- @state = :start_negative_number
- @buf << ch
- when ZERO
- @state = :start_zero
- @buf << ch
- when DIGIT_1_9
- @state = :start_int
- @buf << ch
- when WS
- # ignore
- else
- error("Expected value")
- end
- end
-
- # Advance the state machine and notify `value` observers that a
- # string, number or keyword (true, false, null) value was parsed.
- #
- # value - The object to broadcast to observers.
- #
- # Returns nothing.
- def end_value(value)
- @state = :end_value
- notify(:start_document) if @stack.empty?
- notify(:value, value)
- notify_end_document if @stack.empty?
- end
-
- def error(message)
- raise ParserError, "#{message}: char #{@pos}"
- end
- end
- end
-end
diff --git a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
index b4de6424fa6..ec97acffd83 100644
--- a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
+++ b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
@@ -3,55 +3,14 @@
module DiscourseAi
module Completions
class JsonStreamingTracker
- attr_reader :current_key, :current_value, :stream_consumer
+ attr_reader :stream_consumer
def initialize(stream_consumer)
@stream_consumer = stream_consumer
- @current_key = nil
- @current_value = nil
- @tracking_array = false
+ @escaped_buffer = +""
+ @completer = JsonCompleter.new
@broken = false
- @parser = DiscourseAi::Completions::JsonStreamingParser.new
-
- @parser.key do |k|
- next if @broken
- if @tracking_array && @current_key.present?
- mark_broken!
- next
- end
-
- @current_key = k
- @current_value = nil
- end
-
- @parser.value do |value|
- next if @broken
- if @current_key
- if @tracking_array
- @current_value ||= []
- @current_value << value
- stream_consumer.notify_progress(@current_key, @current_value)
- else
- stream_consumer.notify_progress(@current_key, value)
- @current_key = nil
- end
- end
- end
-
- @parser.start_array do
- next if @broken
- next if !@current_key
-
- @tracking_array = true
- @current_value = []
- end
-
- @parser.end_array do
- next if @broken
- @tracking_array = false
- @current_key = nil
- @current_value = nil
- end
+ @last_notified = {}
end
def broken?
@@ -59,58 +18,40 @@ module DiscourseAi
end
def <<(raw_json)
- # llm could send broken json
- # in that case just deal with it later
- # don't stream
return if @broken
- begin
- pre_append_buffer = @parser.buf.dup
- @parser << raw_json
- rescue DiscourseAi::Completions::ParserError
- # Note: We're parsing JSON content that was itself embedded as a string inside another JSON object.
- # During the outer JSON.parse, any escaped control characters (like "\\n") are unescaped to real characters ("\n"),
- # which corrupts the inner JSON structure when passed to the parser here.
- # To handle this, we retry parsing with the string JSON-escaped again (`.dump[1..-2]`) if the first attempt fails.
- try_escape_and_parse(raw_json, pre_append_buffer)
- return if @broken
+ if !raw_json.is_a?(String)
+ @broken = true
+ return
end
- if @parser.state == :start_string && @current_key
- buffered = @tracking_array ? [@parser.buf] : @parser.buf
- # this is is worth notifying
- stream_consumer.notify_progress(@current_key, buffered)
- end
+ @escaped_buffer << DiscourseAi::Utils::BestEffortJsonParser.escape_control_characters(
+ raw_json,
+ )
- @current_key = nil if @parser.state == :end_value
+ parsed =
+ begin
+ @completer.parse(@escaped_buffer)
+ rescue JsonCompleter::ParseError
+ @broken = true
+ return
+ end
+
+ notify_changes(parsed) if parsed.is_a?(Hash)
end
private
- def mark_broken!
- @broken = true
- @tracking_array = false
- @current_key = nil
- @current_value = nil
- end
+ def notify_changes(parsed)
+ parsed.each do |key, value|
+ next if value.nil?
+ next if @last_notified[key] == value
- def try_escape_and_parse(raw_json, pre_append_buffer)
- if !raw_json.is_a?(String)
- @broken = true
- return
+ # the completer mutates parsed containers in place between calls, so
+ # compare against a snapshot
+ @last_notified[key] = value.deep_dup
+ stream_consumer.notify_progress(key, value)
end
- # Escape the string as JSON and remove surrounding quotes
- escaped_json = raw_json.dump[1..-2]
-
- # Assume we could have already processed some of the chunk which was stored in the parser's buffer.
- already_processed_chunk = @parser.buf
- last_seen_char_idx = already_processed_chunk.length - pre_append_buffer.length
-
- escaped_json = escaped_json[last_seen_char_idx..]
-
- @parser << escaped_json
- rescue DiscourseAi::Completions::ParserError
- @broken = true
end
end
end
diff --git a/plugins/discourse-ai/lib/completions/structured_output.rb b/plugins/discourse-ai/lib/completions/structured_output.rb
index 1f726899adc..8405663f382 100644
--- a/plugins/discourse-ai/lib/completions/structured_output.rb
+++ b/plugins/discourse-ai/lib/completions/structured_output.rb
@@ -58,6 +58,7 @@ module DiscourseAi
if @partial_json_tracker.broken?
if @done
return nil if @type_map[prop_name.to_sym].nil?
+ log_broken_stream
return(
DiscourseAi::Utils::BestEffortJsonParser.extract_key(
@raw_response,
@@ -90,6 +91,17 @@ module DiscourseAi
@tracked[key_sym] = value
end
+
+ private
+
+ def log_broken_stream
+ return if @broken_logged
+ @broken_logged = true
+ Rails.logger.warn(
+ "Discourse AI: structured output response was not valid JSON, " \
+ "falling back to best-effort parsing (#{@raw_response.bytesize} bytes)",
+ )
+ end
end
end
end
diff --git a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb
index c2f2f2bf2e4..a050706575d 100644
--- a/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb
+++ b/plugins/discourse-ai/lib/utils/best_effort_json_parser.rb
@@ -1,7 +1,5 @@
# frozen_string_literal: true
-require "json"
-
module DiscourseAi
module Utils
class BestEffortJsonParser
@@ -10,108 +8,62 @@ module DiscourseAi
return helper_response unless helper_response.is_a?(String)
schema_type = schema_type.to_sym
- schema_key = schema_key&.to_sym
+ key = schema_key.to_s
cleaned = remove_markdown_fences(helper_response.strip)
- parsed =
- try_parse(cleaned) || try_parse(fix_common_issues(cleaned)) ||
- manual_extract(cleaned, schema_key, schema_type)
-
- value = parsed.is_a?(Hash) ? parsed[schema_key.to_s] : parsed
+ parsed = best_effort_parse(cleaned, key)
+ value = parsed.is_a?(Hash) ? parsed[key] : parsed
cast_value(value, schema_type)
end
- private
-
- def remove_markdown_fences(text)
- return text unless text.match?(/^```(?:json)?\s*\n/i)
-
- text.gsub(/^```(?:json)?\s*\n/i, "").gsub(/\n```\s*$/, "")
- end
-
- def fix_common_issues(text)
- text.gsub(/(\w+):/, '"\1":').gsub(/'/, "\"")
- end
-
- def try_parse(text)
- JSON.parse(text)
- rescue JSON::ParserError
- nil
- end
-
- def manual_extract(text, key, schema_type)
- return default_for(schema_type) unless key
-
- case schema_type
- when :object
- extract_object(text, key.to_s)
- when :array, :string
- extract_scalar(text, key.to_s, schema_type)
- else
- default_for(schema_type)
- end
+ # Some providers deliver JSON whose string values had their control
+ # characters unescaped by an outer JSON parse (real newlines inside
+ # string values). Re-escaping them restores a parseable document.
+ def escape_control_characters(text)
+ text.gsub(/[\x00-\x1F]/) { |char| format("\\u%04x", char.ord) }
end
- def extract_scalar(text, key, schema_type)
- patterns =
- if schema_type == :array
- [
- /"#{key}"\s*:\s*\[([^\]]+)\]/,
- /'#{key}'\s*:\s*\[([^\]]+)\]/,
- /#{key}\s*:\s*\[([^\]]+)\]/,
- ]
- else
- [
- /"#{key}"\s*:\s*"([^"]+)"/,
- /'#{key}'\s*:\s*'([^']+)'/,
- /#{key}\s*:\s*"([^"]+)"/,
- /#{key}\s*:\s*'([^']+)'/,
- ]
- end
-
- patterns.each do |pattern|
- match = text.match(pattern)
- next unless match
+ private
- value = match[1]
- return schema_type == :array ? parse_array(value) : value
+ # Parse attempts, in order:
+ # 1. JsonCompleter: strict JSON that tolerates truncation, completing
+ # unclosed structures and strings.
+ # 2. JsonCompleter over a control-character-escaped copy: recovers
+ # values containing real newlines/tabs.
+ # 3. SmarterJSON: lenient parsing for single quotes, unquoted keys,
+ # prose-wrapped JSON, and other LLM formatting quirks.
+ #
+ # A candidate wins if it holds the schema key; a parseable document
+ # missing the key is kept only as a last resort.
+ def best_effort_parse(cleaned, key)
+ attempts = [
+ -> { JsonCompleter.parse(cleaned) },
+ -> { JsonCompleter.parse(escape_control_characters(cleaned)) },
+ -> { SmarterJSON.process_one(cleaned) },
+ ]
+
+ fallback = nil
+ attempts.each do |attempt|
+ candidate =
+ begin
+ attempt.call
+ rescue StandardError
+ nil
+ end
+
+ next if candidate.nil?
+ return candidate if !candidate.is_a?(Hash) || candidate.key?(key)
+ fallback ||= candidate
end
- default_for(schema_type)
+ fallback
end
- def parse_array(value)
- JSON.parse("[#{value}]")
- rescue JSON::ParserError
- value.split(",").map { |item| item.strip.gsub(/^['"]|['"]$/, "") }
- end
-
- def extract_object(text, key)
- pattern = /("#{key}"|'#{key}'|#{key})\s*:\s*\{/
- match = text.match(pattern) or return {}
-
- start = match.end(0) - 1
- return {} unless text[start] == "{"
-
- end_pos = find_matching_brace(text, start)
- return {} unless end_pos
-
- obj_str = text[start..end_pos]
- try_parse(obj_str) || try_parse(fix_common_issues(obj_str)) || {}
- end
-
- def find_matching_brace(text, start_pos)
- brace_count = 0
+ def remove_markdown_fences(text)
+ return text unless text.match?(/^```(?:json)?\s*\n/i)
- text[start_pos..-1].each_char.with_index do |char, idx|
- brace_count += 1 if char == "{"
- if char == "}"
- brace_count -= 1
- return start_pos + idx if brace_count.zero?
- end
- end
- nil
+ text.gsub(/^```(?:json)?\s*\n/i, "").gsub(/\n```\s*$/, "")
end
def cast_value(value, schema_type)
@@ -123,14 +75,13 @@ module DiscourseAi
when :boolean
return value if [true, false, nil].include?(value)
value.to_s.downcase == "true"
+ when :integer, :number
+ return value if value.is_a?(Numeric)
+ Integer(value.to_s, exception: false) || Float(value.to_s, exception: false)
else
value.to_s
end
end
-
- def default_for(schema_type)
- schema_type == :array ? [] : schema_type == :object ? {} : ""
- end
end
end
end
diff --git a/plugins/discourse-ai/plugin.rb b/plugins/discourse-ai/plugin.rb
index 2bd6de3cb24..304a92edc83 100644
--- a/plugins/discourse-ai/plugin.rb
+++ b/plugins/discourse-ai/plugin.rb
@@ -11,6 +11,8 @@ require "tokenizers"
require "tiktoken_ruby"
require "discourse_ai/tokenizers"
require "ed25519"
+require "smarter_json"
+require "json_completer"
enabled_site_setting :discourse_ai_enabled
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
index 3766d35244d..cc545b4f7a7 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
@@ -1024,7 +1024,7 @@ TEXT
end
end
- expect(partials.compact.length).to eq(128)
+ expect(partials.compact.length).to eq(134)
params =
partials
diff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
index 8b45aab8eec..8ffc129de15 100644
--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
@@ -53,10 +53,10 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
structured_output << chunks[3]
expect(structured_output.read_buffered_property(:bool)).to eq(true)
- # Waiting for number to be fully buffered.
+ # Numbers stream too: mid-stream reads return the value buffered so far.
structured_output << chunks[4]
expect(structured_output.read_buffered_property(:bool)).to eq(true)
- expect(structured_output.read_buffered_property(:number)).to be_nil
+ expect(structured_output.read_buffered_property(:number)).to eq(4)
structured_output << chunks[5]
expect(structured_output.read_buffered_property(:number)).to eq(42)
@@ -102,7 +102,9 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
structured_output << chunks[6]
structured_output << chunks[7]
- expect(structured_output.read_buffered_property(:list)).to eq(["Hello! I am a chunk"])
+ # A trailing comma opens a slot for the next element, which reads as a
+ # nil placeholder until its value arrives.
+ expect(structured_output.read_buffered_property(:list)).to eq(["Hello! I am a chunk", nil])
structured_output << chunks[8]
expect(structured_output.read_buffered_property(:list)).to eq(
@@ -128,14 +130,16 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
described_class.new({ ratings: { type: "array", items: { type: "object" } } })
end
- it "falls back to full parsing without raising" do
+ it "streams partial objects and completes them at the end" do
chunks = [
+"{\"ratings\":[{\"candidate\":\"alpha\",\"rating\":9",
+"},{\"candidate\":\"bravo\",\"rating\":6}]}",
]
- expect { structured_output << chunks[0] }.not_to raise_error
- expect(structured_output.read_buffered_property(:ratings)).to eq(nil)
+ structured_output << chunks[0]
+ expect(structured_output.read_buffered_property(:ratings)).to eq(
+ [{ "candidate" => "alpha", "rating" => 9 }],
+ )
structured_output << chunks[1]
structured_output.finish
@@ -147,6 +151,37 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
end
end
+ describe "recovering from streams with unescaped control characters" do
+ # regression specs for https://meta.discourse.org/t/407251
+ it "streams the full content instead of truncating at escaped quotes" do
+ content = "Erste Zeile.\n\nZweite Zeile mit einem \"Zitat\" und 😊 Emoji."
+ raw = { message: content }.to_json.gsub("\\n", "\n")
+
+ streamed = +""
+ raw
+ .each_char
+ .each_slice(13) do |chunk|
+ structured_output << chunk.join
+ streamed << (structured_output.read_buffered_property(:message) || "")
+ end
+ structured_output.finish
+ streamed << (structured_output.read_buffered_property(:message) || "")
+
+ expect(structured_output.broken?).to eq(false)
+ expect(streamed).to eq(content)
+ end
+
+ it "recovers the full content at finish when the response is also fenced" do
+ content = "A \"quoted\" word\nand a second line"
+ raw = "```json\n#{{ message: content }.to_json.gsub("\\n", "\n")}\n```"
+
+ structured_output << raw
+ structured_output.finish
+
+ expect(structured_output.read_buffered_property(:message)).to eq(content)
+ end
+ end
+
describe "dealing with non-JSON responses" do
it "treat it as plain text once we determined it's invalid JSON" do
chunks = [+"I'm not", +"a", +"JSON :)"]
diff --git a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb
index 002d85865bc..db6d2f4bdad 100644
--- a/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb
+++ b/plugins/discourse-ai/spec/lib/utils/best_effort_json_parser_spec.rb
@@ -161,6 +161,30 @@ RSpec.describe DiscourseAi::Utils::BestEffortJsonParser do
end
end
+ context "with responses where JSON streaming broke" do
+ # regression specs for https://meta.discourse.org/t/407251
+ it "extracts the complete value when string values contain real newlines" do
+ content = "First line.\n\nA \"quoted\" word 😊 and everything after it."
+ input = { output: content }.to_json.gsub("\\n", "\n")
+
+ result = described_class.extract_key(input, "string", :output)
+
+ expect(result).to eq(content)
+ end
+
+ it "completes truncated JSON instead of scraping garbage" do
+ input = "{\"output\": \"A translation that got cut off midway thro"
+
+ result = described_class.extract_key(input, "string", :output)
+
+ expect(result).to eq("A translation that got cut off midway thro")
+ end
+
+ it "casts values to numbers for integer schemas" do
+ expect(described_class.extract_key('{"output": 42}', "integer", :output)).to eq(42)
+ end
+ end
+
context "when very broken JSON is entered" do
it "returns empty string when no valid JSON can be extracted for string type" do
input = "This is just plain text with no JSON"