Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Accept JSON replies from models that ignore the forced tool call (#82974)

Andrei Simionescu35e8d0675fce
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 10ea0ccf861..71991f1d61d 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -10,6 +10,7 @@
TODO:
- figure out what's lacking compared to ai-service"
(:require
+ [clojure.string :as str]
[metabase.analytics-interface.core :as analytics]
[metabase.analytics.core :as analytics.core]
[metabase.api.common :as api]
@@ -21,7 +22,9 @@
[metabase.metabot.usage :as usage]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
+ [metabase.util.json :as json]
[metabase.util.log :as log]
+ [metabase.util.malli.registry :as mr]
[metabase.util.o11y :refer [with-span]]))
(set! *warn-on-reflection* true)
@@ -490,6 +493,37 @@
#(reduce rf* init (make-source))
(fn [_e] (not @emitted?))))))))))))
+(defn- json-schema->malli
+ "Malli equivalent of `json-schema`, for the JSON Schema subset [[core/LLMRequestOpts]] accepts as `:schema`."
+ [{:keys [type properties required additionalProperties items minimum maximum]}]
+ (let [schema (case type
+ "object" (into [:map {:closed (false? additionalProperties)}]
+ (for [[k v] properties]
+ [(keyword k) {:optional (not-any? #{(name k)} required)} (json-schema->malli v)]))
+ "array" [:sequential (if items (json-schema->malli items) :any)]
+ "string" :string
+ "integer" :int
+ "number" number?
+ "boolean" :boolean
+ :any)]
+ (if (or minimum maximum)
+ (cond-> [:and schema]
+ minimum (conj [:>= minimum])
+ maximum (conj [:<= maximum]))
+ schema)))
+
+(defn- structured-output-in-text
+ "JSON matching `json-schema` in the text reply of a model that didn't call the structured-output tool.
+ Tries the whole reply, then each fenced code block in it from the last one back. Nil when none of them matches."
+ [parts json-schema]
+ (let [text (str/join (keep #(when (= :text (:type %)) (:text %)) parts))
+ schema (json-schema->malli json-schema)]
+ (some (fn [candidate]
+ (let [value (try (json/decode-document+kw candidate) (catch Exception _ nil))]
+ (when (mr/validate schema value)
+ value)))
+ (cons text (reverse (map second (re-seq #"(?is)```(?:json)?\s*(.*?)```" text)))))))
+
(defn call-llm-structured-with-trace
"Like [[call-llm-structured]], but returns `{:result <map> :parts [<part>...]}`
so callers can inspect everything the model emitted — any non-tool text, the
@@ -578,8 +612,12 @@
{:parts parts :error error :error-code "llm-stream-error"}))
:else
- (throw (ex-info "LLM returned no tool call in structured response"
- {:parts parts})))))))))
+ (if-let [output (structured-output-in-text parts json-schema)]
+ (do (log/info "LLM answered in text instead of calling the structured-output tool"
+ {:provider provider :model model :tag (:tag opts)})
+ {:result output :parts parts})
+ (throw (ex-info "LLM returned no tool call in structured response"
+ {:parts parts}))))))))))
(defn call-llm-structured
"Make an LLM call that returns structured JSON output.
@@ -600,8 +638,9 @@
tracking fields and [[call-llm-structured-with-trace]] for
`:required-permission`.
- Returns the parsed JSON map from the forced tool call. For access to the
- full streamed trace (non-tool text), see
+ Returns the parsed JSON map from the forced tool call. When the model answers
+ in text instead, the JSON in that text is returned if it matches `json-schema`.
+ For access to the full streamed trace (non-tool text), see
[[call-llm-structured-with-trace]]."
[provider-and-model messages json-schema temperature max-tokens opts]
(:result (call-llm-structured-with-trace
diff --git a/src/metabase/util/json.clj b/src/metabase/util/json.clj
index 4b2496b9693..1eb8fcbd9b9 100644
--- a/src/metabase/util/json.clj
+++ b/src/metabase/util/json.clj
@@ -10,7 +10,7 @@
[clojure.string :as str])
(:import
(com.fasterxml.jackson.core JsonGenerator)
- (java.io InputStream Reader)))
+ (java.io InputStream Reader StringReader)))
(set! *warn-on-reflection* true)
@@ -98,6 +98,15 @@
[source]
(decode source true))
+(defn decode-document+kw
+ "Decode the single JSON value that makes up string `s`, keywordizing map keys.
+ Throws if anything other than whitespace follows that value."
+ [^String s]
+ (let [[value & more] (cheshire/parsed-seq (io/reader (StringReader. s)) true)]
+ (when more
+ (throw (ex-info "Expected one JSON value, found more" {})))
+ value))
+
(defn- parse-charset
"Parse charset from content-type header, e.g. 'application/json; charset=utf-8' -> 'utf-8'.
Returns nil if no charset is specified."
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 40e44cf926b..6516e674d44 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -1397,6 +1397,58 @@
"the provider error message is surfaced, not hidden behind 'no tool call'")
(is (= "llm-stream-error" (:error-code (ex-data e))))))))))
+(deftest call-llm-structured-text-reply-test
+ (llm.tu/with-default-connections
+ (let [schema {:type "object"
+ :properties {"title" {:type "string"}
+ "tags" {:type "array" :items {:type "string"}}
+ "score" {:type "number" :minimum 0 :maximum 1}}
+ :required ["title"]
+ :additionalProperties false}
+ answer {:title "Q2 revenue" :tags ["revenue"] :score 0.5}
+ reply (fn [text] {:type :text :id "t1" :text text})
+ fenced (fn [json-text] (str "Here you go:\n```json\n" json-text "\n```"))
+ call! (fn [& parts]
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter
+ (constantly (test-util/mock-llm-response
+ (cons {:type :start :id "m1"} parts)))]
+ (self/call-llm-structured "openrouter/test-model" [{:role "user" :content "test"}]
+ schema 0.3 1024 {:tag "metabot_agent"})))]
+ (testing "a model that answers in text instead of calling the tool has its JSON used"
+ (are [text] (= answer (call! (reply text)))
+ (json/encode answer)
+ (fenced (json/encode answer))))
+ (testing "the last fenced code block that matches the schema is used"
+ (are [text] (= answer (call! (reply text)))
+ (str (fenced (json/encode {:title "Draft"})) "\n" (fenced (json/encode answer)))
+ (str (fenced (json/encode answer)) "\n" (fenced (json/encode {:title 42})))))
+ (testing "a line break the model left unescaped inside a string doesn't stop the JSON from being used"
+ (are [text] (= {:title "Q2 revenue\nby region"} (call! (reply text)))
+ "{\"title\": \"Q2 revenue\nby region\"}"
+ (fenced "{\"title\": \"Q2 revenue\nby region\"}")))
+ (testing "a text reply without JSON matching the schema still fails"
+ (are [text] (thrown-with-msg? clojure.lang.ExceptionInfo #"no tool call" (call! (reply text)))
+ "Q2 revenue"
+ "{\"title\": \"Q2 revenue\""
+ (json/encode {:tags ["revenue"]})
+ (json/encode {:title "Q2 revenue" :note "x"})
+ (json/encode {:title 42})
+ (json/encode {:title "Q2 revenue" :tags [1]})
+ (json/encode {:title "Q2 revenue" :score 2})))
+ (testing "JSON followed by anything but whitespace is not used, in the whole reply or in a fence"
+ (are [text] (thrown-with-msg? clojure.lang.ExceptionInfo #"no tool call" (call! (reply text)))
+ (str (json/encode answer) " Hope that helps!")
+ (str (json/encode answer) (json/encode {:title "Other"}))
+ (fenced (str (json/encode answer) " Hope that helps!"))
+ (fenced (str (json/encode answer) (json/encode {:title "Other"})))))
+ (testing "a tool call is used as is, whatever text comes with it"
+ (is (= {:title "From the tool"}
+ (call! (reply (json/encode answer))
+ {:type :tool-input
+ :id "c1"
+ :function "structured_output"
+ :arguments {:title "From the tool"}})))))))
+
(deftest call-llm-does-not-replay-after-partial-emission-test
(llm.tu/with-default-connections
(testing "a retryable failure AFTER parts were emitted does not replay the stream (no duplicate output / re-run tools)"
diff --git a/test/metabase/util/json_test.clj b/test/metabase/util/json_test.clj
index cd8aa89a34c..80b1a624a24 100644
--- a/test/metabase/util/json_test.clj
+++ b/test/metabase/util/json_test.clj
@@ -1,6 +1,6 @@
(ns metabase.util.json-test
(:require
- [clojure.test :refer [deftest is]]
+ [clojure.test :refer [are deftest is]]
[metabase.util.json :as json]))
(set! *warn-on-reflection* true)
@@ -9,3 +9,14 @@
(let [result (json/decode "[1]")]
(is (vector? result))
(is (= [1] result))))
+
+(deftest ^:parallel decode-document+kw-test
+ (are [s expected] (= expected (json/decode-document+kw s))
+ "{\"a\": [1]}" {:a [1]}
+ " {\"a\": 1} \n" {:a 1}
+ "{\"a\": \"x\ny\"}" {:a "x\ny"}
+ "" nil)
+ (are [s] (thrown? Exception (json/decode-document+kw s))
+ "{\"a\": 1} and more"
+ "{\"a\": 1}{\"b\": 2}"
+ "[1] [2]"))