The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Accept JSON replies from models that ignore the forced tool call (#82974)
Andrei Simionescu35e8d0675fce
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 10ea0ccf861..71991f1d61d 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -10,6 +10,7 @@TODO:- figure out what's lacking compared to ai-service"(:require+ [clojure.string :as str][metabase.analytics-interface.core :as analytics][metabase.analytics.core :as analytics.core][metabase.api.common :as api]@@ -21,7 +22,9 @@[metabase.metabot.usage :as usage][metabase.util :as u][metabase.util.i18n :refer [tru]]+ [metabase.util.json :as json][metabase.util.log :as log]+ [metabase.util.malli.registry :as mr][metabase.util.o11y :refer [with-span]]))(set! *warn-on-reflection* true)@@ -490,6 +493,37 @@#(reduce rf* init (make-source))(fn [_e] (not @emitted?))))))))))))+(defn- json-schema->malli+ "Malli equivalent of `json-schema`, for the JSON Schema subset [[core/LLMRequestOpts]] accepts as `:schema`."+ [{:keys [type properties required additionalProperties items minimum maximum]}]+ (let [schema (case type+ "object" (into [:map {:closed (false? additionalProperties)}]+ (for [[k v] properties]+ [(keyword k) {:optional (not-any? #{(name k)} required)} (json-schema->malli v)]))+ "array" [:sequential (if items (json-schema->malli items) :any)]+ "string" :string+ "integer" :int+ "number" number?+ "boolean" :boolean+ :any)]+ (if (or minimum maximum)+ (cond-> [:and schema]+ minimum (conj [:>= minimum])+ maximum (conj [:<= maximum]))+ schema)))++(defn- structured-output-in-text+ "JSON matching `json-schema` in the text reply of a model that didn't call the structured-output tool.+ Tries the whole reply, then each fenced code block in it from the last one back. Nil when none of them matches."+ [parts json-schema]+ (let [text (str/join (keep #(when (= :text (:type %)) (:text %)) parts))+ schema (json-schema->malli json-schema)]+ (some (fn [candidate]+ (let [value (try (json/decode-document+kw candidate) (catch Exception _ nil))]+ (when (mr/validate schema value)+ value)))+ (cons text (reverse (map second (re-seq #"(?is)```(?:json)?\s*(.*?)```" text)))))))+(defn call-llm-structured-with-trace"Like [[call-llm-structured]], but returns `{:result <map> :parts [<part>...]}`so callers can inspect everything the model emitted — any non-tool text, the@@ -578,8 +612,12 @@{:parts parts :error error :error-code "llm-stream-error"})):else- (throw (ex-info "LLM returned no tool call in structured response"- {:parts parts})))))))))+ (if-let [output (structured-output-in-text parts json-schema)]+ (do (log/info "LLM answered in text instead of calling the structured-output tool"+ {:provider provider :model model :tag (:tag opts)})+ {:result output :parts parts})+ (throw (ex-info "LLM returned no tool call in structured response"+ {:parts parts}))))))))))(defn call-llm-structured"Make an LLM call that returns structured JSON output.@@ -600,8 +638,9 @@tracking fields and [[call-llm-structured-with-trace]] for`:required-permission`.- Returns the parsed JSON map from the forced tool call. For access to the- full streamed trace (non-tool text), see+ Returns the parsed JSON map from the forced tool call. When the model answers+ in text instead, the JSON in that text is returned if it matches `json-schema`.+ For access to the full streamed trace (non-tool text), see[[call-llm-structured-with-trace]]."[provider-and-model messages json-schema temperature max-tokens opts](:result (call-llm-structured-with-tracediff --git a/src/metabase/util/json.clj b/src/metabase/util/json.cljindex 4b2496b9693..1eb8fcbd9b9 100644--- a/src/metabase/util/json.clj+++ b/src/metabase/util/json.clj@@ -10,7 +10,7 @@[clojure.string :as str])(:import(com.fasterxml.jackson.core JsonGenerator)- (java.io InputStream Reader)))+ (java.io InputStream Reader StringReader)))(set! *warn-on-reflection* true)@@ -98,6 +98,15 @@[source](decode source true))+(defn decode-document+kw+ "Decode the single JSON value that makes up string `s`, keywordizing map keys.+ Throws if anything other than whitespace follows that value."+ [^String s]+ (let [[value & more] (cheshire/parsed-seq (io/reader (StringReader. s)) true)]+ (when more+ (throw (ex-info "Expected one JSON value, found more" {})))+ value))+(defn- parse-charset"Parse charset from content-type header, e.g. 'application/json; charset=utf-8' -> 'utf-8'.Returns nil if no charset is specified."diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 40e44cf926b..6516e674d44 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -1397,6 +1397,58 @@"the provider error message is surfaced, not hidden behind 'no tool call'")(is (= "llm-stream-error" (:error-code (ex-data e))))))))))+(deftest call-llm-structured-text-reply-test+ (llm.tu/with-default-connections+ (let [schema {:type "object"+ :properties {"title" {:type "string"}+ "tags" {:type "array" :items {:type "string"}}+ "score" {:type "number" :minimum 0 :maximum 1}}+ :required ["title"]+ :additionalProperties false}+ answer {:title "Q2 revenue" :tags ["revenue"] :score 0.5}+ reply (fn [text] {:type :text :id "t1" :text text})+ fenced (fn [json-text] (str "Here you go:\n```json\n" json-text "\n```"))+ call! (fn [& parts]+ (mt/with-dynamic-fn-redefs [openrouter/openrouter+ (constantly (test-util/mock-llm-response+ (cons {:type :start :id "m1"} parts)))]+ (self/call-llm-structured "openrouter/test-model" [{:role "user" :content "test"}]+ schema 0.3 1024 {:tag "metabot_agent"})))]+ (testing "a model that answers in text instead of calling the tool has its JSON used"+ (are [text] (= answer (call! (reply text)))+ (json/encode answer)+ (fenced (json/encode answer))))+ (testing "the last fenced code block that matches the schema is used"+ (are [text] (= answer (call! (reply text)))+ (str (fenced (json/encode {:title "Draft"})) "\n" (fenced (json/encode answer)))+ (str (fenced (json/encode answer)) "\n" (fenced (json/encode {:title 42})))))+ (testing "a line break the model left unescaped inside a string doesn't stop the JSON from being used"+ (are [text] (= {:title "Q2 revenue\nby region"} (call! (reply text)))+ "{\"title\": \"Q2 revenue\nby region\"}"+ (fenced "{\"title\": \"Q2 revenue\nby region\"}")))+ (testing "a text reply without JSON matching the schema still fails"+ (are [text] (thrown-with-msg? clojure.lang.ExceptionInfo #"no tool call" (call! (reply text)))+ "Q2 revenue"+ "{\"title\": \"Q2 revenue\""+ (json/encode {:tags ["revenue"]})+ (json/encode {:title "Q2 revenue" :note "x"})+ (json/encode {:title 42})+ (json/encode {:title "Q2 revenue" :tags [1]})+ (json/encode {:title "Q2 revenue" :score 2})))+ (testing "JSON followed by anything but whitespace is not used, in the whole reply or in a fence"+ (are [text] (thrown-with-msg? clojure.lang.ExceptionInfo #"no tool call" (call! (reply text)))+ (str (json/encode answer) " Hope that helps!")+ (str (json/encode answer) (json/encode {:title "Other"}))+ (fenced (str (json/encode answer) " Hope that helps!"))+ (fenced (str (json/encode answer) (json/encode {:title "Other"})))))+ (testing "a tool call is used as is, whatever text comes with it"+ (is (= {:title "From the tool"}+ (call! (reply (json/encode answer))+ {:type :tool-input+ :id "c1"+ :function "structured_output"+ :arguments {:title "From the tool"}})))))))+(deftest call-llm-does-not-replay-after-partial-emission-test(llm.tu/with-default-connections(testing "a retryable failure AFTER parts were emitted does not replay the stream (no duplicate output / re-run tools)"diff --git a/test/metabase/util/json_test.clj b/test/metabase/util/json_test.cljindex cd8aa89a34c..80b1a624a24 100644--- a/test/metabase/util/json_test.clj+++ b/test/metabase/util/json_test.clj@@ -1,6 +1,6 @@(ns metabase.util.json-test(:require- [clojure.test :refer [deftest is]]+ [clojure.test :refer [are deftest is]][metabase.util.json :as json]))(set! *warn-on-reflection* true)@@ -9,3 +9,14 @@(let [result (json/decode "[1]")](is (vector? result))(is (= [1] result))))++(deftest ^:parallel decode-document+kw-test+ (are [s expected] (= expected (json/decode-document+kw s))+ "{\"a\": [1]}" {:a [1]}+ " {\"a\": 1} \n" {:a 1}+ "{\"a\": \"x\ny\"}" {:a "x\ny"}+ "" nil)+ (are [s] (thrown? Exception (json/decode-document+kw s))+ "{\"a\": 1} and more"+ "{\"a\": 1}{\"b\": 2}"+ "[1] [2]"))