The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Reject non-object Metabot tool arguments, and give them one key shape (#82808)
Paolo Rascunàff1fbe68ca2b
diff --git a/src/metabase/metabot/agent/messages.clj b/src/metabase/metabot/agent/messages.cljindex 697cc3a56c0..eb0bce44b30 100644--- a/src/metabase/metabot/agent/messages.clj+++ b/src/metabase/metabot/agent/messages.clj@@ -9,6 +9,7 @@[metabase.metabot.agent.memory :as memory][metabase.metabot.agent.prompts :as prompts][metabase.metabot.agent.user-context :as user-context]+ [metabase.metabot.self.core :as self.core][metabase.metabot.skills :as skills][metabase.util.json :as json][metabase.util.log :as log]))@@ -25,17 +26,26 @@role(keyword role)))+(defn- decode-arguments-json+ "Decode the tool call arguments JSON string `s`, or return `s` unchanged when it is not valid JSON."+ [s]+ (try+ (json/decode s)+ (catch Exception e+ (log/warnf "Failed to decode tool call arguments: %s" (ex-message e))+ s)))+(defn- decode-tool-arguments- "Decode tool call arguments from JSON when provided as a string.- Falls back to the raw string if decoding fails."+ "Decode tool call arguments, with keyword argument names at the top level only.++ The JSON string `{\"query\": {\"stages\": []}}` becomes `{:query {\"stages\" []}}`, the shape the stream parser gives.+ A map gets the same shape: `{\"query\" {:stages []}}` becomes `{:query {\"stages\" []}}`. A string that is not valid+ JSON comes back unchanged."[arguments]- (if (string? arguments)- (try- (json/decode arguments)- (catch Exception e- (log/warnf "Failed to decode tool call arguments: %s" (ex-message e))- arguments))- arguments))+ (let [decoded (cond-> arguments+ (string? arguments) decode-arguments-json)]+ (cond-> decoded+ (map? decoded) (-> (update-keys keyword) self.core/stringify-nested-keys))))(defn- tool-result-content"Normalise tool result content to a string suitable for :tool-output."diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 8c79c6807c6..d29f92cc2a4 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -587,14 +587,15 @@(defn- structured-output-in-text"JSON matching `json-schema` in the text reply of a model that didn't call the structured-output tool.- Tries the whole reply, then each fenced code block in it from the last one back. Nil when none of them matches."+ Tries the whole reply, then each fenced code block in it from the last one back. Nil when none of them matches.+ Keys below the top level are strings, as in a tool call's arguments ([[core/stringify-nested-keys]])."[parts json-schema](let [text (str/join (keep #(when (= :text (:type %)) (:text %)) parts))schema (json-schema->malli json-schema)](some (fn [candidate](let [value (try (json/decode-document+kw candidate) (catch Exception _ nil))](when (mr/validate schema value)- value)))+ (core/stringify-nested-keys value))))(cons text (reverse (map second (re-seq #"(?is)```(?:json)?\s*(.*?)```" text)))))))(defn- incomplete-structured-output-error@@ -677,23 +678,25 @@(with-retriestracking-opts(fn []- (let [parts (into []- (comp (core/aisdk-xf)- (report-aisdk-errors-xf tracking-opts)- (report-token-usage-xf tracking-opts))- (stream-fn streaming-opts))- result (some (fn [{:keys [type arguments]}]- (when (= type :tool-input)- arguments))- parts)- error (some (fn [{:keys [type error]}]- (when (= type :error)- error))- parts)+ (let [parts (into []+ (comp (core/aisdk-xf)+ (report-aisdk-errors-xf tracking-opts)+ (report-token-usage-xf tracking-opts))+ (stream-fn streaming-opts))+ result (some (fn [{:keys [type arguments]}]+ (when (= type :tool-input)+ arguments))+ parts)+ error (some (fn [{:keys [type error]}]+ (when (= type :error)+ error))+ parts)+ unparsed (core/unparsed-arguments result);; Only `length` or `content-filter` can turn up here: `"tool-calls"` needs a `:finish`;; part, which the agent loop emits and this single-shot path never runs.incomplete-reason (core/parts->incomplete-finish-reason parts)- malformed? (and (map? result) (contains? result :_raw_arguments))]+ ;; Only malformed JSON can be a cut-off tool call. A non-object reply parsed in full.+ malformed? (contains? unparsed :raw-arguments)](cond;; A tool call cut off mid-JSON is not a model emitting bad JSON — the turn ran out of;; room — so report why it stopped. Only `length` reroutes this branch: a content filter@@ -701,14 +704,15 @@(and malformed? (= incomplete-reason "length"))(throw (incomplete-structured-output-error parts incomplete-reason))- ;; The tool call's JSON failed to parse; `parse-tool-arguments` returned the- ;; `{:_raw_arguments ...}` sentinel. Reject it as invalid rather than handing a- ;; bogus map back to the caller as if it were a valid structured result.- malformed?- (throw (ex-info "LLM returned malformed JSON in its structured tool call"- {:parts parts- :error-code "structured-output-invalid"- :raw-arguments (:_raw_arguments result)}))+ ;; The tool call's arguments were malformed JSON or valid JSON that isn't an object. Reject them+ ;; rather than handing the parse sentinel back to the caller as if it were a structured result.+ unparsed+ (throw (ex-info (if malformed?+ "LLM returned malformed JSON in its structured tool call"+ "LLM returned a JSON value that is not an object in its structured tool call")+ (merge {:parts parts+ :error-code "structured-output-invalid"}+ unparsed)))result{:result result :parts parts}diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 49a8383a56f..6d0d1b5a40f 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -116,8 +116,10 @@[:type {:optional true} [:maybe :string]]]]]]])(mr/def ::decoded-json- "A value decoded from JSON.- Object keys are strings from `json/decode` (replayed history) or keywords from `json/decode+kw` (the stream)."+ "A value decoded from JSON with `json/decode`, with string object keys at every depth.++ `{\"stages\" [{\"limit\" 1}]}` passes and `{:stages [{:limit 1}]}` does not. [[stringify-nested-keys]] gives this+ shape below the argument names."[:or:string:keyword@@ -125,13 +127,15 @@:boolean:nil[:sequential [:ref ::decoded-json]]- [:map-of [:or :string :keyword] [:ref ::decoded-json]]])+ [:map-of :string [:ref ::decoded-json]]])(def ^:private ToolCallArguments- "A tool call's arguments as the LLM wrote them against the tool's own schema, keyed by that tool's argument names:- string keys off the wire, keyword keys when built in Clojure."+ "A tool call's arguments as the LLM wrote them against the tool's own schema, keyed by its argument names as keywords.++ `{:query {\"stages\" []}}` passes. `{\"query\" {\"stages\" []}}` and `{:query {:stages []}}` do not: the argument+ names are keywords, and the values are `::decoded-json`, with string keys."[:map-of {::mr/deliberately-open true, :description "tool call arguments"}- [:or :string :keyword] ::decoded-json])+ :keyword ::decoded-json])(def ^:private AISDKPart"One element of the `:input` sequence passed to a provider adapter: an AISDK part keyed by@@ -371,22 +375,59 @@(get stop-reasons raw "other")))(defn- parse-tool-arguments- "Parse concatenated tool input deltas as JSON.- Falls back to returning the raw string wrapped in a map when parsing fails,- e.g. when the LLM produces malformed JSON in tool call arguments."+ "Parse concatenated tool input deltas as a JSON object with keyword argument names.++ `{\"query\":{\"stages\":[]}}` becomes `{:query {\"stages\" []}}`: only the top-level names are keywordized. Blank+ input (a call to a tool with no arguments) parses to nil. Anything else comes back as a sentinel map that+ [[unparsed-arguments]] describes and [[reject-unparsed-arguments!]] turns into an error for the model:+ `{::raw-arguments raw}` for malformed JSON, and `{::non-object-arguments decoded}` for valid JSON that isn't an+ object, e.g. `[\"orders\"]`. The keys are namespaced so they don't clash with a tool's argument names."[chunks](let [raw (->> (map :inputTextDelta chunks)(str/join ""))](try- (json/decode+kw raw)+ ;; Keywordize only the top-level argument names. Nested objects are data the model wrote and stay string-keyed+ ;; JSON. The executor and title-fn keywordize the full arguments themselves.+ (let [arguments (json/decode raw)]+ (cond+ (map? arguments) (update-keys arguments keyword)+ (nil? arguments) nil+ :else {::non-object-arguments arguments}))(catch Exception e(log/warn "Failed to parse tool arguments as JSON, passing raw string"{:tool (:toolName (first chunks)):error (.getMessage e):raw-len (count raw)})- ;; Return a map with a sentinel key so the tool sees an error via schema validation+ ;; Return a sentinel map so the model gets a repair message from [[reject-unparsed-arguments!]];; rather than a cryptic JSON parse stacktrace.- {:_raw_arguments raw}))))+ {::raw-arguments raw}))))++(defn unparsed-arguments+ "Diagnostic data for tool `arguments` that are a [[parse-tool-arguments]] sentinel, or nil for parsed arguments.++ `{::raw-arguments \"{oops\"}` gives `{:raw-arguments \"{oops\"}`, `{::non-object-arguments [\"orders\"]}` gives+ `{:non-object-arguments [\"orders\"]}`, and `{:table \"orders\"}` or nil gives nil."+ [arguments]+ (when (map? arguments)+ (cond+ (contains? arguments ::raw-arguments) {:raw-arguments (::raw-arguments arguments)}+ (contains? arguments ::non-object-arguments) {:non-object-arguments (::non-object-arguments arguments)})))++(defn stringify-nested-keys+ "Return `arguments` with every map key below the top level as a string.++ `{:query {:stages [{:limit 1}]}}` becomes `{:query {\"stages\" [{\"limit\" 1}]}}`. The top-level keys do not change,+ so the argument names and the [[parse-tool-arguments]] sentinel keys stay keywords. A namespaced keyword key+ becomes `\"ns/name\"`, the key `json/encode` writes for it."+ [arguments]+ (letfn [(below [x]+ (cond+ (map? x) (u/for-map [[k v] x]+ [(cond-> k (keyword? k) u/qualified-name) (below v)])+ (sequential? x) (mapv below x)+ :else x))]+ (cond-> arguments+ (map? arguments) (update-vals below))))(defn- try-decode-json-string"If `v` is a string that looks like a JSON object or array, decode it.@@ -972,23 +1013,36 @@(str "; received " (json-type-name (get arguments field))))"."))))+(defn- non-object-arguments-message+ "The error for tool `arguments` that are not a JSON object, e.g. `[\"orders\"]`."+ [arguments]+ (str "Invalid tool arguments: expected an object of named arguments; received " (json-type-name arguments) "."))+(defn- invalid-arguments-message"A repair-oriented message describing how `arguments` violate `schema`, or nil when they match."[schema arguments](when-let [error (mr/explain schema arguments)](let [humanized (me/humanize error)]- (str "Invalid tool arguments: "- (if (map? humanized)+ (if (map? humanized)+ (str "Invalid tool arguments: "(str/join " " (for [[field messages] (sort-by (comp name key) humanized)]- (argument-error-text arguments field messages)))- (str "expected an object of named arguments; received "- (json-type-name arguments) "."))))))+ (argument-error-text arguments field messages))))+ ;; The error isn't per-field. Today that only happens when a tool's own `:decode` returns a non-map: a+ ;; JSON array the model sent is rejected earlier by [[reject-unparsed-arguments!]].+ (non-object-arguments-message arguments)))))++(defn- reject-unparsed-arguments!+ "Throw an agent error when `arguments` is a [[parse-tool-arguments]] sentinel rather than a JSON object."+ [arguments]+ (when-let [unparsed (unparsed-arguments arguments)]+ (if (contains? unparsed :raw-arguments)+ (throw (ex-info "Invalid tool arguments: the arguments were not valid JSON. Send the call again as a JSON object."+ {:agent-error? true}))+ (throw (ex-info (non-object-arguments-message (:non-object-arguments unparsed))+ {:agent-error? true})))))(defn- validate-tool-arguments![tool arguments]- (when (and (map? arguments) (contains? arguments :_raw_arguments))- (throw (ex-info "Invalid tool arguments: the arguments were not valid JSON. Send the call again as a JSON object."- {:agent-error? true})))(when-let [schema (tool-args-schema tool)](when-let [message (invalid-arguments-message schema arguments)](throw (ex-info message {:agent-error? true})))))@@ -1039,6 +1093,9 @@(str/join ", " (sort (keys tools))) "."){:agent-error? true})))(let [{:keys [arguments]} (into {} (aisdk-xf) chunks)+ ;; Before coercion: `coerce-stringified-json` would decode a double-encoded+ ;; `{::non-object-arguments "{...}"}` into an object and garble the message.+ _ (reject-unparsed-arguments! arguments)arguments (walk/keywordize-keys (or (coerce-stringified-json arguments) {}))arguments (coerce-stringified-scalars tool arguments)decode (tool-decode-fn tool)diff --git a/test/metabase/metabot/agent/messages_test.clj b/test/metabase/metabot/agent/messages_test.cljindex f0fb328be9d..ed82ecfc280 100644--- a/test/metabase/metabot/agent/messages_test.clj+++ b/test/metabase/metabot/agent/messages_test.clj@@ -6,6 +6,8 @@[metabase.metabot.agent.memory :as memory][metabase.metabot.agent.messages :as messages][metabase.metabot.agent.user-context :as user-context]+ [metabase.metabot.persistence :as metabot.persistence]+ [metabase.metabot.self.openai.chat-completions :as chat-completions][metabase.test :as mt]));;; ──────────────────────────────────────────────────────────────────@@ -26,7 +28,7 @@(messages/input-message->parts {:role :assistant :content "Hi there"}))))(testing "assistant with tool_calls (OpenAI style)"(is (=? [{:type :text :text "Searching..."}- {:type :tool-input :id "t1" :function "search" :arguments {"q" "test"}}]+ {:type :tool-input :id "t1" :function "search" :arguments {:q "test"}}](messages/input-message->parts{:role :assistant:content "Searching..."@@ -52,7 +54,14 @@(is (=? [{:type :tool-input :id "t1" :function "search" :arguments "{bad-json"}](messages/input-message->parts{:role :assistant- :tool_calls [{:id "t1" :name "search" :arguments "{bad-json"}]})))))+ :tool_calls [{:id "t1" :name "search" :arguments "{bad-json"}]}))))+ (testing "map tool_call arguments get keyword argument names and string keys below them"+ (is (= {:query {"stages" [{"limit" 1}]}}+ (-> (messages/input-message->parts+ {:role :assistant+ :tool_calls [{:id "t1" :name "search" :arguments {"query" {:stages [{:limit 1}]}}}]})+ first+ :arguments)))))(deftest ^:parallel input-message-tool-test(testing "tool result message"@@ -73,6 +82,21 @@:content [{:type "tool_result" :tool_use_id "t1" :content "Result 1"}{:type "tool_result" :tool_use_id "t2" :content "Result 2"}]})))))+(deftest ^:parallel persisted-tool-call-replays-into-adapter-test+ (testing "a stored tool call replays with keyword argument names and string nested keys, and the adapter accepts it"+ (let [stored {:type "tool-construct_notebook_query"+ :toolCallId "call-1"+ :state "output-available"+ :input {:query {:stages [{:source-table 1}]} :limit 5}+ :output {:output "ok"}}+ parts (into [] (mapcat messages/input-message->parts)+ (metabot.persistence/tool-part->llm-messages stored))]+ (is (= {:query {"stages" [{"source-table" 1}]} :limit 5}+ (:arguments (first parts))))+ (is (=? {:messages [{:role "assistant" :tool_calls [{:id "call-1" :function {:arguments string?}}]}+ {:role "tool" :tool_call_id "call-1"}]}+ (chat-completions/request-body {:model "some/model" :input parts}))))))+;;; ──────────────────────────────────────────────────────────────────;;; build-message-history;;; ──────────────────────────────────────────────────────────────────diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.cljindex 03a33f254aa..565e3ec1aab 100644--- a/test/metabase/metabot/self/claude_test.clj+++ b/test/metabase/metabot/self/claude_test.clj@@ -340,6 +340,27 @@(testing "no speed through the AI proxy"(is (nil? (speed {:model "claude-opus-5" :fast? true :ai-proxy? true}))))))+(deftest ^:parallel claude-request-body-nested-object-arguments-test+ (testing "a replayed tool call whose arguments hold nested string-keyed objects is accepted and sent as-is"+ ;; Replayed tool calls go through this instrumented entry point, so their arguments must validate here.+ (let [arguments {:query {"lib/type" "mbql/query"+ "stages" [{"source-table" ["db" "public" "orders"]}]}+ :visualization {"chart_type" "line"}+ :title "Orders per month"+ :description "Monthly orders in 2025"}+ body (claude/claude-request-body+ {:input [{:role :user :content "How many orders were placed each month in 2025?"}+ {:type :tool-input :id "call-1" :function "construct_notebook_query" :arguments arguments}+ {:type :tool-output+ :id "call-1"+ :function "construct_notebook_query"+ :result {:output "query built"}}]})]+ (is (=? [{:role "user"}+ {:role "assistant"+ :content [{:type "tool_use" :id "call-1" :name "construct_notebook_query" :input arguments}]}+ {:role "user" :content [{:type "tool_result" :tool_use_id "call-1"}]}]+ (:messages body))))))+(defn- close-tracking-json-body"A streamed JSON error body that flips `closed?` when closed, like the real `:as :stream`response body the adapter must not leak."diff --git a/test/metabase/metabot/self/openai/chat_completions_test.clj b/test/metabase/metabot/self/openai/chat_completions_test.cljindex fb57d6fb54d..8d438345a03 100644--- a/test/metabase/metabot/self/openai/chat_completions_test.clj+++ b/test/metabase/metabot/self/openai/chat_completions_test.clj@@ -265,7 +265,7 @@:id "call-1":function "construct_notebook_query":arguments {:query query}}- ;; the stream parser decodes the arguments with keyword keys at every depth+ ;; the stream parser keywordizes only the top-level argument namesparts (into [] (self.core/aisdk-xf) (metabot.tu/parts->aisdk-chunks [call]))](is (=? {:messages [{:role "assistant":tool_calls [{:function {:name "construct_notebook_query"@@ -280,18 +280,21 @@:id "call-1":function "f":arguments arguments}]}))]- (testing "decoded JSON replays at any depth, keyed by strings, keywords or both"+ (testing "decoded JSON replays at any depth, with keyword argument names and string keys below them"(are [arguments] (=? {:messages [{:tool_calls [{:function {:arguments string?}}]}]}(replay arguments))- ;; keyword keys, as the stream decodes them- {:a {:b {:c [{:d [1 nil true "x" :kw]}]}}}- ;; string keys, as replayed history decodes them- {"a" {"b" {"c" [{"d" [1.5 [[]] {}]}]}}}- ;; both- {:a {"b" [{:c {"d" [{:e 1}]}}]}}))+ ;; string keys below the argument names, as the stream parser and replayed history decode them+ {:a {"b" {"c" [{"d" [1 nil true "x" :kw]}]}}}+ {:a {"b" {"c" [{"d" [1.5 [[]] {}]}]}}}))+ (testing "a string argument name is rejected"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid input"+ (replay {"a" 1}))))+ (testing "a keyword key below the argument names is rejected"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid input"+ (replay {:a {"b" [{:c 1}]}}))))(testing "a value JSON cannot hold is rejected, however deep"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid input"- (replay {:a {"b" [{:c (java.time.Instant/now)}]}}))))))+ (replay {:a {"b" [{"c" (java.time.Instant/now)}]}}))))));;; ──────────────────────────────────────────────────────────────────;;; Streaming chunk conversion testsdiff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex e523209ae57..a9819657235 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -538,6 +538,50 @@{:type :tool-input-delta :toolCallId "call-1" :inputTextDelta "{}"}{:type :tool-input-available :toolCallId "call-1" :toolName "search"}])))))))+(deftest ^:parallel aisdk-xf-nested-object-arguments-validate-test+ (testing "a tool call with object-valued arguments replays as valid LLMRequestOpts input"+ ;; The agent loop replays tool calls on its next iteration, where every adapter validates them against+ ;; `LLMRequestOpts`.+ (let [chunks [{:type :tool-input-start :toolCallId "call-1" :toolName "construct_notebook_query"}+ {:type :tool-input-delta+ :toolCallId "call-1"+ :inputTextDelta "{\"query\":{\"lib/type\":\"mbql/query\","}+ {:type :tool-input-delta+ :toolCallId "call-1"+ :inputTextDelta "\"stages\":[{\"source-table\":[\"db\",\"public\",\"orders\"]}]},"}+ {:type :tool-input-delta+ :toolCallId "call-1"+ :inputTextDelta "\"visualization\":{\"chart_type\":\"line\"},"}+ {:type :tool-input-delta+ :toolCallId "call-1"+ :inputTextDelta "\"title\":\"Orders per month\",\"description\":\"Monthly orders in 2025\"}"}+ {:type :tool-input-available :toolCallId "call-1" :toolName "construct_notebook_query"}]+ parts (into [] (self.core/aisdk-xf) chunks)]+ (is (= [{:type :tool-input+ :id "call-1"+ :function "construct_notebook_query"+ :arguments {:query {"lib/type" "mbql/query"+ "stages" [{"source-table" ["db" "public" "orders"]}]}+ :visualization {"chart_type" "line"}+ :title "Orders per month"+ :description "Monthly orders in 2025"}}]+ parts))+ (is (nil? (mr/explain self.core/LLMRequestOpts {:input parts}))))))++(deftest ^:parallel aisdk-xf-non-object-arguments-validate-test+ (testing "a tool call whose arguments aren't a JSON object replays as valid LLMRequestOpts input"+ ;; The executor rejects the call, and the agent loop replays it with that error on its next iteration.+ (doseq [[raw decoded] [["[\"orders\"]" ["orders"]]+ ["false" false]+ ["[{\"a\":{\"b\":1}}]" [{"a" {"b" 1}}]]]]+ (testing raw+ (let [parts (into [] (self.core/aisdk-xf)+ [{:type :tool-input-start :toolCallId "call-1" :toolName "search"}+ {:type :tool-input-delta :toolCallId "call-1" :inputTextDelta raw}+ {:type :tool-input-available :toolCallId "call-1" :toolName "search"}])]+ (is (= {::self.core/non-object-arguments decoded} (:arguments (first parts))))+ (is (nil? (mr/explain self.core/LLMRequestOpts {:input parts}))))))))+;;; tool executor(deftest ^:parallel tool-executor-xf-test@@ -720,6 +764,43 @@(is (= "Invalid tool arguments: the arguments were not valid JSON. Send the call again as a JSON object."(-> (into [] (self.core/tool-executor-xf tools) chunks) last :error :message))))))+(deftest ^:parallel tool-non-object-arguments-test+ (let [received (atom ::not-called)+ tools {"open" {:fn (fn [args] (reset! received args) {:output "ok"})+ :doc "accepts any object"+ :schema [:=> [:cat [:map]] :any]}+ "untyped" {:fn (fn [args] (reset! received args) {:output "ok"})+ :doc "declares no argument schema"+ :schema nil}}+ run (fn run+ ([raw] (run "open" raw))+ ([tool-name raw]+ (reset! received ::not-called)+ (-> (into [] (self.core/tool-executor-xf tools)+ [{:type :tool-input-start :toolName tool-name :toolCallId "call-n"}+ {:type :tool-input-delta :toolCallId "call-n" :inputTextDelta raw}+ {:type :tool-input-available :toolName tool-name :toolCallId "call-n"}])+ last :error :message)))]+ (testing "valid JSON that isn't an object is rejected before it reaches a tool whose schema accepts any map"+ (is (= "Invalid tool arguments: expected an object of named arguments; received an array."+ (run "[\"orders\"]")))+ (is (= ::not-called @received)))+ (testing "a JSON string holding an object is reported as the string the model sent, not decoded into one"+ (is (= "Invalid tool arguments: expected an object of named arguments; received a string."+ (run "\"{\\\"names\\\": [\\\"orders\\\"]}\"")))+ (is (= ::not-called @received)))+ (testing "a JSON boolean is rejected, not run as an empty argument map"+ (is (= "Invalid tool arguments: expected an object of named arguments; received a boolean."+ (run "false")))+ (is (= ::not-called @received)))+ (testing "JSON null still runs the tool with an empty argument map"+ (is (nil? (run "null")))+ (is (= {} @received)))+ (testing "a tool with no argument schema rejects a non-object too"+ (is (= "Invalid tool arguments: expected an object of named arguments; received an array."+ (run "untyped" "[\"orders\"]")))+ (is (= ::not-called @received)))))+(deftest ^:parallel tool-without-schema-is-not-validated-test(testing "a tool with no declared argument schema is left alone"(let [tools {"anything" {:fn (fn [_args] {:output "ok"}) :doc "d" :schema nil}}@@ -1405,7 +1486,7 @@(defn- malformed-tool-input-response"A reducible LLM stream whose forced tool call streams invalid JSON, so- `parse-tool-arguments` yields the `{:_raw_arguments ...}` sentinel.+ `parse-tool-arguments` yields the raw-arguments sentinel.`trailing-parts` are appended after it, e.g. a [[usage-part]] carrying a finish reason."[& trailing-parts](let [chunks (concat [{:type :start :messageId "m1"}@@ -1463,8 +1544,28 @@0.3 1024 {:tag "metabot_agent"})(catch clojure.lang.ExceptionInfo e e))](is (instance? clojure.lang.ExceptionInfo e)- "malformed JSON must throw, not return the {:_raw_arguments ...} sentinel as a result")- (is (= "structured-output-invalid" (:error-code (ex-data e))))))))))+ "malformed JSON must throw, not return the raw-arguments sentinel as a result")+ (is (= "structured-output-invalid" (:error-code (ex-data e))))+ (is (= "{not valid json" (:raw-arguments (ex-data e))))))))))++(deftest call-llm-structured-rejects-non-object-json-test+ (llm.tu/with-default-connections+ (testing "valid JSON that isn't an object is rejected as an error, not returned as a bogus result"+ (mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)+ openrouter/openrouter (constantly+ (test-util/mock-llm-response+ [{:type :start :id "m1"}+ {:type :tool-input :id "c1" :function "json" :arguments ["orders"]}]))]+ (let [e (try+ (self/call-llm-structured "openrouter/test-model"+ [{:role "user" :content "test"}]+ {:type "object" :properties {:answer {:type "string"}}}+ 0.3 1024 {:tag "metabot_agent"})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (instance? clojure.lang.ExceptionInfo e)+ "a non-object payload must throw, not come back as the structured result")+ (is (= "structured-output-invalid" (:error-code (ex-data e))))+ (is (= ["orders"] (:non-object-arguments (ex-data e)))))))))(deftest call-llm-structured-surfaces-provider-error-test(llm.tu/with-default-connections@@ -1706,6 +1807,42 @@(is (= "structured-output-incomplete" (:error-code (ex-data e))))(is (= "length" (:finish-reason (ex-data e)))))))))+(deftest call-llm-structured-nested-keys-test+ (llm.tu/with-default-connections+ (let [schema {:type "object"+ :properties {"items" {:type "array"+ :items {:type "object"+ :properties {"name" {:type "string"}}+ :required ["name"]}}}+ :required ["items"]}+ answer {:items [{:name "x"}]}+ call! (fn [part]+ (mt/with-dynamic-fn-redefs [openrouter/openrouter+ (constantly (test-util/mock-llm-response+ [{:type :start :id "m1"} part]))]+ (self/call-llm-structured "openrouter/test-model" [{:role "user" :content "test"}]+ schema 0.3 1024 {:tag "metabot_agent"})))]+ (testing "a tool call and a text reply give the same result: keyword argument names, string keys below them"+ (are [part] (= {:items [{"name" "x"}]} (call! part))+ {:type :tool-input :id "c1" :function "structured_output" :arguments answer}+ {:type :text :id "t1" :text (json/encode answer)})))))++(deftest ^:parallel stringify-nested-keys-test+ (testing "keys below the top level become strings, and the top-level keys stay keywords"+ (is (= {:query {"stages" [{"limit" 1}]}}+ (self.core/stringify-nested-keys {:query {:stages [{:limit 1}]}}))))+ (testing "a nested namespaced keyword key becomes \"ns/name\", the key json/encode writes"+ (is (= {:a {"foo/bar" 1}}+ (self.core/stringify-nested-keys {:a {:foo/bar 1}}))))+ (testing "the parse sentinels come back unchanged"+ (are [arguments] (= arguments (self.core/stringify-nested-keys arguments))+ {::self.core/raw-arguments "{oops"}+ {::self.core/non-object-arguments [{"a" 1} "x"]}))+ (testing "a value that is not a map comes back unchanged"+ (are [arguments] (= arguments (self.core/stringify-nested-keys arguments))+ nil+ "{oops")))+(deftest call-llm-does-not-replay-after-partial-emission-test(llm.tu/with-default-connections(testing "a retryable failure AFTER parts were emitted does not replay the stream (no duplicate output / re-run tools)"