Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Reject non-object Metabot tool arguments, and give them one key shape (#82808)

Paolo Rascunàff1fbe68ca2b
diff --git a/src/metabase/metabot/agent/messages.clj b/src/metabase/metabot/agent/messages.clj
index 697cc3a56c0..eb0bce44b30 100644
--- a/src/metabase/metabot/agent/messages.clj
+++ b/src/metabase/metabot/agent/messages.clj
@@ -9,6 +9,7 @@
[metabase.metabot.agent.memory :as memory]
[metabase.metabot.agent.prompts :as prompts]
[metabase.metabot.agent.user-context :as user-context]
+ [metabase.metabot.self.core :as self.core]
[metabase.metabot.skills :as skills]
[metabase.util.json :as json]
[metabase.util.log :as log]))
@@ -25,17 +26,26 @@
role
(keyword role)))
+(defn- decode-arguments-json
+ "Decode the tool call arguments JSON string `s`, or return `s` unchanged when it is not valid JSON."
+ [s]
+ (try
+ (json/decode s)
+ (catch Exception e
+ (log/warnf "Failed to decode tool call arguments: %s" (ex-message e))
+ s)))
+
(defn- decode-tool-arguments
- "Decode tool call arguments from JSON when provided as a string.
- Falls back to the raw string if decoding fails."
+ "Decode tool call arguments, with keyword argument names at the top level only.
+
+ The JSON string `{\"query\": {\"stages\": []}}` becomes `{:query {\"stages\" []}}`, the shape the stream parser gives.
+ A map gets the same shape: `{\"query\" {:stages []}}` becomes `{:query {\"stages\" []}}`. A string that is not valid
+ JSON comes back unchanged."
[arguments]
- (if (string? arguments)
- (try
- (json/decode arguments)
- (catch Exception e
- (log/warnf "Failed to decode tool call arguments: %s" (ex-message e))
- arguments))
- arguments))
+ (let [decoded (cond-> arguments
+ (string? arguments) decode-arguments-json)]
+ (cond-> decoded
+ (map? decoded) (-> (update-keys keyword) self.core/stringify-nested-keys))))
(defn- tool-result-content
"Normalise tool result content to a string suitable for :tool-output."
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 8c79c6807c6..d29f92cc2a4 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -587,14 +587,15 @@
(defn- structured-output-in-text
"JSON matching `json-schema` in the text reply of a model that didn't call the structured-output tool.
- Tries the whole reply, then each fenced code block in it from the last one back. Nil when none of them matches."
+ Tries the whole reply, then each fenced code block in it from the last one back. Nil when none of them matches.
+ Keys below the top level are strings, as in a tool call's arguments ([[core/stringify-nested-keys]])."
[parts json-schema]
(let [text (str/join (keep #(when (= :text (:type %)) (:text %)) parts))
schema (json-schema->malli json-schema)]
(some (fn [candidate]
(let [value (try (json/decode-document+kw candidate) (catch Exception _ nil))]
(when (mr/validate schema value)
- value)))
+ (core/stringify-nested-keys value))))
(cons text (reverse (map second (re-seq #"(?is)```(?:json)?\s*(.*?)```" text)))))))
(defn- incomplete-structured-output-error
@@ -677,23 +678,25 @@
(with-retries
tracking-opts
(fn []
- (let [parts (into []
- (comp (core/aisdk-xf)
- (report-aisdk-errors-xf tracking-opts)
- (report-token-usage-xf tracking-opts))
- (stream-fn streaming-opts))
- result (some (fn [{:keys [type arguments]}]
- (when (= type :tool-input)
- arguments))
- parts)
- error (some (fn [{:keys [type error]}]
- (when (= type :error)
- error))
- parts)
+ (let [parts (into []
+ (comp (core/aisdk-xf)
+ (report-aisdk-errors-xf tracking-opts)
+ (report-token-usage-xf tracking-opts))
+ (stream-fn streaming-opts))
+ result (some (fn [{:keys [type arguments]}]
+ (when (= type :tool-input)
+ arguments))
+ parts)
+ error (some (fn [{:keys [type error]}]
+ (when (= type :error)
+ error))
+ parts)
+ unparsed (core/unparsed-arguments result)
;; Only `length` or `content-filter` can turn up here: `"tool-calls"` needs a `:finish`
;; part, which the agent loop emits and this single-shot path never runs.
incomplete-reason (core/parts->incomplete-finish-reason parts)
- malformed? (and (map? result) (contains? result :_raw_arguments))]
+ ;; Only malformed JSON can be a cut-off tool call. A non-object reply parsed in full.
+ malformed? (contains? unparsed :raw-arguments)]
(cond
;; A tool call cut off mid-JSON is not a model emitting bad JSON — the turn ran out of
;; room — so report why it stopped. Only `length` reroutes this branch: a content filter
@@ -701,14 +704,15 @@
(and malformed? (= incomplete-reason "length"))
(throw (incomplete-structured-output-error parts incomplete-reason))
- ;; The tool call's JSON failed to parse; `parse-tool-arguments` returned the
- ;; `{:_raw_arguments ...}` sentinel. Reject it as invalid rather than handing a
- ;; bogus map back to the caller as if it were a valid structured result.
- malformed?
- (throw (ex-info "LLM returned malformed JSON in its structured tool call"
- {:parts parts
- :error-code "structured-output-invalid"
- :raw-arguments (:_raw_arguments result)}))
+ ;; The tool call's arguments were malformed JSON or valid JSON that isn't an object. Reject them
+ ;; rather than handing the parse sentinel back to the caller as if it were a structured result.
+ unparsed
+ (throw (ex-info (if malformed?
+ "LLM returned malformed JSON in its structured tool call"
+ "LLM returned a JSON value that is not an object in its structured tool call")
+ (merge {:parts parts
+ :error-code "structured-output-invalid"}
+ unparsed)))
result
{:result result :parts parts}
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 49a8383a56f..6d0d1b5a40f 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -116,8 +116,10 @@
[:type {:optional true} [:maybe :string]]]]]]])
(mr/def ::decoded-json
- "A value decoded from JSON.
- Object keys are strings from `json/decode` (replayed history) or keywords from `json/decode+kw` (the stream)."
+ "A value decoded from JSON with `json/decode`, with string object keys at every depth.
+
+ `{\"stages\" [{\"limit\" 1}]}` passes and `{:stages [{:limit 1}]}` does not. [[stringify-nested-keys]] gives this
+ shape below the argument names."
[:or
:string
:keyword
@@ -125,13 +127,15 @@
:boolean
:nil
[:sequential [:ref ::decoded-json]]
- [:map-of [:or :string :keyword] [:ref ::decoded-json]]])
+ [:map-of :string [:ref ::decoded-json]]])
(def ^:private ToolCallArguments
- "A tool call's arguments as the LLM wrote them against the tool's own schema, keyed by that tool's argument names:
- string keys off the wire, keyword keys when built in Clojure."
+ "A tool call's arguments as the LLM wrote them against the tool's own schema, keyed by its argument names as keywords.
+
+ `{:query {\"stages\" []}}` passes. `{\"query\" {\"stages\" []}}` and `{:query {:stages []}}` do not: the argument
+ names are keywords, and the values are `::decoded-json`, with string keys."
[:map-of {::mr/deliberately-open true, :description "tool call arguments"}
- [:or :string :keyword] ::decoded-json])
+ :keyword ::decoded-json])
(def ^:private AISDKPart
"One element of the `:input` sequence passed to a provider adapter: an AISDK part keyed by
@@ -371,22 +375,59 @@
(get stop-reasons raw "other")))
(defn- parse-tool-arguments
- "Parse concatenated tool input deltas as JSON.
- Falls back to returning the raw string wrapped in a map when parsing fails,
- e.g. when the LLM produces malformed JSON in tool call arguments."
+ "Parse concatenated tool input deltas as a JSON object with keyword argument names.
+
+ `{\"query\":{\"stages\":[]}}` becomes `{:query {\"stages\" []}}`: only the top-level names are keywordized. Blank
+ input (a call to a tool with no arguments) parses to nil. Anything else comes back as a sentinel map that
+ [[unparsed-arguments]] describes and [[reject-unparsed-arguments!]] turns into an error for the model:
+ `{::raw-arguments raw}` for malformed JSON, and `{::non-object-arguments decoded}` for valid JSON that isn't an
+ object, e.g. `[\"orders\"]`. The keys are namespaced so they don't clash with a tool's argument names."
[chunks]
(let [raw (->> (map :inputTextDelta chunks)
(str/join ""))]
(try
- (json/decode+kw raw)
+ ;; Keywordize only the top-level argument names. Nested objects are data the model wrote and stay string-keyed
+ ;; JSON. The executor and title-fn keywordize the full arguments themselves.
+ (let [arguments (json/decode raw)]
+ (cond
+ (map? arguments) (update-keys arguments keyword)
+ (nil? arguments) nil
+ :else {::non-object-arguments arguments}))
(catch Exception e
(log/warn "Failed to parse tool arguments as JSON, passing raw string"
{:tool (:toolName (first chunks))
:error (.getMessage e)
:raw-len (count raw)})
- ;; Return a map with a sentinel key so the tool sees an error via schema validation
+ ;; Return a sentinel map so the model gets a repair message from [[reject-unparsed-arguments!]]
;; rather than a cryptic JSON parse stacktrace.
- {:_raw_arguments raw}))))
+ {::raw-arguments raw}))))
+
+(defn unparsed-arguments
+ "Diagnostic data for tool `arguments` that are a [[parse-tool-arguments]] sentinel, or nil for parsed arguments.
+
+ `{::raw-arguments \"{oops\"}` gives `{:raw-arguments \"{oops\"}`, `{::non-object-arguments [\"orders\"]}` gives
+ `{:non-object-arguments [\"orders\"]}`, and `{:table \"orders\"}` or nil gives nil."
+ [arguments]
+ (when (map? arguments)
+ (cond
+ (contains? arguments ::raw-arguments) {:raw-arguments (::raw-arguments arguments)}
+ (contains? arguments ::non-object-arguments) {:non-object-arguments (::non-object-arguments arguments)})))
+
+(defn stringify-nested-keys
+ "Return `arguments` with every map key below the top level as a string.
+
+ `{:query {:stages [{:limit 1}]}}` becomes `{:query {\"stages\" [{\"limit\" 1}]}}`. The top-level keys do not change,
+ so the argument names and the [[parse-tool-arguments]] sentinel keys stay keywords. A namespaced keyword key
+ becomes `\"ns/name\"`, the key `json/encode` writes for it."
+ [arguments]
+ (letfn [(below [x]
+ (cond
+ (map? x) (u/for-map [[k v] x]
+ [(cond-> k (keyword? k) u/qualified-name) (below v)])
+ (sequential? x) (mapv below x)
+ :else x))]
+ (cond-> arguments
+ (map? arguments) (update-vals below))))
(defn- try-decode-json-string
"If `v` is a string that looks like a JSON object or array, decode it.
@@ -972,23 +1013,36 @@
(str "; received " (json-type-name (get arguments field))))
"."))))
+(defn- non-object-arguments-message
+ "The error for tool `arguments` that are not a JSON object, e.g. `[\"orders\"]`."
+ [arguments]
+ (str "Invalid tool arguments: expected an object of named arguments; received " (json-type-name arguments) "."))
+
(defn- invalid-arguments-message
"A repair-oriented message describing how `arguments` violate `schema`, or nil when they match."
[schema arguments]
(when-let [error (mr/explain schema arguments)]
(let [humanized (me/humanize error)]
- (str "Invalid tool arguments: "
- (if (map? humanized)
+ (if (map? humanized)
+ (str "Invalid tool arguments: "
(str/join " " (for [[field messages] (sort-by (comp name key) humanized)]
- (argument-error-text arguments field messages)))
- (str "expected an object of named arguments; received "
- (json-type-name arguments) "."))))))
+ (argument-error-text arguments field messages))))
+ ;; The error isn't per-field. Today that only happens when a tool's own `:decode` returns a non-map: a
+ ;; JSON array the model sent is rejected earlier by [[reject-unparsed-arguments!]].
+ (non-object-arguments-message arguments)))))
+
+(defn- reject-unparsed-arguments!
+ "Throw an agent error when `arguments` is a [[parse-tool-arguments]] sentinel rather than a JSON object."
+ [arguments]
+ (when-let [unparsed (unparsed-arguments arguments)]
+ (if (contains? unparsed :raw-arguments)
+ (throw (ex-info "Invalid tool arguments: the arguments were not valid JSON. Send the call again as a JSON object."
+ {:agent-error? true}))
+ (throw (ex-info (non-object-arguments-message (:non-object-arguments unparsed))
+ {:agent-error? true})))))
(defn- validate-tool-arguments!
[tool arguments]
- (when (and (map? arguments) (contains? arguments :_raw_arguments))
- (throw (ex-info "Invalid tool arguments: the arguments were not valid JSON. Send the call again as a JSON object."
- {:agent-error? true})))
(when-let [schema (tool-args-schema tool)]
(when-let [message (invalid-arguments-message schema arguments)]
(throw (ex-info message {:agent-error? true})))))
@@ -1039,6 +1093,9 @@
(str/join ", " (sort (keys tools))) ".")
{:agent-error? true})))
(let [{:keys [arguments]} (into {} (aisdk-xf) chunks)
+ ;; Before coercion: `coerce-stringified-json` would decode a double-encoded
+ ;; `{::non-object-arguments "{...}"}` into an object and garble the message.
+ _ (reject-unparsed-arguments! arguments)
arguments (walk/keywordize-keys (or (coerce-stringified-json arguments) {}))
arguments (coerce-stringified-scalars tool arguments)
decode (tool-decode-fn tool)
diff --git a/test/metabase/metabot/agent/messages_test.clj b/test/metabase/metabot/agent/messages_test.clj
index f0fb328be9d..ed82ecfc280 100644
--- a/test/metabase/metabot/agent/messages_test.clj
+++ b/test/metabase/metabot/agent/messages_test.clj
@@ -6,6 +6,8 @@
[metabase.metabot.agent.memory :as memory]
[metabase.metabot.agent.messages :as messages]
[metabase.metabot.agent.user-context :as user-context]
+ [metabase.metabot.persistence :as metabot.persistence]
+ [metabase.metabot.self.openai.chat-completions :as chat-completions]
[metabase.test :as mt]))
;;; ──────────────────────────────────────────────────────────────────
@@ -26,7 +28,7 @@
(messages/input-message->parts {:role :assistant :content "Hi there"}))))
(testing "assistant with tool_calls (OpenAI style)"
(is (=? [{:type :text :text "Searching..."}
- {:type :tool-input :id "t1" :function "search" :arguments {"q" "test"}}]
+ {:type :tool-input :id "t1" :function "search" :arguments {:q "test"}}]
(messages/input-message->parts
{:role :assistant
:content "Searching..."
@@ -52,7 +54,14 @@
(is (=? [{:type :tool-input :id "t1" :function "search" :arguments "{bad-json"}]
(messages/input-message->parts
{:role :assistant
- :tool_calls [{:id "t1" :name "search" :arguments "{bad-json"}]})))))
+ :tool_calls [{:id "t1" :name "search" :arguments "{bad-json"}]}))))
+ (testing "map tool_call arguments get keyword argument names and string keys below them"
+ (is (= {:query {"stages" [{"limit" 1}]}}
+ (-> (messages/input-message->parts
+ {:role :assistant
+ :tool_calls [{:id "t1" :name "search" :arguments {"query" {:stages [{:limit 1}]}}}]})
+ first
+ :arguments)))))
(deftest ^:parallel input-message-tool-test
(testing "tool result message"
@@ -73,6 +82,21 @@
:content [{:type "tool_result" :tool_use_id "t1" :content "Result 1"}
{:type "tool_result" :tool_use_id "t2" :content "Result 2"}]})))))
+(deftest ^:parallel persisted-tool-call-replays-into-adapter-test
+ (testing "a stored tool call replays with keyword argument names and string nested keys, and the adapter accepts it"
+ (let [stored {:type "tool-construct_notebook_query"
+ :toolCallId "call-1"
+ :state "output-available"
+ :input {:query {:stages [{:source-table 1}]} :limit 5}
+ :output {:output "ok"}}
+ parts (into [] (mapcat messages/input-message->parts)
+ (metabot.persistence/tool-part->llm-messages stored))]
+ (is (= {:query {"stages" [{"source-table" 1}]} :limit 5}
+ (:arguments (first parts))))
+ (is (=? {:messages [{:role "assistant" :tool_calls [{:id "call-1" :function {:arguments string?}}]}
+ {:role "tool" :tool_call_id "call-1"}]}
+ (chat-completions/request-body {:model "some/model" :input parts}))))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; build-message-history
;;; ──────────────────────────────────────────────────────────────────
diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.clj
index 03a33f254aa..565e3ec1aab 100644
--- a/test/metabase/metabot/self/claude_test.clj
+++ b/test/metabase/metabot/self/claude_test.clj
@@ -340,6 +340,27 @@
(testing "no speed through the AI proxy"
(is (nil? (speed {:model "claude-opus-5" :fast? true :ai-proxy? true}))))))
+(deftest ^:parallel claude-request-body-nested-object-arguments-test
+ (testing "a replayed tool call whose arguments hold nested string-keyed objects is accepted and sent as-is"
+ ;; Replayed tool calls go through this instrumented entry point, so their arguments must validate here.
+ (let [arguments {:query {"lib/type" "mbql/query"
+ "stages" [{"source-table" ["db" "public" "orders"]}]}
+ :visualization {"chart_type" "line"}
+ :title "Orders per month"
+ :description "Monthly orders in 2025"}
+ body (claude/claude-request-body
+ {:input [{:role :user :content "How many orders were placed each month in 2025?"}
+ {:type :tool-input :id "call-1" :function "construct_notebook_query" :arguments arguments}
+ {:type :tool-output
+ :id "call-1"
+ :function "construct_notebook_query"
+ :result {:output "query built"}}]})]
+ (is (=? [{:role "user"}
+ {:role "assistant"
+ :content [{:type "tool_use" :id "call-1" :name "construct_notebook_query" :input arguments}]}
+ {:role "user" :content [{:type "tool_result" :tool_use_id "call-1"}]}]
+ (:messages body))))))
+
(defn- close-tracking-json-body
"A streamed JSON error body that flips `closed?` when closed, like the real `:as :stream`
response body the adapter must not leak."
diff --git a/test/metabase/metabot/self/openai/chat_completions_test.clj b/test/metabase/metabot/self/openai/chat_completions_test.clj
index fb57d6fb54d..8d438345a03 100644
--- a/test/metabase/metabot/self/openai/chat_completions_test.clj
+++ b/test/metabase/metabot/self/openai/chat_completions_test.clj
@@ -265,7 +265,7 @@
:id "call-1"
:function "construct_notebook_query"
:arguments {:query query}}
- ;; the stream parser decodes the arguments with keyword keys at every depth
+ ;; the stream parser keywordizes only the top-level argument names
parts (into [] (self.core/aisdk-xf) (metabot.tu/parts->aisdk-chunks [call]))]
(is (=? {:messages [{:role "assistant"
:tool_calls [{:function {:name "construct_notebook_query"
@@ -280,18 +280,21 @@
:id "call-1"
:function "f"
:arguments arguments}]}))]
- (testing "decoded JSON replays at any depth, keyed by strings, keywords or both"
+ (testing "decoded JSON replays at any depth, with keyword argument names and string keys below them"
(are [arguments] (=? {:messages [{:tool_calls [{:function {:arguments string?}}]}]}
(replay arguments))
- ;; keyword keys, as the stream decodes them
- {:a {:b {:c [{:d [1 nil true "x" :kw]}]}}}
- ;; string keys, as replayed history decodes them
- {"a" {"b" {"c" [{"d" [1.5 [[]] {}]}]}}}
- ;; both
- {:a {"b" [{:c {"d" [{:e 1}]}}]}}))
+ ;; string keys below the argument names, as the stream parser and replayed history decode them
+ {:a {"b" {"c" [{"d" [1 nil true "x" :kw]}]}}}
+ {:a {"b" {"c" [{"d" [1.5 [[]] {}]}]}}}))
+ (testing "a string argument name is rejected"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid input"
+ (replay {"a" 1}))))
+ (testing "a keyword key below the argument names is rejected"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid input"
+ (replay {:a {"b" [{:c 1}]}}))))
(testing "a value JSON cannot hold is rejected, however deep"
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid input"
- (replay {:a {"b" [{:c (java.time.Instant/now)}]}}))))))
+ (replay {:a {"b" [{"c" (java.time.Instant/now)}]}}))))))
;;; ──────────────────────────────────────────────────────────────────
;;; Streaming chunk conversion tests
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index e523209ae57..a9819657235 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -538,6 +538,50 @@
{:type :tool-input-delta :toolCallId "call-1" :inputTextDelta "{}"}
{:type :tool-input-available :toolCallId "call-1" :toolName "search"}])))))))
+(deftest ^:parallel aisdk-xf-nested-object-arguments-validate-test
+ (testing "a tool call with object-valued arguments replays as valid LLMRequestOpts input"
+ ;; The agent loop replays tool calls on its next iteration, where every adapter validates them against
+ ;; `LLMRequestOpts`.
+ (let [chunks [{:type :tool-input-start :toolCallId "call-1" :toolName "construct_notebook_query"}
+ {:type :tool-input-delta
+ :toolCallId "call-1"
+ :inputTextDelta "{\"query\":{\"lib/type\":\"mbql/query\","}
+ {:type :tool-input-delta
+ :toolCallId "call-1"
+ :inputTextDelta "\"stages\":[{\"source-table\":[\"db\",\"public\",\"orders\"]}]},"}
+ {:type :tool-input-delta
+ :toolCallId "call-1"
+ :inputTextDelta "\"visualization\":{\"chart_type\":\"line\"},"}
+ {:type :tool-input-delta
+ :toolCallId "call-1"
+ :inputTextDelta "\"title\":\"Orders per month\",\"description\":\"Monthly orders in 2025\"}"}
+ {:type :tool-input-available :toolCallId "call-1" :toolName "construct_notebook_query"}]
+ parts (into [] (self.core/aisdk-xf) chunks)]
+ (is (= [{:type :tool-input
+ :id "call-1"
+ :function "construct_notebook_query"
+ :arguments {:query {"lib/type" "mbql/query"
+ "stages" [{"source-table" ["db" "public" "orders"]}]}
+ :visualization {"chart_type" "line"}
+ :title "Orders per month"
+ :description "Monthly orders in 2025"}}]
+ parts))
+ (is (nil? (mr/explain self.core/LLMRequestOpts {:input parts}))))))
+
+(deftest ^:parallel aisdk-xf-non-object-arguments-validate-test
+ (testing "a tool call whose arguments aren't a JSON object replays as valid LLMRequestOpts input"
+ ;; The executor rejects the call, and the agent loop replays it with that error on its next iteration.
+ (doseq [[raw decoded] [["[\"orders\"]" ["orders"]]
+ ["false" false]
+ ["[{\"a\":{\"b\":1}}]" [{"a" {"b" 1}}]]]]
+ (testing raw
+ (let [parts (into [] (self.core/aisdk-xf)
+ [{:type :tool-input-start :toolCallId "call-1" :toolName "search"}
+ {:type :tool-input-delta :toolCallId "call-1" :inputTextDelta raw}
+ {:type :tool-input-available :toolCallId "call-1" :toolName "search"}])]
+ (is (= {::self.core/non-object-arguments decoded} (:arguments (first parts))))
+ (is (nil? (mr/explain self.core/LLMRequestOpts {:input parts}))))))))
+
;;; tool executor
(deftest ^:parallel tool-executor-xf-test
@@ -720,6 +764,43 @@
(is (= "Invalid tool arguments: the arguments were not valid JSON. Send the call again as a JSON object."
(-> (into [] (self.core/tool-executor-xf tools) chunks) last :error :message))))))
+(deftest ^:parallel tool-non-object-arguments-test
+ (let [received (atom ::not-called)
+ tools {"open" {:fn (fn [args] (reset! received args) {:output "ok"})
+ :doc "accepts any object"
+ :schema [:=> [:cat [:map]] :any]}
+ "untyped" {:fn (fn [args] (reset! received args) {:output "ok"})
+ :doc "declares no argument schema"
+ :schema nil}}
+ run (fn run
+ ([raw] (run "open" raw))
+ ([tool-name raw]
+ (reset! received ::not-called)
+ (-> (into [] (self.core/tool-executor-xf tools)
+ [{:type :tool-input-start :toolName tool-name :toolCallId "call-n"}
+ {:type :tool-input-delta :toolCallId "call-n" :inputTextDelta raw}
+ {:type :tool-input-available :toolName tool-name :toolCallId "call-n"}])
+ last :error :message)))]
+ (testing "valid JSON that isn't an object is rejected before it reaches a tool whose schema accepts any map"
+ (is (= "Invalid tool arguments: expected an object of named arguments; received an array."
+ (run "[\"orders\"]")))
+ (is (= ::not-called @received)))
+ (testing "a JSON string holding an object is reported as the string the model sent, not decoded into one"
+ (is (= "Invalid tool arguments: expected an object of named arguments; received a string."
+ (run "\"{\\\"names\\\": [\\\"orders\\\"]}\"")))
+ (is (= ::not-called @received)))
+ (testing "a JSON boolean is rejected, not run as an empty argument map"
+ (is (= "Invalid tool arguments: expected an object of named arguments; received a boolean."
+ (run "false")))
+ (is (= ::not-called @received)))
+ (testing "JSON null still runs the tool with an empty argument map"
+ (is (nil? (run "null")))
+ (is (= {} @received)))
+ (testing "a tool with no argument schema rejects a non-object too"
+ (is (= "Invalid tool arguments: expected an object of named arguments; received an array."
+ (run "untyped" "[\"orders\"]")))
+ (is (= ::not-called @received)))))
+
(deftest ^:parallel tool-without-schema-is-not-validated-test
(testing "a tool with no declared argument schema is left alone"
(let [tools {"anything" {:fn (fn [_args] {:output "ok"}) :doc "d" :schema nil}}
@@ -1405,7 +1486,7 @@
(defn- malformed-tool-input-response
"A reducible LLM stream whose forced tool call streams invalid JSON, so
- `parse-tool-arguments` yields the `{:_raw_arguments ...}` sentinel.
+ `parse-tool-arguments` yields the raw-arguments sentinel.
`trailing-parts` are appended after it, e.g. a [[usage-part]] carrying a finish reason."
[& trailing-parts]
(let [chunks (concat [{:type :start :messageId "m1"}
@@ -1463,8 +1544,28 @@
0.3 1024 {:tag "metabot_agent"})
(catch clojure.lang.ExceptionInfo e e))]
(is (instance? clojure.lang.ExceptionInfo e)
- "malformed JSON must throw, not return the {:_raw_arguments ...} sentinel as a result")
- (is (= "structured-output-invalid" (:error-code (ex-data e))))))))))
+ "malformed JSON must throw, not return the raw-arguments sentinel as a result")
+ (is (= "structured-output-invalid" (:error-code (ex-data e))))
+ (is (= "{not valid json" (:raw-arguments (ex-data e))))))))))
+
+(deftest call-llm-structured-rejects-non-object-json-test
+ (llm.tu/with-default-connections
+ (testing "valid JSON that isn't an object is rejected as an error, not returned as a bogus result"
+ (mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)
+ openrouter/openrouter (constantly
+ (test-util/mock-llm-response
+ [{:type :start :id "m1"}
+ {:type :tool-input :id "c1" :function "json" :arguments ["orders"]}]))]
+ (let [e (try
+ (self/call-llm-structured "openrouter/test-model"
+ [{:role "user" :content "test"}]
+ {:type "object" :properties {:answer {:type "string"}}}
+ 0.3 1024 {:tag "metabot_agent"})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (instance? clojure.lang.ExceptionInfo e)
+ "a non-object payload must throw, not come back as the structured result")
+ (is (= "structured-output-invalid" (:error-code (ex-data e))))
+ (is (= ["orders"] (:non-object-arguments (ex-data e)))))))))
(deftest call-llm-structured-surfaces-provider-error-test
(llm.tu/with-default-connections
@@ -1706,6 +1807,42 @@
(is (= "structured-output-incomplete" (:error-code (ex-data e))))
(is (= "length" (:finish-reason (ex-data e)))))))))
+(deftest call-llm-structured-nested-keys-test
+ (llm.tu/with-default-connections
+ (let [schema {:type "object"
+ :properties {"items" {:type "array"
+ :items {:type "object"
+ :properties {"name" {:type "string"}}
+ :required ["name"]}}}
+ :required ["items"]}
+ answer {:items [{:name "x"}]}
+ call! (fn [part]
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter
+ (constantly (test-util/mock-llm-response
+ [{:type :start :id "m1"} part]))]
+ (self/call-llm-structured "openrouter/test-model" [{:role "user" :content "test"}]
+ schema 0.3 1024 {:tag "metabot_agent"})))]
+ (testing "a tool call and a text reply give the same result: keyword argument names, string keys below them"
+ (are [part] (= {:items [{"name" "x"}]} (call! part))
+ {:type :tool-input :id "c1" :function "structured_output" :arguments answer}
+ {:type :text :id "t1" :text (json/encode answer)})))))
+
+(deftest ^:parallel stringify-nested-keys-test
+ (testing "keys below the top level become strings, and the top-level keys stay keywords"
+ (is (= {:query {"stages" [{"limit" 1}]}}
+ (self.core/stringify-nested-keys {:query {:stages [{:limit 1}]}}))))
+ (testing "a nested namespaced keyword key becomes \"ns/name\", the key json/encode writes"
+ (is (= {:a {"foo/bar" 1}}
+ (self.core/stringify-nested-keys {:a {:foo/bar 1}}))))
+ (testing "the parse sentinels come back unchanged"
+ (are [arguments] (= arguments (self.core/stringify-nested-keys arguments))
+ {::self.core/raw-arguments "{oops"}
+ {::self.core/non-object-arguments [{"a" 1} "x"]}))
+ (testing "a value that is not a map comes back unchanged"
+ (are [arguments] (= arguments (self.core/stringify-nested-keys arguments))
+ nil
+ "{oops")))
+
(deftest call-llm-does-not-replay-after-partial-emission-test
(llm.tu/with-default-connections
(testing "a retryable failure AFTER parts were emitted does not replay the stream (no duplicate output / re-run tools)"