The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Stop treating metadata database failures as input errors (#84904)
Andrei Simionescu078317bc7c61
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex e2a8bd9d2dc..ce1c6f3dbb0 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1600,6 +1600,7 @@metabase.agent-api.schemametabase.agent-api.settings}:uses #{ai-tracing+ analytics.interfaceapichannelcollectionsdiff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.cljindex 8040ce8014d..80daca7e92e 100644--- a/src/metabase/agent_api/api.clj+++ b/src/metabase/agent_api/api.clj@@ -8,6 +8,7 @@[metabase.agent-api.settings :as agent-api.settings][metabase.agent-api.validation :as agent-api.validation][metabase.ai-tracing.core :as ait]+ [metabase.analytics-interface.core :as analytics][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.macros.scope :as scope]@@ -1828,8 +1829,29 @@;;; ---------------------------------------------------- Routes ------------------------------------------------------+(defn- wrap-database-errors+ "Return a generic 500 for unhandled database failures, logging the original exception."+ [handler]+ (fn [request respond raise]+ (let [raised? (volatile! false)+ raise (fn [e]+ (vreset! raised? true)+ (if (metabot-construct/database-error? e)+ (do+ (when (nil? (:status-code (ex-data e)))+ (analytics/inc! :metabase-api/unhandled-errors))+ (log/error e "Database failure in Agent API")+ (respond {:status 500 :body {:message (tru "Something went wrong")}}))+ (raise e)))]+ (try+ (handler request respond raise)+ (catch Throwable e+ (if @raised?+ (throw e)+ (raise e)))))))+(def ^:private base-routes- (api.macros/ns-handler *ns* +auth))+ (api.macros/ns-handler *ns* +auth wrap-database-errors))(def ^{:arglists '([request respond raise])} routes"`/api/agent/` routes."diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex e106835516d..79be4a926ec 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -31,7 +31,9 @@[metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr])+ (:import+ (java.sql SQLException)))(set! *warn-on-reflection* true)@@ -650,19 +652,30 @@;;; ---------------------------------------- Query execution ----------------------------------------+(defn database-error?+ "Returns true for database failures that must not be relayed as input errors."+ [e]+ (let [{:keys [status-code agent-error? error]} (ex-data e)+ causes (u/full-exception-chain e)]+ (boolean+ (and (not (and status-code (<= 400 status-code 499)))+ (or (some #(instance? SQLException %) causes)+ (and (not (and (nil? status-code)+ (or agent-error? (tools/pipeline-error-code error))))+ (some #(contains? (ex-data %) :toucan2/context-trace) causes)))))))+(defn- as-agent-input-error- "Wrap `e` as an agent-input error.+ "Return database failures unchanged; wrap other exceptions as agent-input errors.- Tool callers look at `:agent-error?` to decide whether to relay the message to the LLM.- HTTP callers additionally need a status code; default representation/repair failures to- 400 so individual repair passes don't all have to repeat `:status-code 400` in ex-data.- Existing statuses are preserved (notably permission 403s, which callers should normally- avoid wrapping in the first place)."+ The wrapper sets `:agent-error?` for tool callers and defaults `:status-code` to 400 for HTTP+ callers. Existing status codes are preserved."[^clojure.lang.ExceptionInfo e]- (let [base (assoc (or (ex-data e) {}) :agent-error? true)- data (cond-> base- (nil? (:status-code base)) (assoc :status-code 400))]- (ex-info (ex-message e) data e)))+ (if (database-error? e)+ e+ (let [base (assoc (or (ex-data e) {}) :agent-error? true)+ data (cond-> base+ (nil? (:status-code base)) (assoc :status-code 400))]+ (ex-info (ex-message e) data e))))(defn- not-runnable-problem"What is wrong at a Malli schema `error`, phrased for the LLM, or nil when there is nothing to add@@ -770,10 +783,7 @@;; Permission checks happen before repair/resolve so the metadata-provider-backed pipeline;; never inspects table/card metadata that the current user cannot use.checked (check-source-table-query-permissions! mp parsed)]- ;; Everything after the MP is built can surface LLM-input errors (lib.schema validation- ;; in resolve, missing-column complaints from lib/query in `result-columns-for-query`,- ;; etc.). Wrap the whole rest of the pipeline in a single `:agent-error?` relay so any of- ;; them reach the tool wrapper with the flag set.+ ;; Relay query-input errors while preserving database and permission failures.(try(let [;; Repair's mini-resolve lookups are best-effort - a failure is logged at debug and the;; clause left alone - and `resolve-query` below resolves the query for real. A refusal@@ -929,14 +939,7 @@:chart-typechart-type->keyword):table)- ;; The one place a representations-pipeline error is allowed to become something the- ;; model reads. A `:error` code the pipeline declares becomes that recoverable error with- ;; its recovery steps; anything else — a lib, toucan2 or JDBC failure that the pipeline's- ;; `as-agent-input-error` stamped `:agent-error?` onto without authoring a sentence for- ;; it — stays unrecoverable and ends the turn.- ;;- ;; `execute-representations-query` either throws or returns a structured output carrying- ;; both `:query-id` and `:query`, so there is nothing to branch on after it.+ ;; Declared pipeline errors and refusals are recoverable; anything else ends the turn.structured (:structured-output (tools/with-pipeline-errors(execute-representations-query query)));; A query built only to be read needs run_query, so without it the chart is the one waydiff --git a/src/metabase/metabot/tools/core.clj b/src/metabase/metabot/tools/core.cljindex 3d68608ad7f..626544b83f9 100644--- a/src/metabase/metabot/tools/core.clj+++ b/src/metabase/metabot/tools/core.clj@@ -470,7 +470,7 @@test to say so."(str (ns-name (:ns (meta #'recoverable.pipeline/unknown-table!)))))-(defn- pipeline-error-code+(defn pipeline-error-code"The `recoverable.pipeline` code for a pipeline `:error` keyword, or nil when none is declared."[error](when (keyword? error)@@ -487,10 +487,7 @@(let [{:keys [agent-error? status-code error entity-type entity-id]} (ex-data e)code (pipeline-error-code error)](cond- ;; `:agent-error?` *and* an `:error` key. The pipeline's `as-agent-input-error` stamps- ;; `:agent-error?` onto foreign exceptions from lib, toucan2 and JDBC as well, and those- ;; messages are not authored for anyone — only an `:error` code says "this sentence was- ;; written for a model".+ ;; Lib exceptions can also carry `:agent-error?`, but their messages aren't written for the model.(and agent-error? code)(tools.error/throw-recoverable!codediff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.cljindex e85e68ba9a2..16b5fd3e3c7 100644--- a/test/metabase/agent_api/api_test.clj+++ b/test/metabase/agent_api/api_test.clj@@ -14,6 +14,7 @@[metabase.collections.models.collection :as collection][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.normalize :as lib.normalize][metabase.mcp.core :as mcp][metabase.metabot.tools.core :as tools.core]@@ -32,7 +33,9 @@[metabase.util.json :as json][metabase.util.malli.registry :as mr][oidc-provider.util :as oidc.util]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.sql SQLException)))(set! *warn-on-reflection* true)@@ -314,6 +317,60 @@:aggregation [(orders-field-ref "ID")])})](is (=? {:error "aggregation-entry-not-aggregation"} response)))))+(deftest construct-query-metadata-lookup-failure-test+ (mt/with-prometheus-system! [_ system]+ (let [query (orders-query :fields [(orders-field-ref "ID")])]+ (doseq [[failure delta] [[(SQLException. "Connection unavailable") 1]+ [(ex-info "SELECT private_metadata failed" {:toucan2/context-trace []}) 1]+ [(ex-info "Metadata lookup failed" {} (SQLException. "Connection unavailable")) 1]+ [(ex-info "SELECT private_metadata failed"+ {:toucan2/context-trace [] :status-code 500}) 0]]]+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table (fn [& _] (throw failure))]+ (let [initial (mt/metric-value system :metabase-api/unhandled-errors)]+ (is (= {:message "Something went wrong"}+ (mt/user-http-request :rasta :post 500 "agent/v2/construct-query" {:query query})))+ (is (= (+ initial delta) (mt/metric-value system :metabase-api/unhandled-errors)))))))))++(deftest construct-query-declared-errors-in-toucan-scopes-test+ (let [query (orders-query :fields [(orders-field-ref "ID")])]+ (doseq [status-code [nil 400]+ agent-error? [false true]+ [error message expected] [[:invalid-representations-query+ "Representations query has an invalid structure."+ "Representations query has an invalid structure."]+ [:unknown-field+ "Unknown field"+ (str "Unknown field\nCall `read_resource` with `metabase://table/<numeric id>/fields` "+ "to list this table's columns.")]]]+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table+ (fn [& _]+ (t2/with-connection [_conn]+ (throw (ex-info message (cond-> {:error error}+ status-code (assoc :status-code status-code)+ agent-error? (assoc :agent-error? true))))))]+ (is (=? {:error (name error) :message expected}+ (mt/user-http-request :rasta :post 400 "agent/v2/construct-query" {:query query})))))))++(deftest database-error-callback-failure-test+ (mt/with-prometheus-system! [_ system]+ (doseq [database-error? [false true]+ delivery [:throw :raise]]+ (let [failure (if database-error? (SQLException. "Database failure") (Exception. "Handler failure"))+ callback-failure (if database-error? (SQLException. "Callback failure") (Exception. "Callback failure"))+ calls (atom 0)+ callback (fn [_] (swap! calls inc) (throw callback-failure))+ handler (#'agent-api.api/wrap-database-errors+ (fn [_request _respond raise]+ (case delivery+ :throw (throw failure)+ :raise (raise failure))))+ initial (mt/metric-value system :metabase-api/unhandled-errors)+ caught (try (handler {} callback callback) (catch Throwable e e))]+ (is (identical? callback-failure caught))+ (is (= 1 @calls))+ (is (= (+ initial (if database-error? 1 0))+ (mt/metric-value system :metabase-api/unhandled-errors)))))))+(deftest construct-query-permission-checks-test(testing "Rejects a first-stage source-table the current user cannot query"(mt/with-no-data-perms-for-all-users!diff --git a/test/metabase/mcp/v2/tools/query_test.clj b/test/metabase/mcp/v2/tools/query_test.cljindex 58ec9f76679..542acca3888 100644--- a/test/metabase/mcp/v2/tools/query_test.clj+++ b/test/metabase/mcp/v2/tools/query_test.clj@@ -12,6 +12,7 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.mcp.v2.message :as message][metabase.mcp.v2.queries :as v2.queries][metabase.mcp.v2.registry :as registry]@@ -22,7 +23,10 @@[metabase.test :as mt][metabase.test.data.interface :as tx][metabase.test.fixtures :as fixtures]- [metabase.util.json :as json]))+ [metabase.util.json :as json]+ [toucan2.core :as t2])+ (:import+ (java.sql SQLException)))(set! *warn-on-reflection* true)@@ -582,6 +586,36 @@(catch Exception e e))](is (= 400 (:status-code (ex-data e)))))))))+(deftest ^:parallel metadata-lookup-failure-is-internal-test+ (mt/with-current-user (mt/user->id :rasta)+ (let [query (orders-query {:fields [(field-name-ref :orders :id)]})]+ (doseq [failure [(ex-info "SELECT private_metadata failed" {:toucan2/context-trace []})+ (ex-info "Metadata lookup failed" {} (SQLException. "Connection unavailable"))]]+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table (fn [& _] (throw failure))]+ (is (= {:result {:isError true :content [{:type "text" :text "Internal error"}]}}+ (registry/call-tool execute-scope (str (random-uuid)) "execute_query" {:query query}))))))))++(deftest ^:parallel declared-errors-in-toucan-scopes-are-recoverable-test+ (mt/with-current-user (mt/user->id :rasta)+ (let [query (orders-query {:fields [(field-name-ref :orders :id)]})]+ (doseq [status-code [nil 400]+ agent-error? [false true]+ [error message expected] [[:invalid-representations-query+ "Representations query has an invalid structure."+ "\"Representations query has an invalid structure.\""]+ [:unknown-field+ "Unknown field"+ (str "\"Unknown field\"\nCall \"browse_data\" with action \"get_fields\" for "+ "the table to list its columns with their numeric ids.")]]]+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table+ (fn [& _]+ (t2/with-connection [_conn]+ (throw (ex-info message (cond-> {:error error}+ status-code (assoc :status-code status-code)+ agent-error? (assoc :agent-error? true))))))]+ (is (= {:result {:isError true :content [{:type "text" :text expected}]}}+ (registry/call-tool execute-scope (str (random-uuid)) "execute_query" {:query query}))))))))+(deftest ^:parallel native-query-rejection-test(mt/with-current-user (mt/user->id :rasta)(let [sid (str (random-uuid))]diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.cljindex 0e65bdb8c45..9e600629c95 100644--- a/test/metabase/metabot/tools/construct_representations_test.clj+++ b/test/metabase/metabot/tools/construct_representations_test.clj@@ -8,7 +8,7 @@unknown name, ambiguous name, missing name, mismatched DB component in source-table)."(:require[clojure.string :as str]- [clojure.test :refer [are deftest is testing]]+ [clojure.test :refer [are deftest is testing use-fixtures]][clojure.walk :as walk][malli.core :as mc][metabase.agent-lib.representations.repair :as repr.repair]@@ -17,9 +17,11 @@[metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.test-util :as lib.tu][metabase.mcp.v2.message :as message][metabase.mcp.v2.recovery-hints :as v2-hints]+ [metabase.metabot.agent.core :as agent][metabase.metabot.agent.links :as links][metabase.metabot.db :as metabot.db][metabase.metabot.tools.construct :as construct]@@ -27,15 +29,22 @@[metabase.metabot.tools.entity-details :as entity-details][metabase.metabot.tools.error :as tools.error][metabase.metabot.tools.recovery-hints :as v1-hints]+ [metabase.metabot.tools.runtime :as tools.runtime][metabase.models.interface :as mi][metabase.models.serialization.resolve :as serdes.resolve][metabase.models.serialization.resolve.mp :as resolve.mp][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][metabase.util.json :as json]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2])+ (:import+ (java.sql SQLException)))(set! *warn-on-reflection* true)+(use-fixtures :once (fixtures/initialize :db))+(def ^:private mp(lib.tu/mock-metadata-provider{:database {:id 1 :name "Sample"}@@ -205,6 +214,108 @@(is (true? (:agent-error? d)))(is (= :unknown-table (:error d)))))))))+(defn- metadata-lookup-failures []+ (let [message "SELECT private_metadata failed: connection unavailable"+ root-cause (RuntimeException. message)]+ {"Toucan" (ex-info message {:toucan2/context-trace []})+ "nested Toucan" (ex-info message {} (ex-info message {:toucan2/context-trace nil} root-cause))+ "JDBC" (SQLException. message)+ "nested JDBC" (ex-info message {} (RuntimeException. message (SQLException. message root-cause)))+ "declared JDBC" (ex-info message {:error :unknown-field :toucan2/context-trace []}+ (SQLException. message))+ "unknown code" (ex-info message {:error :not-a-pipeline-error :toucan2/context-trace []})+ "Malli data" (ex-info message {:error {:type :invalid-type} :toucan2/context-trace []})}))++(def ^:private orders-id-query+ {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql"+ :source-table ["Sample" "PUBLIC" "ORDERS"]+ :fields [["field" {} ["Sample" "PUBLIC" "ORDERS" "ID"]]]}]})++(deftest metadata-lookup-failures-propagate-test+ (with-mp-and-stubs!+ (fn []+ (doseq [[label failure] (metadata-lookup-failures)]+ (testing label+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table (fn [& _] (throw failure))]+ (let [caught (try+ (construct/execute-representations-query orders-id-query)+ (catch Throwable e e))]+ (is (identical? failure caught))+ (is (nil? (:agent-error? (ex-data caught))))+ (is (nil? (:status-code (ex-data caught)))))))))))++(deftest metadata-validation-errors-are-agent-input-errors-test+ (with-mp-and-stubs!+ (fn []+ (let [failure (ex-info "Unknown field" {:error :unknown-field})]+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table (fn [& _] (throw failure))]+ (let [caught (try+ (construct/execute-representations-query orders-id-query)+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= {:error :unknown-field :agent-error? true :status-code 400} (ex-data caught)))+ (is (= "Unknown field" (ex-message caught)))+ (is (identical? failure (ex-cause caught)))))))))++(deftest ^:parallel authored-errors-in-toucan-scopes-test+ (doseq [[status database-error?] [[nil false] [500 true]]+ data [{:error :unknown-field} {:agent-error? true}]]+ (is (= database-error?+ (construct/database-error?+ (ex-info "Failure" (assoc data :status-code status :toucan2/context-trace [])))))))++(deftest declared-errors-in-toucan-scopes-are-recoverable-test+ (with-mp-and-stubs!+ (fn []+ (doseq [status-code [nil 400]+ agent-error? [false true]]+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table+ (fn [& _]+ (t2/with-connection [_conn]+ (throw (ex-info "Representations query has an invalid structure."+ (cond-> {:error :invalid-representations-query}+ status-code (assoc :status-code status-code)+ agent-error? (assoc :agent-error? true))))))]+ (let [caught (try+ (construct/execute-representations-query orders-id-query)+ (catch clojure.lang.ExceptionInfo e e))+ entries (tools/entries [construct/construct-notebook-query-tool])+ outcome (tools.runtime/invoke entries+ {:profile-id :nlq :metabot nil+ :user-scope #{"agent:notebook:create"}+ :tool-names (set (keys entries))}+ "construct_notebook_query"+ {:query orders-id-query+ :title "Order IDs" :description "IDs of orders."})]+ (is (contains? (ex-data caught) :toucan2/context-trace))+ (is (=? {:agent-error? true :status-code 400 :error :invalid-representations-query}+ (ex-data caught)))+ (is (= {:output "Representations query has an invalid structure."+ :error {:class :recoverable+ :code :metabase.metabot.tools.recoverable.pipeline/invalid-representations-query}}+ outcome))+ (is (nil? (#'agent/terminal-error-message [{:type :tool-output :result outcome}])))))))))++(deftest metadata-lookup-failures-end-metabot-turn-test+ (with-mp-and-stubs!+ (fn []+ (doseq [[label failure] (metadata-lookup-failures)]+ (testing label+ (mt/with-dynamic-fn-redefs [lib.metadata.protocols/metadatas-for-table (fn [& _] (throw failure))]+ (let [entries (tools/entries [construct/construct-notebook-query-tool])+ outcome (tools.runtime/invoke entries+ {:profile-id :nlq :metabot nil+ :user-scope #{"agent:notebook:create"}+ :tool-names (set (keys entries))}+ "construct_notebook_query"+ {:query orders-id-query+ :title "Order IDs" :description "IDs of orders."})]+ (is (= {:output "This call failed and the user was shown the error (:internal). Don't retry it."+ :error {:class :unrecoverable :code :internal}}+ outcome))+ (is (= "Something went wrong."+ (#'agent/terminal-error-message [{:type :tool-output :result outcome}]))))))))))+(deftest implicit-join-happy-path-test(testing "cross-table breakout gets auto-wired with a :source-field after repair"(with-mp-and-stubs!diff --git a/test/metabase/metabot/tools/core_test.clj b/test/metabase/metabot/tools/core_test.cljindex 8c2d0a8b711..250e9aa25f6 100644--- a/test/metabase/metabot/tools/core_test.clj+++ b/test/metabase/metabot/tools/core_test.clj@@ -352,9 +352,7 @@(pipeline-failure "no" {:status-code status}))))))(deftest ^:parallel with-pipeline-errors-does-not-relay-foreign-messages-test- (testing "`as-agent-input-error` stamps :agent-error? onto exceptions from lib, toucan2 and JDBC- too. Without an :error code there is nothing to say the message was written for a model,- so it stays unrecoverable and the model never sees it."+ (testing "Lib exceptions stay unrecoverable without a declared `:error` code, even if `:agent-error?` is true."(is (=? {:class :unrecoverable:code :internal::ex-message "No matching clause: :whatever"}diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.cljindex 79997201861..47f61c75a4c 100644--- a/test/metabase/metabot/tools_test.clj+++ b/test/metabase/metabot/tools_test.clj@@ -267,11 +267,7 @@(is (= :metabase.metabot.tools.recoverable.pipeline/unknown-table code))(is (str/includes? text "No table found matching portable FK"))(is (str/includes? text "metabase://database/<numeric id>/tables"))))- (testing (str "an exception nobody authored a model-facing sentence for ends the turn, whether "- "or not the pipeline stamped `:agent-error?` onto it. `as-agent-input-error` "- "stamps the flag on failures from lib, toucan2 and JDBC as well, so the flag alone "- "never meant the message was written for anyone — only a declared `:error` code "- "does.")+ (testing "An exception without a declared `:error` code ends the turn, even if `:agent-error?` is true."(doseq [[what e] {"stamped by the pipeline, no :error code"(ex-info "No matching clause: :metabase.lib.schema/bogus" {:agent-error? true})