Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Keep SQL and stack traces out of Agent API and MCP errors (#85087)

Andrei Simionescuf45a658e42ab
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index a8028c0b361..c0fada3f42b 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -4,6 +4,13 @@ title: Driver interface changelog
# Driver Interface Changelog
+## Metabase 0.66.0
+
+- `metabase.driver.sql-jdbc.execute/impl-error-in-query?` `[driver exception]` identifies query errors raised while
+ reading rows. The default recognizes SQLState classes `21`, `22`, `23` and `42`. Override it to add vendor query
+ error codes or exclude SQLStates that also cover operational failures. Unknown failures remain QP errors;
+ cancellation takes precedence over query-error evidence.
+
## Metabase 0.65.0
- `metabase.driver/disallowed-connection-parameters` `[driver]` -- new multimethod returning the names of client
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index 4970b81763e..8938462986d 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -588,6 +588,12 @@
[_ _]
"NONE")
+(defmethod sql-jdbc.execute/impl-error-in-query? :clickhouse
+ [driver ^SQLException e]
+ ;; ClickHouse also uses class 22 for operational failures such as memory and result limits.
+ (and (not (str/starts-with? (str (.getSQLState e)) "22"))
+ ((get-method sql-jdbc.execute/impl-error-in-query? :sql-jdbc) driver e)))
+
(defmethod sql-jdbc/impl-table-known-to-not-exist? :clickhouse
[_ ^SQLException e]
;; the clickhouse driver doesn't set ErrorCode, we must parse it from the message
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 28e53e28c85..0a6a2810fd2 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -29,7 +29,7 @@
[toucan2.tools.with-temp :as t2.with-temp])
(:import
(com.clickhouse.jdbc ConnectionImpl)
- (java.sql Connection)))
+ (java.sql Connection SQLException)))
(set! *warn-on-reflection* true)
@@ -903,3 +903,10 @@
;; a lone double-quote is a one-character role name, not an already-quoted empty one -- it must still come
;; out as a terminated identifier
"\"" "SET ROLE \"\"\"\""))
+
+(deftest ^:parallel clickhouse-server-errors-need-more-than-class-22-test
+ (mt/test-driver :clickhouse
+ (doseq [[message code] [["Memory limit exceeded" 241] ["Too many rows or bytes" 396]]]
+ (let [e (SQLException. message "22000" code)]
+ (is (false? (#'sql-jdbc.execute/error-in-query? :clickhouse e)))
+ (is (true? (#'sql-jdbc.execute/error-in-query? :h2 e)))))))
diff --git a/modules/drivers/presto-jdbc/test/metabase/driver/presto_jdbc_test.clj b/modules/drivers/presto-jdbc/test/metabase/driver/presto_jdbc_test.clj
index a8351661e47..5fa0346fa7f 100644
--- a/modules/drivers/presto-jdbc/test/metabase/driver/presto_jdbc_test.clj
+++ b/modules/drivers/presto-jdbc/test/metabase/driver/presto_jdbc_test.clj
@@ -18,10 +18,15 @@
[metabase.test :as mt]
[metabase.test.data.presto-jdbc :as data.presto-jdbc]
[metabase.test.fixtures :as fixtures]
+ [metabase.util.json :as json]
[metabase.warehouses.core :as warehouses]
[toucan2.core :as t2])
(:import
- (java.io File)))
+ (com.sun.net.httpserver HttpExchange HttpHandler HttpServer)
+ (java.io File)
+ (java.net InetSocketAddress)
+ (java.sql DriverManager SQLException)
+ (java.util Properties)))
(set! *warn-on-reflection* true)
@@ -333,3 +338,52 @@
(lib/native-query (mt/metadata-provider))
(qp/process-query)
(mt/rows)))))))
+
+(defn- do-with-presto-cancellation!
+ "Run `f` with a Presto HTTP endpoint that cancels after its first page of rows."
+ [f]
+ (let [server (HttpServer/create (InetSocketAddress. "127.0.0.1" 0) 0)
+ base (str "http://127.0.0.1:" (.getPort (.getAddress server)))
+ shared {:id "canceled-query", :infoUri (str base "/info"), :warnings []}
+ page (merge shared {:nextUri (str base "/next")
+ :columns [{:name "x", :type "bigint", :typeSignature {:rawType "bigint", :arguments []}}]
+ :data [[1]], :stats {:state "RUNNING"}})
+ failed (merge shared {:stats {:state "FAILED"}
+ :error {:message "Query was canceled", :sqlState nil, :errorCode 3
+ :errorName "USER_CANCELED", :errorType "USER_ERROR", :retriable false}})]
+ (.createContext server "/"
+ (reify HttpHandler
+ (handle [_ exchange]
+ (let [^HttpExchange exchange exchange]
+ (.readAllBytes (.getRequestBody exchange))
+ (if (= "DELETE" (.getRequestMethod exchange))
+ (.sendResponseHeaders exchange 204 -1)
+ (let [body (.getBytes ^String (json/encode (if (= "/next" (.getPath (.getRequestURI exchange)))
+ failed page)) "UTF-8")]
+ (.set (.getResponseHeaders exchange) "Content-Type" "application/json")
+ (.sendResponseHeaders exchange 200 (alength body))
+ (with-open [out (.getResponseBody exchange)] (.write out body))))
+ (.close ^HttpExchange exchange)))))
+ (.start server)
+ (try
+ (f (.getPort (.getAddress server)))
+ (finally (.stop server 0)))))
+
+(deftest presto-server-cancellation-is-qp-error-test
+ (mt/test-driver :presto-jdbc
+ (Class/forName "com.facebook.presto.jdbc.PrestoDriver")
+ (do-with-presto-cancellation!
+ (fn [port]
+ (with-open [conn (DriverManager/getConnection (str "jdbc:presto://127.0.0.1:" port)
+ (doto (Properties.) (.setProperty "user" "test")))
+ stmt (.createStatement conn)
+ rs (.executeQuery stmt "SELECT 1 AS x")]
+ (is (.next rs))
+ (let [failure (try (.next rs) (catch SQLException e e))]
+ (is (nil? (.getSQLState ^SQLException failure)))
+ (is (= 3 (.getErrorCode ^SQLException failure)))))
+ (mt/with-temp [:model/Database db {:engine :presto-jdbc
+ :details {:host "127.0.0.1", :port port, :user "test", :ssl false}}]
+ (mt/with-db db
+ (is (=? {:status :failed, :error_type :qp}
+ (qp/process-query (qp/userland-query (lib/native-query (mt/metadata-provider) "SELECT 1 AS x")))))))))))
diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.clj
index 80daca7e92e..e4ceff35c18 100644
--- a/src/metabase/agent_api/api.clj
+++ b/src/metabase/agent_api/api.clj
@@ -27,6 +27,7 @@
[metabase.lib.core :as lib]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.metabot.core :as metabot]
+ [metabase.metabot.query-execution :as query-execution]
[metabase.metabot.tools.construct :as metabot-construct]
[metabase.metabot.tools.core :as tools.core]
[metabase.metabot.tools.recovery-hints :as recovery-hints]
@@ -42,12 +43,15 @@
[metabase.request.core :as request]
[metabase.server.streaming-response :as streaming-response]
[metabase.util :as u]
+ [metabase.util.api-error :as api-error]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (org.eclipse.jetty.io EofException)))
;;; --------------------------------------------------- Defaults ------------------------------------------------------
@@ -1827,31 +1831,71 @@
"Wrap routes so they may only be accessed when the Agent API is enabled."
agent-api.validation/+agent-api-enabled)
-;;; ---------------------------------------------------- Routes ------------------------------------------------------
-
-(defn- wrap-database-errors
- "Return a generic 500 for unhandled database failures, logging the original exception."
+(defn- exception-response
+ "The response to an exception raised while handling an Agent API request, never with a stack trace.
+ A 4xx exception without a database cause keeps its message and client-facing ex-data. Any other exception goes to the
+ server log, as a database failure when [[metabot-construct/database-error?]] says it is one, and the caller gets a
+ generic 500 whatever `hide-stacktraces` says."
+ [e]
+ (let [{:keys [status-code] :as data} (ex-data e)
+ client-data (api-error/response-data data)
+ database-error? (metabot-construct/database-error? e {:ignore-status? true})]
+ (if (and (not database-error?) status-code (<= 400 status-code 499))
+ {:status status-code
+ :body (if (empty? client-data)
+ (ex-message e)
+ (merge {:message (ex-message e)} client-data))}
+ (do
+ (if database-error?
+ (log/error e "Database failure in Agent API")
+ (log/error e "Unhandled error in Agent API request"))
+ (when (nil? status-code)
+ (analytics/inc! :metabase-api/unhandled-errors))
+ {:status 500
+ :body {:message (tru "Something went wrong")}}))))
+
+(defn- query-failure-body
+ "The body for an error that a streamed query run writes.
+ A failure that [[query-execution/caller-facing-failure?]] accepts keeps its status, error and error type. Any other
+ failure goes to the server log without the query, and the caller gets a generic message."
+ [error]
+ (if (query-execution/caller-facing-failure? error)
+ (merge {:status :failed} (select-keys error [:error :error_type]))
+ (do
+ (log/errorf "Agent API query run failed: %s" (u/pprint-to-str (dissoc error :json_query :preprocessed :native)))
+ {:status :failed
+ :error (tru "Something went wrong")})))
+
+(defn- wrap-error-responses
+ "Middleware that keeps internal detail out of Agent API error responses, since an agent passes them to its LLM.
+ A raised exception gets [[exception-response]], and an error that a streamed query run writes gets
+ [[query-failure-body]]. When the client went away before its response was written, the default exception handling
+ logs a canceled request, as on any other route."
[handler]
(fn [request respond raise]
- (let [raised? (volatile! false)
- raise (fn [e]
- (vreset! raised? true)
- (if (metabot-construct/database-error? e)
- (do
- (when (nil? (:status-code (ex-data e)))
- (analytics/inc! :metabase-api/unhandled-errors))
- (log/error e "Database failure in Agent API")
- (respond {:status 500 :body {:message (tru "Something went wrong")}}))
- (raise e)))]
+ (let [callback-called? (volatile! false)
+ respond' (fn [response]
+ (let [response (streaming-response/with-error-response-fn response query-failure-body)]
+ (vreset! callback-called? true)
+ (respond response)))
+ raise' (fn [e]
+ (when @callback-called?
+ (throw e))
+ (vreset! callback-called? true)
+ (if (instance? EofException e)
+ (raise e)
+ (respond' (exception-response e))))]
(try
- (handler request respond raise)
+ (handler request respond' raise')
(catch Throwable e
- (if @raised?
+ (if @callback-called?
(throw e)
- (raise e)))))))
+ (raise' e)))))))
+
+;;; ---------------------------------------------------- Routes ------------------------------------------------------
(def ^:private base-routes
- (api.macros/ns-handler *ns* +auth wrap-database-errors))
+ (api.macros/ns-handler *ns* +auth wrap-error-responses))
(def ^{:arglists '([request respond raise])} routes
"`/api/agent/` routes."
diff --git a/src/metabase/agent_api/reference.md b/src/metabase/agent_api/reference.md
index a5db24abbc4..069079e81a1 100644
--- a/src/metabase/agent_api/reference.md
+++ b/src/metabase/agent_api/reference.md
@@ -400,6 +400,9 @@ On failure:
{ "status": "failed", "error": "Query error message" }
```
+When the run fails inside Metabase, such as in an app DB lookup, `error` is
+`Something went wrong`, and the details go to the server log.
+
Row limits:
- Simple queries (no aggregation): 2000 rows max
@@ -630,5 +633,6 @@ Response:
| 401 | Authentication failure |
| 403 | Insufficient permissions |
| 404 | Entity not found |
+| 500 | Server error. A failure inside Metabase says only `Something went wrong` |
---
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index 8931bcf4079..c5b9deb6f99 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -1310,6 +1310,14 @@
:order-by [:a.table_name]}
:dialect (sql.qp/quote-style driver)))
+(def ^:private ^:const mariadb-json-table-field-error-code 4176)
+
+(defmethod sql-jdbc.execute/impl-error-in-query? :mysql
+ [driver ^SQLException e]
+ (or ((get-method sql-jdbc.execute/impl-error-in-query? :sql-jdbc) driver e)
+ (and (= "HY000" (.getSQLState e))
+ (= mariadb-json-table-field-error-code (.getErrorCode e)))))
+
(defmethod sql-jdbc/impl-query-canceled? :mysql [_ ^SQLException e]
;; ok to hardcode driver name here because this function only supports app DB types
(driver-api/query-canceled-exception? :mysql e))
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index bfae73bf45f..f8c3ede345b 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -33,7 +33,8 @@
(:import
(com.mchange.v2.c3p0 PooledDataSource)
(com.mchange.v2.resourcepool TimeoutException)
- (java.sql Connection JDBCType PreparedStatement ResultSet ResultSetMetaData SQLFeatureNotSupportedException Statement Types)
+ (java.sql Connection JDBCType PreparedStatement ResultSet ResultSetMetaData SQLException SQLFeatureNotSupportedException
+ Statement Types)
(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
(java.util.concurrent Executors)
(javax.sql DataSource)))
@@ -874,6 +875,36 @@
(log/infof "Statement cancelation failed: %s" (ex-message e))
false)))
+(defn- query-error
+ "An `:invalid-query` error for `e`, which the database raised while running `sql`."
+ [driver sql params e]
+ (ex-info (tru "Error executing query: {0}" (ex-message e))
+ (cond-> {:driver driver
+ :sql (str/split-lines (driver/prettify-native-form driver sql))
+ :params params
+ :type driver-api/qp.error-type.invalid-query}
+ (driver/query-canceled? driver e)
+ (assoc :query/query-canceled? true))
+ e))
+
+(defmulti impl-error-in-query?
+ "Whether a row-read `SQLException` identifies an error in the query or its data.
+ The default recognizes SQLState classes `21` (cardinality), `22` (data), `23` (integrity) and `42` (syntax or access
+ rule). Drivers can extend or narrow that evidence. Unknown failures return false. Cancellation is checked separately."
+ {:added "0.66.0" :arglists '([driver ^SQLException e])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod impl-error-in-query? :sql-jdbc
+ [_ ^SQLException e]
+ (boolean (re-find #"^(?:21|22|23|42)" (str (.getSQLState e)))))
+
+(defn- error-in-query?
+ [driver ^SQLException e]
+ (and (not (or (some-> (driver-api/canceled-chan) a/poll!)
+ (driver/query-canceled? driver e)))
+ (impl-error-in-query? driver e)))
+
(defn execute-reducible-query
"Default impl of [[metabase.driver/execute-reducible-query]] for sql-jdbc drivers."
{:added "0.35.0", :arglists '([driver query context respond])}
@@ -902,14 +933,7 @@
^ResultSet rs (try
(execute-statement-or-prepared-statement! driver stmt max-rows params sql)
(catch Throwable e
- (throw (ex-info (tru "Error executing query: {0}" (ex-message e))
- (cond-> {:driver driver
- :sql (str/split-lines (driver/prettify-native-form driver sql))
- :params params
- :type driver-api/qp.error-type.invalid-query}
- (driver/query-canceled? driver e)
- (assoc :query/query-canceled? true))
- e))))]
+ (throw (query-error driver sql params e))))]
(let [rsmeta (.getMetaData rs)
results-metadata {:cols (column-metadata driver rsmeta)}
;; whether the ResultSet ran out of rows is only observable while reducing, but it is needed after
@@ -918,7 +942,14 @@
next-row (row-thunk driver rs rsmeta)
rows (driver-api/reducible-rows
(fn []
- (let [row (next-row)]
+ ;; some databases report an error in the query only when they read the row
+ ;; that causes it
+ (let [row (try
+ (next-row)
+ (catch SQLException e
+ (throw (if (error-in-query? driver e)
+ (query-error driver sql params e)
+ e))))]
(when-not row
(vreset! exhausted? true))
row))
@@ -933,6 +964,8 @@
(finally
(when-not (or @exhausted? (drivers-exempt-from-cancelation driver))
(vreset! canceled? (cancel-statement! driver stmt)))))))
+ (catch SQLException e
+ (throw (ex-info (ex-message e) {:type driver-api/qp.error-type.qp} e)))
;; `with-open` has closed the ResultSet and Statement by the time this runs, which is required: on
;; ClickHouse and Presto their `.close` round-trips to the server and throws once the Connection is gone.
;; Discarding is safe here because this Connection was acquired for this query alone -- see
diff --git a/src/metabase/driver/sqlite.clj b/src/metabase/driver/sqlite.clj
index ccd12dee74b..1af1f00a6e6 100644
--- a/src/metabase/driver/sqlite.clj
+++ b/src/metabase/driver/sqlite.clj
@@ -12,6 +12,7 @@
[metabase.driver.common :as driver.common]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql :as driver.sql]
+ [metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
@@ -26,9 +27,10 @@
[metabase.util.performance :refer [mapv some]])
(:import
(java.net MalformedURLException URL)
- (java.sql Connection ResultSet Types)
+ (java.sql Connection ResultSet SQLException Types)
(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
- (java.time.temporal Temporal)))
+ (java.time.temporal Temporal)
+ (org.sqlite SQLiteErrorCode)))
(set! *warn-on-reflection* true)
@@ -744,5 +746,18 @@
[driver value]
(sql.qp/->integer-with-round driver value))
+(defmethod sql-jdbc.execute/impl-error-in-query? :sqlite
+ [driver ^SQLException e]
+ (or ((get-method sql-jdbc.execute/impl-error-in-query? :sql-jdbc) driver e)
+ (and (nil? (.getSQLState e))
+ (contains? #{(.code SQLiteErrorCode/SQLITE_ERROR)
+ (.code SQLiteErrorCode/SQLITE_CONSTRAINT)
+ (.code SQLiteErrorCode/SQLITE_MISMATCH)}
+ (.getErrorCode e)))))
+
+(defmethod sql-jdbc/impl-query-canceled? :sqlite
+ [_driver ^SQLException e]
+ (= (.code SQLiteErrorCode/SQLITE_INTERRUPT) (.getErrorCode e)))
+
(defmethod driver/llm-sql-dialect-resource :sqlite [_]
"metabot/prompts/dialects/sqlite.md")
diff --git a/src/metabase/mcp/v2/tools/query.clj b/src/metabase/mcp/v2/tools/query.clj
index 3d1373177a4..30551e069be 100644
--- a/src/metabase/mcp/v2/tools/query.clj
+++ b/src/metabase/mcp/v2/tools/query.clj
@@ -117,7 +117,7 @@
(message/msg ["Query failed: unknown error"])))
(defn- execute-page!
- "Run [[query-execution/execute-page!]] as an `:agent` query, turning a failed run into a teaching error."
+ "Run [[query-execution/execute-page!]] as an `:agent` query, turning a caller-facing failure into a teaching error."
[serialized-query row-limit]
;; Only the key whitelist in [[query-execution/execute-page!]] keeps unknown keys away from the QP.
;; A fresh `:query` is also rejected upstream by the closed `:metabase.lib.schema/query`, but that covers only the
@@ -126,8 +126,8 @@
(try
(query-execution/execute-page! serialized-query row-limit :agent)
(catch clojure.lang.ExceptionInfo e
- (let [{:keys [error query-error]} (ex-data e)]
- (if (= :query-failed error)
+ (let [{:keys [error query-error caller-facing?]} (ex-data e)]
+ (if (and (= :query-failed error) caller-facing?)
(common/throw-teaching-error (query-failure-message query-error))
(throw e))))))
@@ -554,7 +554,8 @@ Dialect (JSON): tables and columns go by NUMERIC ID (from browse_data list_table
"Run the saved card through [[qp.card/process-query-for-card]] — the same domain entry the
REST card-query endpoint uses, so the read check, sandboxing, and impersonation all apply —
with `row-limit` injected as the userland constraints. Returns the QP result; surfaces a
- failed run as a teaching error.
+ failed run as a teaching error when [[query-execution/caller-facing-failure?]] accepts it, and
+ as an internal error otherwise.
Binds [[qp.card/*allow-arbitrary-mbql-parameters*]] true so a saved question's declared
MBQL filter-widget parameters run, not just native template tags — the same relaxation the
@@ -575,7 +576,9 @@ Dialect (JSON): tables and columns go by NUMERIC ID (from browse_data list_table
(fn [query info]
(qp (update query :info merge info) nil)))))]
(when-not (= (:status result) :completed)
- (common/throw-teaching-error (query-failure-message (:error result))))
+ (if (query-execution/caller-facing-failure? result)
+ (common/throw-teaching-error (query-failure-message (:error result)))
+ (throw (ex-info (str "Query failed: " (:error result)) {}))))
result))
(defn- saved-question-steering-line
diff --git a/src/metabase/metabot/query_execution.clj b/src/metabase/metabot/query_execution.clj
index 532c4c9dbaf..82b6e04ad90 100644
--- a/src/metabase/metabot/query_execution.clj
+++ b/src/metabase/metabot/query_execution.clj
@@ -5,6 +5,7 @@
[metabase.api.common :as api]
[metabase.driver.settings :as driver.settings]
[metabase.query-processor.core :as qp]
+ [metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
@@ -68,6 +69,18 @@
(finally
(.cancel task false)))))
+(defn caller-facing-failure?
+ "Whether failed QP `result` is about the query or its database, so that the caller may see its error text.
+ The database rejecting the query is one such failure. A failure inside Metabase, such as an app DB lookup, is not:
+ its error text can carry app DB SQL and class names."
+ [{error-type :error_type :as result}]
+ (let [status-code (get-in result [:ex-data :status-code])]
+ (boolean
+ (or (and status-code (<= 400 status-code 499))
+ (qp.error-type/client-error? error-type)
+ (qp.error-type/connection-pool-saturated? error-type)
+ (contains? #{qp.error-type/db qp.error-type/timed-out} error-type)))))
+
(defn- execute!
"Run a serialized MBQL query through the QP with the standard agent userland preparation.
Caps this call's rows at `row-limit` (within the backend's 2000/10000 userland ceilings) and records the run
@@ -116,6 +129,7 @@
{:agent-error? true
:error :query-failed
:query-error (:error result)
+ :caller-facing? (caller-facing-failure? result)
;; The QP's own refusal, as opposed to an error from the database.
:permissions-error? (= :missing-required-permissions (:error_type result))})))
result))
@@ -126,7 +140,8 @@
The `truncated?` flag is true only when more rows exist, so a result that fills the page exactly is complete.
A query failure throws an agent error with the QP's error text in quotes.
Its ex-data is `{:agent-error? true :error :query-failed :query-error <QP error text or nil>}`, with a true
- `:permissions-error?` when the QP refused the query because the current user may not run it.
+ `:permissions-error?` when the QP refused the query because the current user may not run it, and a true
+ `:caller-facing?` when [[caller-facing-failure?]] accepts the failure.
Two options stop a run early, each with an agent error of its own:
- `:request-canceled-chan`, a promise channel that receives a value when the request being served is canceled.
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index fb2d53e09b0..5290b616fdf 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -781,17 +781,31 @@
;;; ---------------------------------------- Query execution ----------------------------------------
+(defn- input-error-data?
+ [{:keys [status-code agent-error? error]}]
+ (or (and status-code (<= 400 status-code 499))
+ (and (nil? status-code) (or agent-error? (tools/pipeline-error-code error)))))
+
(defn database-error?
- "Returns true for database failures that must not be relayed as input errors."
- [e]
- (let [{:keys [status-code agent-error? error]} (ex-data e)
- causes (u/full-exception-chain e)]
- (boolean
- (and (not (and status-code (<= 400 status-code 499)))
- (or (some #(instance? SQLException %) causes)
- (and (not (and (nil? status-code)
- (or agent-error? (tools/pipeline-error-code error))))
- (some #(contains? (ex-data %) :toucan2/context-trace) causes)))))))
+ "Returns true for database failures that must not be relayed as input errors.
+ With `:ignore-status?`, inspect database causes even when an exception carries an explicit client status."
+ ([e]
+ (database-error? e {}))
+ ([e {:keys [ignore-status?]}]
+ (let [{:keys [status-code] :as data} (ex-data e)
+ causes (u/full-exception-chain e)
+ client-error? (and status-code (<= 400 status-code 499))]
+ (boolean
+ (and (or ignore-status? (not client-error?))
+ (or (some #(instance? SQLException %) causes)
+ (if ignore-status?
+ (some (fn [cause]
+ (let [data (ex-data cause)]
+ (and (contains? data :toucan2/context-trace)
+ (not (input-error-data? data)))))
+ causes)
+ (and (not (input-error-data? data))
+ (some #(contains? (ex-data %) :toucan2/context-trace) causes)))))))))
(defn- has-valid-trailing-clause?
"Return whether a stage has a nonempty trailing clause with a valid portable structure."
diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.clj
index 15a01b39084..0d78a539953 100644
--- a/test/metabase/agent_api/api_test.clj
+++ b/test/metabase/agent_api/api_test.clj
@@ -10,8 +10,10 @@
[metabase.agent-api.settings :as agent-api.settings]
[metabase.ai-tracing.log :as ait.log]
[metabase.ai-tracing.settings :as ai-tracing.settings]
+ [metabase.api.macros :as api.macros]
[metabase.api.macros.scope :as api.scope]
[metabase.collections.models.collection :as collection]
+ [metabase.lib-be.db :as lib-be.db]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
@@ -22,6 +24,7 @@
[metabase.metabot.tools :as metabot.tools]
[metabase.metabot.tools.core :as tools.core]
[metabase.metabot.tools.resources :as metabot-resources]
+ [metabase.metabot.tools.search :as metabot-search]
[metabase.oauth-server.test-util :as oauth-server.tu]
[metabase.permissions.core :as perms]
[metabase.permissions.models.data-permissions :as data-perms]
@@ -33,13 +36,15 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.test.http-client :as client]
+ [metabase.test.server.handler :as test.server.handler]
[metabase.util :as u]
[metabase.util.json :as json]
[metabase.util.malli.registry :as mr]
[oidc-provider.util :as oidc.util]
[toucan2.core :as t2])
(:import
- (java.sql SQLException)))
+ (java.sql SQLException)
+ (org.eclipse.jetty.io EofException)))
(set! *warn-on-reflection* true)
@@ -355,26 +360,6 @@
(is (=? {:error (name error) :message expected}
(mt/user-http-request :rasta :post 400 "agent/v2/construct-query" {:query query})))))))
-(deftest database-error-callback-failure-test
- (mt/with-prometheus-system! [_ system]
- (doseq [database-error? [false true]
- delivery [:throw :raise]]
- (let [failure (if database-error? (SQLException. "Database failure") (Exception. "Handler failure"))
- callback-failure (if database-error? (SQLException. "Callback failure") (Exception. "Callback failure"))
- calls (atom 0)
- callback (fn [_] (swap! calls inc) (throw callback-failure))
- handler (#'agent-api.api/wrap-database-errors
- (fn [_request _respond raise]
- (case delivery
- :throw (throw failure)
- :raise (raise failure))))
- initial (mt/metric-value system :metabase-api/unhandled-errors)
- caught (try (handler {} callback callback) (catch Throwable e e))]
- (is (identical? callback-failure caught))
- (is (= 1 @calls))
- (is (= (+ initial (if database-error? 1 0))
- (mt/metric-value system :metabase-api/unhandled-errors)))))))
-
(deftest construct-query-permission-checks-test
(testing "Rejects a first-stage source-table the current user cannot query"
(mt/with-no-data-perms-for-all-users!
@@ -2013,9 +1998,11 @@
(let [resp (mt/user-http-request :crowberto :post 400 "agent/v1/execute-sql"
{:database_id (mt/id)
:sql "SELECT * FROM table_that_does_not_exist"})]
- ;; The streaming pipeline wraps the failure in :via[0]/:status.
- (is (= "failed" (some-> resp :via first :status)))
- (is (string? (some-> resp :via first :error)))))
+ (is (=? {:status "failed"
+ :error #(str/includes? % "TABLE_THAT_DOES_NOT_EXIST")
+ :error_type "invalid-query"}
+ resp))
+ (is (= #{:status :error :error_type} (set (keys resp))))))
(testing "A successful query records a QueryExecution audit row tagged :agent"
;; Bypass-of-userland regression (#1): without `prepare-agent-query` no audit row was
;; written. Verify a row lands and carries the agent context. QueryExecutions are saved in async batches by
@@ -2184,6 +2171,155 @@
(is (every? (fn [stage] (not (contains? stage :qp/stage-had-source-card))) (:stages q))
"internal keys are stripped from every stage"))))
+;;; ------------------------------------------------- Error Responses ---------------------------------------------
+
+(defn- reject-app-db-lookup
+ "Throw an app DB lookup error without leaving the surrounding transaction aborted."
+ [& _]
+ (t2/with-transaction [_conn]
+ (t2/query ["SELECT no_such_column FROM metabase_field"])))
+
+(defn- logs-app-db-error?
+ "Whether captured log `messages` include the app DB error that [[reject-app-db-lookup]] causes."
+ [messages]
+ (some #(re-find #"(?i)no_such_column" (str (:message %) (some-> (:e %) ex-message))) messages))
+
+(deftest unexpected-error-is-generic-test
+ (testing "An unexpected failure is a generic 500 whatever `hide-stacktraces` says, and the log gets the detail"
+ (mt/with-prometheus-system! [_ system]
+ (doseq [[search log-message detail] [[reject-app-db-lookup "Database failure in Agent API" #"(?i)no_such_column"]
+ [(fn [_] (throw (ex-info "Search index is unavailable" {})))
+ "Unhandled error in Agent API request"
+ #"Search index is unavailable"]]
+ hide-stacktraces? [false true]]
+ (mt/with-dynamic-fn-redefs [metabot-search/search search]
+ (mt/with-temporary-setting-values [hide-stacktraces hide-stacktraces?]
+ (mt/with-log-messages-for-level [messages [metabase.agent-api :error]]
+ (let [initial (mt/metric-value system :metabase-api/unhandled-errors)]
+ (is (= {:message "Something went wrong"}
+ (mt/user-http-request :rasta :post 500 "agent/v1/search" {:term_queries ["orders"]})))
+ (is (= (inc initial) (mt/metric-value system :metabase-api/unhandled-errors)))
+ (is (=? [{:message log-message, :e #(re-find detail (ex-message %))}] (messages)))))))))))
+
+(deftest error-response-callback-failure-test
+ (mt/with-prometheus-system! [_ system]
+ (doseq [endpoint-kind [:bare :sync :async]
+ callback-kind [:ordinary :database :disconnect]
+ delivery [:respond :throw :raise]]
+ (let [failure (if (= callback-kind :database) (SQLException. "Database failure") (Exception. "Handler failure"))
+ callback-failure (case callback-kind
+ :ordinary (Exception. "Callback failure")
+ :database (SQLException. "Callback failure")
+ :disconnect (EofException. "Client went away"))
+ calls (atom 0)
+ callback (fn [_] (swap! calls inc) (throw callback-failure))
+ handler (fn [_request respond raise]
+ (case delivery
+ :respond (respond {:status 200 :body "ok"})
+ :throw (throw failure)
+ :raise (raise failure)))
+ endpoint (case endpoint-kind
+ :bare handler
+ :sync (api.macros/endpoint-handler* nil
+ (fn [& _] (handler {} identity #(throw %)))
+ {:async? false})
+ :async (api.macros/endpoint-handler* nil
+ (api.macros/-core-fn-async
+ (fn [respond raise & _] (handler {} respond raise)))
+ {:async? true}))
+ handler (#'agent-api.api/wrap-error-responses endpoint)
+ initial (mt/metric-value system :metabase-api/unhandled-errors)
+ caught (try (handler {} callback callback) (catch Throwable e e))]
+ (is (identical? callback-failure caught))
+ (is (= 1 @calls))
+ (is (= (if (= delivery :respond) initial (inc initial))
+ (mt/metric-value system :metabase-api/unhandled-errors)))))))
+
+(deftest client-error-wrapping-database-error-is-generic-test
+ (testing "A client status does not expose an app DB failure in the cause chain"
+ (mt/with-prometheus-system! [_ system]
+ (doseq [status-code [400 403]]
+ (mt/with-dynamic-fn-redefs [metabot-search/search
+ (fn [& args]
+ (try
+ (apply reject-app-db-lookup args)
+ (catch Throwable e
+ (throw (ex-info "App DB lookup detail"
+ {:status-code status-code :error "internal-app-db"} e)))))]
+ (mt/with-log-messages-for-level [messages [metabase.agent-api :error]]
+ (let [initial (mt/metric-value system :metabase-api/unhandled-errors)]
+ (is (= {:message "Something went wrong"}
+ (mt/user-http-request :rasta :post 500 "agent/v1/search" {:term_queries ["orders"]})))
+ (is (= initial (mt/metric-value system :metabase-api/unhandled-errors)))
+ (is (=? [{:message "Database failure in Agent API"}] (messages)))
+ (is (some #(re-find #"(?i)no_such_column" (ex-message %))
+ (take-while some? (iterate ex-cause (:e (first (messages))))))))))))))
+
+(deftest client-error-wrapping-toucan-failure-is-generic-test
+ (doseq [status-code [400 403]]
+ (let [cause (ex-info "Metadata connection failure" {:toucan2/context-trace []}
+ (IllegalStateException. "Metadata connection unavailable"))]
+ (mt/with-dynamic-fn-redefs [metabot-search/search
+ (fn [& _] (throw (ex-info "Internal metadata detail" {:status-code status-code} cause)))]
+ (mt/with-log-messages-for-level [messages [metabase.agent-api :error]]
+ (is (= {:message "Something went wrong"}
+ (mt/user-http-request :rasta :post 500 "agent/v1/search" {:term_queries ["orders"]})))
+ (is (=? [{:message "Database failure in Agent API"}] (messages))))))))
+
+(deftest client-disconnect-is-not-an-error-test
+ (testing "A client that disconnects before its response is written gets no error response, count or error log"
+ (mt/with-prometheus-system! [_ system]
+ (mt/with-log-messages-for-level [messages [metabase.agent-api :error]]
+ (let [statuses (atom [])
+ request (#'client/build-mock-request {:method :get
+ :url "agent/v1/ping"
+ :credentials (mt/user->credentials :rasta)})
+ initial (mt/metric-value system :metabase-api/unhandled-errors)]
+ (try
+ ((test.server.handler/test-handler) request
+ (fn [response]
+ (swap! statuses conj (:status response))
+ (throw (EofException. "Client went away")))
+ (fn [_e]))
+ (catch EofException _))
+ (is (= 200 (first @statuses)))
+ (is (not-any? #{500} @statuses))
+ (is (= initial (mt/metric-value system :metabase-api/unhandled-errors)))
+ (is (empty? (messages))))))))
+
+(deftest client-error-has-no-stacktrace-test
+ (testing "A 400 keeps its message and error code but no stack trace, whatever `hide-stacktraces` says"
+ (doseq [hide-stacktraces? [false true]]
+ (mt/with-temporary-setting-values [hide-stacktraces hide-stacktraces?]
+ (let [body (mt/user-http-request :rasta :post 400 "agent/v2/construct-query"
+ {:query (orders-query :aggregation [(orders-field-ref "ID")])})]
+ (is (=? {:message string? :error "aggregation-entry-not-aggregation"} body))
+ (is (= #{:message :error} (set (keys body)))))))))
+
+(deftest database-error-while-reading-rows-keeps-its-text-test
+ (testing "A database error raised while rows are read keeps its text, like one raised when the query starts"
+ (let [details (update (:details (mt/db)) :db str ";LAZY_QUERY_EXECUTION=TRUE")
+ sql "SELECT X, 10 / (X - 4) AS Y FROM SYSTEM_RANGE(1, 10)"]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2 :details details}]
+ (is (=? {:status "failed"
+ :error #(str/includes? % "Division by zero")
+ :error_type "invalid-query"}
+ (mt/user-http-request :crowberto :post 400 "agent/v1/execute-sql" {:database_id db-id :sql sql})))))))
+
+(deftest query-run-app-db-failure-is-generic-test
+ (testing "A query run that fails in an app DB lookup is a generic failure, and the log gets the detail"
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (orders-limit-query 5)}]
+ (let [handle (u/encode-base64 (json/encode (lib/prepare-for-serialization (orders-limit-query 5))))]
+ (mt/with-dynamic-fn-redefs [lib-be.db/columns reject-app-db-lookup]
+ (doseq [[endpoint body] [["agent/v1/execute" {:query handle}]
+ ["agent/v2/query" {:query handle}]
+ [(format "agent/v1/question/%d/query" card-id) {}]]]
+ (testing endpoint
+ (mt/with-log-messages-for-level [messages [metabase.agent-api :error]]
+ (is (= {:status "failed" :error "Something went wrong"}
+ (mt/user-http-request :crowberto :post 500 endpoint body)))
+ (is (logs-app-db-error? (messages)))))))))))
+
(deftest construct-query-structural-locations-test
(let [response (mt/user-http-request :rasta :post 400 "agent/v2/construct-query"
{:query (orders-query :filters [{:op ">"} {:op "<"}])})]
diff --git a/test/metabase/driver/h2_test.clj b/test/metabase/driver/h2_test.clj
index a8defb6603d..9de358a5fe4 100644
--- a/test/metabase/driver/h2_test.clj
+++ b/test/metabase/driver/h2_test.clj
@@ -39,6 +39,18 @@
(is (= "INFORMATION_SCHEMA"
(driver.sql/default-schema :h2 database))))))))
+(deftest database-error-while-reading-rows-is-invalid-query-test
+ (mt/test-driver :h2
+ (testing "An error that lazy execution raises while rows are read is :invalid-query, like one raised at the start"
+ (let [details (update (:details (mt/db)) :db str ";LAZY_QUERY_EXECUTION=TRUE")
+ sql "SELECT X, 10 / (X - 4) AS Y FROM SYSTEM_RANGE(1, 10)"]
+ (mt/with-temp [:model/Database database {:engine :h2, :details details}]
+ (mt/with-db database
+ (is (=? {:status :failed
+ :error_type :invalid-query
+ :error #"(?s)Division by zero.*"}
+ (qp/process-query (qp/userland-query (lib/native-query (mt/metadata-provider) sql)))))))))))
+
(deftest ^:parallel connection-hosts-test
(testing "local H2 databases have no network host"
(is (= [] (driver/connection-hosts :h2 {:db "file:./sample.db"})))
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index a92e0f2a43d..cbc79955f18 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -49,6 +49,7 @@
[toucan2.core :as t2])
(:import
(java.io File)
+ (java.sql Connection SQLException SQLTransientConnectionException)
(java.util Properties)
(org.mariadb.jdbc UrlParser)))
@@ -1394,3 +1395,74 @@
(is (= [["CREATE INDEX `by_cat` ON `t` (`category`)"]] stmts))
(is (not (str/includes? (ffirst stmts) "PREPARE")))
(is (not (str/includes? (ffirst stmts) "UNHEX")))))))
+
+(deftest error-while-reading-rows-is-invalid-query-test
+ (mt/test-driver :mysql
+ (testing "An error in the query that the database raises after the first fetch of rows is an invalid query"
+ ;; MariaDB raises this with SQLState HY000, which its JDBC driver throws as SQLTransientConnectionException
+ (is (=? {:status :failed
+ :error_type :invalid-query
+ :error #"(?s).*JSON_TABLE.*"}
+ (qp/process-query
+ (qp/userland-query
+ (lib/native-query
+ (mt/metadata-provider)
+ (str "WITH RECURSIVE seq (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 601) "
+ "SELECT seq.n, jt.a FROM seq STRAIGHT_JOIN JSON_TABLE(IF(seq.n < 601, '{\"a\": 1}', '{}'), '$' "
+ "COLUMNS (a INT PATH '$.a' ERROR ON EMPTY)) AS jt")))))))))
+
+(deftest lock-timeout-while-reading-rows-is-qp-error-test
+ (mt/test-driver :mysql
+ (let [db (mt/db)
+ connection (mt/original-fn #'sql-jdbc.execute/do-with-resolved-connection)
+ row-thunk (mt/original-fn #'sql-jdbc.execute/row-thunk)
+ delivered (atom 0)
+ failure (atom nil)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver/*driver* db {:write? true}
+ (fn [^Connection conn]
+ (.setAutoCommit conn false)
+ (with-open [stmt (.createStatement conn)]
+ (.execute stmt "UPDATE venues SET name = name WHERE id = 100"))
+ (try
+ (mt/with-temporary-setting-values [sql-jdbc-fetch-size 20]
+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/do-with-resolved-connection
+ (fn [driver db options f]
+ (connection driver db options
+ (fn [^Connection c]
+ (let [read-timeout #(-> (jdbc/query {:connection c}
+ ["SELECT @@session.innodb_lock_wait_timeout AS timeout"])
+ first :timeout)
+ timeout (read-timeout)]
+ (try
+ (jdbc/execute! {:connection c} ["SET SESSION innodb_lock_wait_timeout = 1"])
+ (f c)
+ (finally
+ (jdbc/execute! {:connection c}
+ [(str "SET SESSION innodb_lock_wait_timeout = " timeout)])
+ (is (= timeout (read-timeout)))))))))
+ sql-jdbc.execute/row-thunk
+ (fn [& args]
+ (let [next-row (apply row-thunk args)]
+ #(try
+ (let [row (next-row)]
+ (when row (swap! delivered inc))
+ row)
+ (catch SQLException e
+ (reset! failure e)
+ (throw e)))))]
+ (is (=? {:status :failed :error_type :qp}
+ (qp/process-query
+ (qp/userland-query
+ (lib/native-query (mt/metadata-provider)
+ "SELECT id FROM venues ORDER BY id LOCK IN SHARE MODE")))))))
+ (is (pos? @delivered) "The timeout happens during row reads, after the first fetch")
+ (is (= "HY000" (some-> ^SQLException @failure .getSQLState)))
+ (is (= 1205 (some-> ^SQLException @failure .getErrorCode)))
+ (finally (.rollback conn))))))))
+
+(deftest ^:parallel query-error-evidence-test
+ (mt/test-driver :mysql
+ (doseq [[state code expected] [["HY000" 4176 true] ["HY000" 1205 false] [nil 4176 false]]]
+ (is (= expected (sql-jdbc.execute/impl-error-in-query?
+ :mysql (SQLTransientConnectionException. "Row read failed" state code)))))))
diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.clj
index 8c28526ab90..6b39d8ea36d 100644
--- a/test/metabase/driver/sql_jdbc/execute_test.clj
+++ b/test/metabase/driver/sql_jdbc/execute_test.clj
@@ -1,5 +1,6 @@
(ns ^:mb/driver-tests metabase.driver.sql-jdbc.execute-test
(:require
+ [clojure.core.async :as a]
[clojure.test :refer :all]
[malli.error :as me]
[metabase.config.core :as config]
@@ -11,12 +12,15 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
+ [metabase.query-processor.pipeline :as qp.pipeline]
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.util.malli.registry :as mr])
(:import
- (java.sql Connection DatabaseMetaData)
- (javax.sql DataSource)))
+ (java.sql Connection DatabaseMetaData ResultSet SQLException SQLNonTransientConnectionException SQLRecoverableException
+ SQLTimeoutException SQLTransientConnectionException)
+ (javax.sql DataSource)
+ (org.sqlite SQLiteErrorCode SQLiteException)))
(deftest ^:parallel ConnectionOptions-test
(are [options error] (= error
@@ -354,3 +358,68 @@
(reset! cancels 0)
(is (= 4 (count (venues-rows 4))))
(is (pos? @cancels))))))))
+
+(deftest error-while-reading-rows-test
+ (mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc})
+ (let [mp (mt/metadata-provider)
+ query (qp/userland-query (lib/query mp (lib.metadata/table mp (mt/id :venues))))
+ error-type (fn [e]
+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/row-thunk (fn [& _] #(throw e))]
+ (:error_type (qp/process-query query))))]
+ (testing "A database error raised while rows are read is an invalid query, whatever its exception class"
+ (doseq [e [(SQLException. "More than one row" "21000")
+ (SQLException. "Unique constraint violation" "23505")
+ (SQLNonTransientConnectionException. "Unknown column 'a' in 'field list'" "42S22")]]
+ (testing (ex-message e)
+ (is (= :invalid-query (error-type e))))))
+ (testing "A connection failure while closing the result set is a QP error"
+ (let [execute-statement (mt/original-fn #'sql-jdbc.execute/execute-statement-or-prepared-statement!)
+ failure (SQLNonTransientConnectionException. "Connection is broken" "08000")]
+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/row-thunk (fn [& _] #(throw failure))
+ sql-jdbc.execute/execute-statement-or-prepared-statement!
+ (fn [& args]
+ (let [^ResultSet rs (apply execute-statement args)]
+ (reify ResultSet
+ (getMetaData [_] (.getMetaData rs))
+ (close [_]
+ (.close rs)
+ (throw failure)))))]
+ (is (= :qp (:error_type (qp/process-query query)))))))
+ (testing "A failed connection, a canceled or timed-out statement or a transient failure is a QP error"
+ (doseq [e [(SQLException. "Unknown row-read failure")
+ (SQLTransientConnectionException. "Unknown connection failure")
+ (SQLTransientConnectionException. "Lock wait timeout exceeded" "HY000" 1205)
+ (SQLException. "Out of memory" "HY001" 1037)
+ (SQLException. "Query was canceled" nil 3)
+ (SQLException. "Out of memory" "53200")
+ (SQLException. "Lock not available" "55P03")
+ (SQLException. "I/O error" "58030")
+ (SQLException. "Idle transaction timeout" "25P03")
+ (SQLException. "Internal error" "XX000")
+ (SQLException. "Cached plan must not change result type" "0A000")
+ (SQLException. "Object closed" "90007")
+ (SQLiteException. "Database busy" SQLiteErrorCode/SQLITE_BUSY)
+ (SQLiteException. "I/O error" SQLiteErrorCode/SQLITE_IOERR)
+ (SQLiteException. "Disk full" SQLiteErrorCode/SQLITE_FULL)
+ (SQLException. "An I/O error occurred while sending to the backend" "08006")
+ (SQLNonTransientConnectionException. "Connection is broken" "90067")
+ (SQLRecoverableException. "Communications link failure")
+ (SQLTimeoutException. "Statement was canceled")
+ (SQLException. "Deadlock detected" "40P01")
+ (SQLException. "Terminating connection due to administrator command" "57P01")]]
+ (testing (ex-message e)
+ (is (= :qp (error-type e)))))
+ (testing "when the query processor canceled the statement, whatever the driver reports"
+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/wire-up-canceled-chan-to-cancel-Statement! (constantly nil)
+ sql-jdbc.execute/row-thunk
+ (fn [& _]
+ #(do (a/offer! qp.pipeline/*canceled-chan* ::canceled)
+ (throw (SQLException. "Query was canceled" "22012"))))]
+ (is (= :qp (:error_type (qp/process-query query))))))
+ (with-redefs [driver/query-canceled? (constantly true)]
+ (is (= :qp (error-type (SQLException. "Query execution was interrupted" "22012")))))))))
+
+(deftest ^:parallel default-query-error-evidence-test
+ (let [classify (get-method sql-jdbc.execute/impl-error-in-query? :sql-jdbc)]
+ (doseq [state ["21000" "22012" "23505" "42S22"]]
+ (is (true? (classify nil (SQLException. "Query error" state)))))))
diff --git a/test/metabase/driver/sqlite_test.clj b/test/metabase/driver/sqlite_test.clj
index dde2befdfee..2a3df51a46c 100644
--- a/test/metabase/driver/sqlite_test.clj
+++ b/test/metabase/driver/sqlite_test.clj
@@ -2,6 +2,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.sqlite-test]}
metabase.test.data/run-mbql-query {:namespaces [metabase.driver.sqlite-test]}}}}}}
(:require
+ [clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.java.jdbc :as jdbc]
[clojure.test :refer :all]
@@ -9,6 +10,7 @@
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]
@@ -16,12 +18,16 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.serialization :as serdes]
+ [metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.test :as qp]
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.sql ResultSet SQLException)
+ (org.sqlite SQLiteErrorCode)))
(set! *warn-on-reflection* true)
@@ -308,6 +314,45 @@
(mt/test-driver :sqlite
(is (nil? (driver.sql/default-schema :sqlite (mt/db))))))
+(deftest ^:parallel database-error-while-reading-rows-is-invalid-query-test
+ (mt/test-driver :sqlite
+ (testing "A database error raised while rows are read is :invalid-query, like one raised when the query starts"
+ (is (=? {:status :failed
+ :error_type :invalid-query
+ :error #"(?s).*malformed JSON.*"}
+ (qp/process-query
+ (qp/userland-query
+ (lib/native-query (mt/metadata-provider)
+ (str "SELECT json_extract(CASE WHEN ID = 1 THEN '{\"n\":1}' ELSE 'broken' END, '$.n')"
+ " AS N FROM VENUES")))))))))
+
+(deftest canceled-while-reading-rows-is-not-invalid-query-test
+ (mt/test-driver :sqlite
+ (let [query (qp/userland-query
+ (lib/native-query (mt/metadata-provider) "SELECT 1 AS X UNION ALL SELECT 2 AS X"))
+ row-thunk (mt/original-fn #'sql-jdbc.execute/row-thunk)]
+ (doseq [qp-canceled? [false true]]
+ (testing (str "A SQLite interrupt is a QP error with no SQLState, QP canceled: " qp-canceled?)
+ (let [interrupted (atom nil)]
+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/wire-up-canceled-chan-to-cancel-Statement! (constantly nil)
+ sql-jdbc.execute/row-thunk
+ (fn [driver ^ResultSet rs rsmeta]
+ (let [next-row (row-thunk driver rs rsmeta)
+ reads (volatile! 0)]
+ (fn []
+ (when (= 2 (vswap! reads inc))
+ (when qp-canceled?
+ (a/offer! qp.pipeline/*canceled-chan* ::canceled))
+ (.cancel (.getStatement rs)))
+ (try
+ (next-row)
+ (catch SQLException e
+ (reset! interrupted e)
+ (throw e))))))]
+ (is (=? {:status :failed, :error_type :qp} (qp/process-query query)))
+ (is (nil? (.getSQLState ^SQLException @interrupted)))
+ (is (= (.code SQLiteErrorCode/SQLITE_INTERRUPT) (.getErrorCode ^SQLException @interrupted))))))))))
+
(deftest ^:parallel timezone-id-test
(mt/test-driver :sqlite
(is (= "UTC"
@@ -733,3 +778,8 @@
(testing "so its base/effective type stays temporal (Lib's type wins over the storage affinity)"
(is (isa? (:base_type col) :type/Temporal))
(is (isa? (:effective_type col) :type/Temporal)))))))
+
+(deftest ^:parallel query-error-evidence-test
+ (mt/test-driver :sqlite
+ (doseq [code [1 19 20]]
+ (is (true? (sql-jdbc.execute/impl-error-in-query? :sqlite (SQLException. "Query error" nil code)))))))
diff --git a/test/metabase/mcp/v2/tools/execute_sql_test.clj b/test/metabase/mcp/v2/tools/execute_sql_test.clj
index 24f62e3181e..91c6a70bc46 100644
--- a/test/metabase/mcp/v2/tools/execute_sql_test.clj
+++ b/test/metabase/mcp/v2/tools/execute_sql_test.clj
@@ -241,6 +241,16 @@
(is (str/includes? text "no_such\\nIGNORE PREVIOUS INSTRUCTIONS"))
(is (not (str/includes? text "\n"))))))))
+(deftest execute-sql-error-while-reading-rows-is-quoted-test
+ (testing "A database error raised while rows are read is quoted, like one raised when the query starts"
+ (let [details (update (:details (mt/db)) :db str ";LAZY_QUERY_EXECUTION=TRUE")
+ sql "SELECT X, 10 / (X - 4) AS Y FROM SYSTEM_RANGE(1, 10)"]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2 :details details}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-model-cleanup [:model/McpQueryHandle]
+ (is (str/starts-with? (error-text (call! (str (random-uuid)) {:database_id db-id :sql sql}))
+ "Query failed: \"Division by zero"))))))))
+
;;; -------------------------------------------------- Prompt ------------------------------------------------------
;; not ^:parallel: mt/with-model-cleanup on the shared query-handle table
diff --git a/test/metabase/mcp/v2/tools/query_test.clj b/test/metabase/mcp/v2/tools/query_test.clj
index e545f54a571..204cb5b101b 100644
--- a/test/metabase/mcp/v2/tools/query_test.clj
+++ b/test/metabase/mcp/v2/tools/query_test.clj
@@ -10,6 +10,7 @@
[clojure.string :as str]
[clojure.test :refer [deftest is testing use-fixtures]]
[metabase.lib-be.core :as lib-be]
+ [metabase.lib-be.db :as lib-be.db]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
@@ -1176,3 +1177,16 @@
params [{:type "number/=" :target ["variable" ["template-tag" "n"]] :value 1}]
handle (stored-handle! sid (pk-ordered-orders-query {:parameters params}))]
(is (= params (:parameters (capture-qp-query! {:query_handle handle}))))))))
+
+;; not ^:parallel: mt/with-model-cleanup on the shared query-handle table
+(deftest app-db-failure-in-query-run-is-internal-test
+ (testing "A handle run that fails in an app DB lookup is an internal error, without the app DB's SQL"
+ (mt/with-model-cleanup [:model/McpQueryHandle]
+ (let [sid (str (random-uuid))
+ handle (stored-handle! sid (pk-ordered-orders-query nil))]
+ (mt/with-current-user (mt/user->id :rasta)
+ (mt/with-dynamic-fn-redefs [lib-be.db/columns (fn [& _]
+ (t2/with-transaction [_conn]
+ (t2/query ["SELECT no_such_column FROM metabase_field"])))]
+ (is (= {:result {:isError true :content [{:type "text" :text "Internal error"}]}}
+ (call! sid {:query_handle handle})))))))))
diff --git a/test/metabase/mcp/v2/tools/run_saved_question_test.clj b/test/metabase/mcp/v2/tools/run_saved_question_test.clj
index 6d8edcc05b0..abd7bc0a141 100644
--- a/test/metabase/mcp/v2/tools/run_saved_question_test.clj
+++ b/test/metabase/mcp/v2/tools/run_saved_question_test.clj
@@ -8,6 +8,7 @@
(:require
[clojure.string :as str]
[clojure.test :refer [deftest is testing use-fixtures]]
+ [metabase.lib-be.db :as lib-be.db]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.mcp.v2.message :as message]
@@ -20,7 +21,8 @@
[metabase.query-processor.pivot.test-util :as qp.pivot.tu]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
- [metabase.util.json :as json]))
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -413,3 +415,14 @@
(doseq [bad ["0" "-1" "016211" "1.0" "not-an-id"]]
(is (str/includes? (tool-error (call-run-saved-question {:id bad})) "entity_id")
(str "expected " (pr-str bad) " to be rejected"))))))
+
+(deftest ^:parallel app-db-failure-in-query-run-is-internal-test
+ (testing "A run that fails in an app DB lookup is an internal error, without the app DB's SQL"
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/table mp (mt/id :products)))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (mt/with-dynamic-fn-redefs [lib-be.db/columns (fn [& _]
+ (t2/with-transaction [_conn]
+ (t2/query ["SELECT no_such_column FROM metabase_field"])))]
+ (is (= "Internal error" (tool-error (call-run-saved-question {:id card-id}))))))))))
diff --git a/test/metabase/metabot/query_execution_test.clj b/test/metabase/metabot/query_execution_test.clj
index 620bc80bf25..2f1bbb0800f 100644
--- a/test/metabase/metabot/query_execution_test.clj
+++ b/test/metabase/metabot/query_execution_test.clj
@@ -62,6 +62,21 @@
(mt/with-dynamic-fn-redefs [qp/process-query (constantly {:status :failed, :error "Column FOO not found"})]
(is (=? {:error :query-failed, :permissions-error? false} (failure))))))))
+(deftest caller-facing-failure-test
+ (testing "a failure of the query or its database is the caller's to see, and a failure inside Metabase is not"
+ (doseq [[expected failure] [[true {:error_type :invalid-query}]
+ [true {:error_type :missing-required-permissions}]
+ [true {:error_type :connection-pool-checkout-timeout}]
+ [true {:error_type :db}]
+ [true {:error_type :timed-out}]
+ [true {:ex-data {:status-code 404}}]
+ [false {:error_type :qp}]
+ [false {:error_type :driver}]
+ [false {:ex-data {:status-code 500}}]
+ [false {}]]]
+ (is (= expected (query-execution/caller-facing-failure? (assoc failure :status :failed :error "Failed")))
+ (pr-str failure)))))
+
(defn- received
"The value `chan` delivers within 5 seconds, or nil."
[chan]
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index a1adedc1fb3..45a18577ec4 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -316,6 +316,20 @@
(construct/database-error?
(ex-info "Failure" (assoc data :status-code status :toucan2/context-trace [])))))))
+(deftest ^:parallel database-errors-with-client-status-test
+ (doseq [[label cause] (metadata-lookup-failures)
+ status-code [400 403]]
+ (testing label
+ (let [e (ex-info "Lookup failed" {:status-code status-code} cause)]
+ (is (false? (construct/database-error? e)))
+ (is (true? (construct/database-error? e {:ignore-status? true}))))))
+ (doseq [data [{:status-code 400} {:agent-error? true} {:error :unknown-field}]]
+ (is (false? (construct/database-error?
+ (ex-info "Invalid input" {:status-code 400}
+ (ex-info "Validation detail" (assoc data :toucan2/context-trace [])))
+ {:ignore-status? true}))))
+ (is (false? (construct/database-error? (ex-info "Bad input" {:status-code 400}) {:ignore-status? true}))))
+
(deftest declared-errors-in-toucan-scopes-are-recoverable-test
(with-mp-and-stubs!
(fn []