Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve error handling in query responses (#82531)

Nathan Voxlandf062ce0832a6
diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.clj
index b27d7894ae9..2c11c0fe7be 100644
--- a/src/metabase/api_routes/routes.clj
+++ b/src/metabase/api_routes/routes.clj
@@ -58,6 +58,7 @@
[metabase.revisions.api]
[metabase.search.api]
[metabase.segments.rest.api]
+ [metabase.server.streaming-response :as streaming-response]
[metabase.session.api]
[metabase.settings-rest.api]
[metabase.setup-rest.api]
@@ -155,9 +156,29 @@
(cond-> x
(simple-symbol? x) api.macros/ns-handler))
+(defn- sanitize-streaming-errors
+ "Middleware that reduces any error written by a streaming response body to
+ [[metabase.public-sharing-rest.api/error-response]]. The public and embedding route wrappers catch and genericize
+ exceptions thrown by the handler, but a streaming body runs after the handler has returned, so a query failure --
+ or an exception attaching the Card's query as `ex-data` -- would otherwise reach an unauthenticated client as-is.
+
+ This lives here rather than next to those wrappers in [[metabase.api.routes.common]] because the `api` module can't
+ depend on `server` (which depends on the query processor, which depends on `api`)."
+ [handler]
+ (fn [request respond raise]
+ (handler request
+ (fn [response]
+ (respond (streaming-response/with-error-response-fn
+ response
+ metabase.public-sharing-rest.api/error-response)))
+ raise)))
+
+(def ^:private ^{:arglists '([handler])} +sanitize-streaming-errors
+ (routes.common/wrap-middleware-for-open-api-spec-generation sanitize-streaming-errors))
+
(defn- +auth [handler] (routes.common/+auth (->handler handler)))
-(defn- +message-only-exceptions [handler] (routes.common/+message-only-exceptions (->handler handler)))
-(defn- +public-exceptions [handler] (routes.common/+public-exceptions (->handler handler)))
+(defn- +message-only-exceptions [handler] (routes.common/+message-only-exceptions (+sanitize-streaming-errors (->handler handler))))
+(defn- +public-exceptions [handler] (routes.common/+public-exceptions (+sanitize-streaming-errors (->handler handler))))
(declare routes)
diff --git a/src/metabase/public_sharing_rest/api.clj b/src/metabase/public_sharing_rest/api.clj
index f8c8ac062fa..f2938a868b9 100644
--- a/src/metabase/public_sharing_rest/api.clj
+++ b/src/metabase/public_sharing_rest/api.clj
@@ -157,15 +157,25 @@
[:json_query :parameters]
:status]))
-(defmethod transform-qp-result :failed
- [{error-type :error_type, :as results}]
- ;; if the query failed instead, unless the error type is specified and is EXPLICITLY allowed to be shown for embeds,
- ;; instead of returning anything about the query just return a generic error message
+(defn error-response
+ "Reduce a query `error` -- the QP's formatted error response, or the `Throwable->map` of an exception that escaped
+ it -- to what public and embedded endpoints are allowed to return: the status and error type, and a generic message
+ in place of the original unless the error type is EXPLICITLY allowed to be shown in embeds. Nothing about the query
+ itself gets through.
+
+ [[metabase.api-routes.routes]] applies this to every error written by a streaming response under the public and
+ embedding routes."
+ [{error-type :error_type, :as error}]
(merge
- (select-keys results [:status :error :error_type])
+ {:status :failed}
+ (select-keys error [:status :error :error_type])
(when-not (qp.error-type/show-in-embeds? error-type)
{:error (tru "An error occurred while running the query.")})))
+(defmethod transform-qp-result :failed
+ [results]
+ (error-response results))
+
(defn- process-query-for-card-with-id-run-fn
"Create the `:make-run` function used for [[process-query-for-card-with-id]] and [[process-query-for-dashcard]]."
[qp export-format]
diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.clj
index 738e54c70a4..ff29ee60045 100644
--- a/src/metabase/query_processor/api.clj
+++ b/src/metabase/query_processor/api.clj
@@ -256,8 +256,7 @@
(update :middleware select-keys [:js-int-to-string? :ignore-cached-results?]))
:constraints (qp.constraints/default-query-constraints)
:info info)
- rff)
- query)))
+ rff))))
(defn- parameter-field-values
[field-ids query]
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index f503f28d975..ad85bf39289 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -29,6 +29,7 @@
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.metadata :as qp.metadata]
[metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
+ [metabase.query-processor.middleware.catch-exceptions :as qp.catch-exceptions]
[metabase.query-processor.middleware.drop-fields-in-summaries :as qp.drop-fields-in-summaries]
[metabase.query-processor.middleware.nest-for-pivot :as qp.nest-for-pivot]
[metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize]
@@ -800,6 +801,23 @@
(throw t)
(:outcome primary-outcome))))
+(mu/defn- run-pivot-query*
+ "Impl for [[run-pivot-query]]: pick the pivot implementation for `query` and run it through `rff`."
+ [query :- ::qp.schema/any-query
+ rff :- ::qp.schema/rff]
+ (let [query (-> query
+ qp.middleware.normalize/normalize-preprocessing-middleware
+ lib/prepare-after-deserialization)
+ db (query-database query)
+ nativable? (native-path-applicable? db query)
+ use-native? (and nativable? (qp.settings/use-native-pivot-tables))
+ primary (if use-native? run-native-pivot-query run-pivot-query-multi)
+ secondary (if use-native? run-pivot-query-multi run-native-pivot-query)]
+ (binding [qp.pipeline/*pivot?* true]
+ (if (and nativable? (pivot-parity-enabled?))
+ (run-with-parity-check primary secondary query rff use-native?)
+ (primary query rff)))))
+
(mu/defn run-pivot-query
"Run the pivot `query` through `rff`.
@@ -812,6 +830,9 @@
rff, secondary via the default rff for comparison) and disagreement is reported via
[[*on-parity-mismatch*]]. Parity checking is on by default in clojure.test tests.
+ A query with `:info` is run as a userland query, so any error is
+ caught and returned as a formatted error response rather than thrown.
+
Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself."
([query :- ::qp.schema/any-query]
(run-pivot-query query nil))
@@ -824,15 +845,12 @@
;; run-pivot-query, so binding it here from the query's :info map would be
;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.
(qp.setup/with-qp-setup [query query]
- (let [query (-> query
- qp.middleware.normalize/normalize-preprocessing-middleware
- lib/prepare-after-deserialization)
- db (query-database query)
- nativable? (native-path-applicable? db query)
- use-native? (and nativable? (qp.settings/use-native-pivot-tables))
- primary (if use-native? run-native-pivot-query run-pivot-query-multi)
- secondary (if use-native? run-pivot-query-multi run-native-pivot-query)]
- (binding [qp.pipeline/*pivot?* true]
- (if (and nativable? (pivot-parity-enabled?))
- (run-with-parity-check primary secondary query rff use-native?)
- (primary query rff)))))))
+ (let [query (cond-> query
+ (seq (:info query)) qp/userland-query)
+ rff (or rff qp.reducible/default-rff)
+ ;; Everything between here and the first `qp/process-query`
+ ;; runs outside the QP's own middleware, so wrap it in the same exception-catching middleware
+ ;; `qp/process-query` applies to userland queries.
+ ;; No-op for non-userland queries.
+ qp (qp.catch-exceptions/catch-exceptions run-pivot-query*)]
+ (qp query rff)))))
diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.clj
index 3fce2b9c139..2d33fa96255 100644
--- a/src/metabase/server/streaming_response.clj
+++ b/src/metabase/server/streaming_response.clj
@@ -42,7 +42,22 @@
(defn- format-exception [e]
(cond-> (Throwable->map e)
- (server.settings/hide-stacktraces) (dissoc :via :trace)))
+ ;; `:data` is the ex-data, which for QP exceptions routinely carries the whole query; drop it along with the
+ ;; stacktrace and cause chain, as the (non-streaming) exception middleware does.
+ (server.settings/hide-stacktraces) (dissoc :via :trace :data)))
+
+;; Implemented by the output stream handed to a streaming response body, so that [[write-error!]] -- called from the
+;; body with nothing but that stream -- can find the response's `:error-response-fn`. A function applied to any error
+;; body (a formatted error map, or the `Throwable->map` of an exception) right before it is written to the client;
+;; whatever it returns is what gets written. See [[with-error-response-fn]].
+(definterface ErrorResponseFnProvider
+ (errorResponseFn []))
+
+(defn- error-response-fn
+ "The `:error-response-fn` carried by `os`, if it is the output stream handed to a streaming response body."
+ [os]
+ (when (instance? ErrorResponseFnProvider os)
+ (.errorResponseFn ^ErrorResponseFnProvider os)))
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *response*
@@ -121,16 +136,18 @@
(log/errorf "Async context already completed, cannot write error to client: %s"
(if (map? obj) (or (:message obj) (:error obj) (:status obj)) obj))))
-(defn- sanitize-error-obj [obj export-format]
- (-> (if (not= :api export-format)
- (walk/prewalk (fn [x]
- (if (map? x)
- (apply dissoc x [:json_query :preprocessed])
- x))
- obj)
- obj)
- (dissoc :export-format)
- (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via))))
+(defn- sanitize-error-obj [obj export-format response-fn]
+ (let [obj (cond-> obj
+ response-fn response-fn)]
+ (-> (if (not= :api export-format)
+ (walk/prewalk (fn [x]
+ (if (map? x)
+ (apply dissoc x [:json_query :preprocessed])
+ x))
+ obj)
+ obj)
+ (dissoc :export-format)
+ (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via)))))
(defn- abort-connection!
"Abort the underlying Jetty connection for an *already-committed* streaming response, so it
@@ -161,7 +178,7 @@
(with-open [os os]
(log/trace "write-error!")
(try
- (let [obj (sanitize-error-obj obj export-format)]
+ (let [obj (sanitize-error-obj obj export-format (error-response-fn os))]
(with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]
(json/encode-to obj writer {})))
(catch EofException _)
@@ -234,10 +251,11 @@
(.cancel fut true)))))))
(defn- do-f-async
- "Runs `f` asynchronously on `executor` (the shared streaming response `thread-pool` when nil), returning
- immediately. When `f` finishes, completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean`
+ "Runs `f` asynchronously on the `:executor` in `options` (the shared streaming response `thread-pool` when nil),
+ returning immediately. When `f` finishes, completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean`
used to coordinate with Jetty's timeout/error callbacks so that only one path calls `.complete`."
- [^AsyncContext async-context response ^Request request f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed? executor]
+ [^AsyncContext async-context response ^Request request f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?
+ {:keys [executor], :as _options}]
{:pre [(some? os)]}
(let [task (^:once fn* []
(binding [*response* response
@@ -280,9 +298,11 @@
(defn- delay-output-stream
"An OutputStream proxy that fetches the actual output stream by dereffing a delay (or other dereffable) before first
- use."
- [dlay]
- (proxy [OutputStream] []
+ use. It also carries the response's `error-response-fn` (possibly nil) for [[write-error!]]."
+ [dlay error-response-fn]
+ (proxy [OutputStream ErrorResponseFnProvider] []
+ (errorResponseFn []
+ error-response-fn)
(close []
(.close ^OutputStream @dlay))
(flush []
@@ -379,7 +399,7 @@
(defn- respond
[{:keys [^HttpServletResponse response ^AsyncContext async-context request-map response-map request]}
- f {:keys [content-type status headers executor], :as _options} finished-chan]
+ f {:keys [content-type status headers], :as options} finished-chan]
(let [canceled-chan (a/promise-chan)
completed? (AtomicBoolean. false)]
(.addListener async-context
@@ -416,9 +436,9 @@
gzip? (assoc "Content-Encoding" "gzip"))]
(#'servlet/set-headers response headers)
(let [output-stream-delay (output-stream-delay gzip? response)
- delay-os (delay-output-stream output-stream-delay)]
+ delay-os (delay-output-stream output-stream-delay (:error-response-fn options))]
(start-async-cancel-loop! request finished-chan canceled-chan)
- (do-f-async async-context response request f delay-os finished-chan canceled-chan completed? executor)))
+ (do-f-async async-context response request f delay-os finished-chan canceled-chan completed? options)))
(catch Throwable e
(log/errorf "Unexpected exception in do-f-async: %s" (ex-message e))
(try
@@ -500,13 +520,39 @@
which is a small fixed pool: responses that block for the life of a client connection (rather than for the life
of a query) must supply their own executor so they cannot exhaust it. The supplied executor's futures must
support real interruption (`Future.cancel(true)`) — the hung-request escalation interrupts the worker, and a
- ForkJoinPool-backed executor silently ignores it."
+ ForkJoinPool-backed executor silently ignores it.
+ * `:error-response-fn` -- optional function applied to any error body right before [[write-error!]] writes it to the
+ client. Usually added after the fact with [[with-error-response-fn]]."
{:style/indent 2, :arglists '([options [os-binding canceled-chan-binding] & body])}
[options [os-binding canceled-chan-binding :as bindings] & body]
{:pre [(= (count bindings) 2)]}
`(-streaming-response (bound-fn [~(vary-meta os-binding assoc :tag 'java.io.OutputStream) ~canceled-chan-binding] ~@body)
~options))
+(defn with-error-response-fn
+ "Make a streaming `response` pass any error its body writes -- a formatted error map, or the `Throwable->map` of an
+ exception -- through `error-response-fn` first, writing whatever that returns instead.
+
+ For route middleware sanitizing errors for public and embedded endpoints: their `try`/`catch` around the handler
+ cannot see these errors, because a streaming response body runs after the handler has returned.
+
+ `response` may be a [[StreamingResponse]] or a Ring response map whose `:body` is one; anything else is returned
+ unchanged."
+ [response error-response-fn]
+ (cond
+ (instance? StreamingResponse response)
+ (let [^StreamingResponse response response]
+ (->StreamingResponse (.f response)
+ (assoc (.options response) :error-response-fn error-response-fn)
+ (.donechan response)))
+
+ (and (map? response)
+ (instance? StreamingResponse (:body response)))
+ (update response :body with-error-response-fn error-response-fn)
+
+ :else
+ response))
+
;;;; Malli schema for StreamingResponse
(defn streaming-response-schema
diff --git a/test/metabase/embedding_rest/api/embed_test.clj b/test/metabase/embedding_rest/api/embed_test.clj
index b0992847b4f..565ac690c9e 100644
--- a/test/metabase/embedding_rest/api/embed_test.clj
+++ b/test/metabase/embedding_rest/api/embed_test.clj
@@ -18,6 +18,7 @@
[metabase.parameters.custom-values :as custom-values]
[metabase.public-sharing-rest.api-test :as public-test]
[metabase.queries-rest.api.card-test :as api.card-test]
+ [metabase.query-processor.card :as qp.card]
[metabase.query-processor.middleware.constraints :as qp.constraints]
[metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]
[metabase.query-processor.pivot.test-util :as api.pivots]
@@ -1672,6 +1673,38 @@
(is (= "Message seems corrupt or manipulated"
(client/client :get 400 (with-new-secret-key! (pivot-card-query-url card ""))))))))))))
+(deftest embed-pivot-card-error-does-not-leak-query-test
+ (testing "GET /api/embed/pivot/card/:token/query"
+ (testing "an error raised while building the pivot sub-queries must not leak the Card's query or a stacktrace"
+ (with-embedding-enabled-and-new-secret-key!
+ (with-temp-card [card {:enable_embedding true
+ :name "EMBED ERROR LEAK CARD NAME CANARY"
+ :display :pivot
+ :dataset_query {:database (mt/id)
+ :type :native
+ :native {:query "SELECT * FROM no_such_table -- EMBED_ERROR_LEAK_SQL_CANARY"}}}]
+ (let [{:keys [status body]} (client/client-full-response :get (pivot-card-query-url card ""))
+ body-str (pr-str body)]
+ (is (= 500 status))
+ (is (= {:status "failed", :error "An error occurred while running the query.", :error_type "qp"}
+ body))
+ (is (not (str/includes? body-str "CANARY")))
+ (is (not (str/includes? body-str ":trace")))))))))
+
+(deftest embed-card-query-exception-outside-qp-does-not-leak-test
+ (testing "GET /api/embed/card/:token/query"
+ (testing "an exception that escapes the QP entirely is still reduced to the generic embedding error"
+ (with-embedding-enabled-and-new-secret-key!
+ (with-temp-card [card {:enable_embedding true, :name "EMBED ERROR LEAK CARD NAME CANARY"}]
+ (mt/with-dynamic-fn-redefs [qp.card/process-query-for-card-default-qp
+ (fn [query _rff]
+ (throw (ex-info "Boom EMBED_ERROR_LEAK_SQL_CANARY" {:query query})))]
+ (let [{:keys [status body]} (client/client-full-response :get (card-query-url card ""))]
+ (is (= 500 status))
+ (is (= {:status "failed", :error "An error occurred while running the query."}
+ body))
+ (is (not (str/includes? (pr-str body) "CANARY"))))))))))
+
(defn- pivot-dashcard-url
([dashcard] (pivot-dashcard-url dashcard (:dashboard_id dashcard)))
([dashcard dashboard-id & [additional-token-keys]]
diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.clj
index 713ddd17810..5f66c691dc3 100644
--- a/test/metabase/public_sharing_rest/api_test.clj
+++ b/test/metabase/public_sharing_rest/api_test.clj
@@ -21,6 +21,7 @@
[metabase.public-sharing-rest.api :as api.public]
[metabase.public-sharing.core :as public-sharing]
[metabase.queries-rest.api.card-test :as api.card-test]
+ [metabase.query-processor.card :as qp.card]
[metabase.query-processor.card-test :as qp.card-test]
[metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]
[metabase.query-processor.pivot.test-util :as api.pivots]
@@ -2058,6 +2059,104 @@
(is (= "Not found."
(client/client :get 404 (dashcard-url dash card dashcard)))))))))))))
+(def ^:private error-leak-sql-canary "ERROR_LEAK_SQL_CANARY")
+(def ^:private error-leak-card-name-canary "ERROR LEAK CARD NAME CANARY")
+
+(defn- date-param-native-card
+ "A healthy native Card with a date field filter, so a caller-supplied parameter value can drive it to an error without
+ the Card itself being broken."
+ [display]
+ {:name error-leak-card-name-canary
+ :display display
+ :dataset_query {:database (mt/id)
+ :type :native
+ :native {:query (str "SELECT COUNT(*) AS N FROM ORDERS WHERE {{d}} -- " error-leak-sql-canary)
+ :template-tags {"d" {:id "d"
+ :name "d"
+ :display-name "D"
+ :type :dimension
+ :widget-type :date/all-options
+ :dimension [:field (mt/id :orders :created_at) nil]}}}}
+ :parameters [{:id "d"
+ :type :date/all-options
+ :name "D"
+ :slug "d"
+ :target [:dimension [:template-tag "d"]]}]})
+
+(def ^:private unparseable-date-param
+ "A parameter value that passes endpoint validation but blows up while the query is being built."
+ (json/encode [{:id "d"
+ :type "date/all-options"
+ :target ["dimension" ["template-tag" "d"]]
+ :value "NOT-A-DATE"}]))
+
+(defn- assert-generic-query-error
+ "Assert that `response` (from [[client/client-full-response]]) is the generic public-endpoint failure body and that
+ nothing about the Card leaked into it."
+ [{:keys [status body]}]
+ (let [body-str (pr-str body)]
+ (testing "the query genuinely failed"
+ (is (contains? #{400 500} status)))
+ (testing "the body is the generic failed-query shape"
+ (is (=? {:status "failed"
+ :error string?}
+ body))
+ (is (set/subset? (set (keys body)) #{:status :error :error_type})))
+ (testing "the Card's SQL, name, and a stacktrace must not reach an unauthenticated caller"
+ (is (not (str/includes? body-str error-leak-sql-canary)))
+ (is (not (str/includes? body-str error-leak-card-name-canary)))
+ (is (not (str/includes? body-str ":trace")))
+ (is (not (str/includes? body-str ":via"))))))
+
+(deftest public-pivot-card-error-does-not-leak-query-test
+ (testing "GET /api/public/pivot/card/:uuid/query"
+ (testing "an error raised while building the pivot sub-queries must not leak the Card's query or a stacktrace"
+ (mt/dataset test-data
+ (mt/with-temporary-setting-values [enable-public-sharing true]
+ (with-temp-public-card [{uuid :public_uuid} (date-param-native-card :pivot)]
+ (let [url (format "public/pivot/card/%s/query" uuid)]
+ (testing "sanity check: the Card is healthy without the bad parameter"
+ (is (= 202 (:status (client/client-full-response :get url)))))
+ (assert-generic-query-error
+ (client/client-full-response :get url :parameters unparseable-date-param)))))))))
+
+(deftest public-card-with-pivot-display-error-does-not-leak-query-test
+ (testing "GET /api/public/card/:uuid/query"
+ (testing "a Card with :display :pivot takes the pivot path on its ordinary public link too"
+ (mt/dataset test-data
+ (mt/with-temporary-setting-values [enable-public-sharing true]
+ (with-temp-public-card [{uuid :public_uuid} (date-param-native-card :pivot)]
+ (assert-generic-query-error
+ (client/client-full-response :get (format "public/card/%s/query" uuid)
+ :parameters unparseable-date-param))))))))
+
+(deftest public-pivot-dashcard-error-does-not-leak-query-test
+ (testing "GET /api/public/pivot/dashboard/:uuid/dashcard/:dashcard-id/card/:card-id"
+ (testing "an error raised before the QP runs must not leak the query of a Card that is not itself public"
+ (mt/with-temporary-setting-values [enable-public-sharing true]
+ (with-temp-public-dashboard [dash]
+ (mt/with-temp [:model/Card card {:name error-leak-card-name-canary
+ :display :pivot
+ :dataset_query {:database (mt/id)
+ :type :native
+ :native {:query (str "SELECT * FROM no_such_table -- "
+ error-leak-sql-canary)}}}]
+ (let [dashcard (add-card-to-dashboard! card dash)]
+ (is (nil? (:public_uuid card)))
+ (assert-generic-query-error
+ (client/client-full-response :get (pivot-dashcard-url dash card dashcard))))))))))
+
+(deftest public-card-query-exception-outside-qp-does-not-leak-test
+ (testing "GET /api/public/card/:uuid/query"
+ (testing "an exception that escapes the QP entirely (thrown outside its error-handling middleware) is still sanitized"
+ (mt/with-temporary-setting-values [enable-public-sharing true]
+ (with-temp-public-card [{uuid :public_uuid} {:name error-leak-card-name-canary}]
+ (mt/with-dynamic-fn-redefs [qp.card/process-query-for-card-default-qp
+ (fn [query _rff]
+ (throw (ex-info (str "Boom " error-leak-sql-canary) {:query query})))]
+ (assert-generic-query-error
+ (client/client-full-response :get (format "public/card/%s/query" uuid)))))))))
+
;;; ------------------------- POST /api/public/dashboard/:dashboard-uuid/dashcard/:uuid/execute ------------------------------
(deftest execute-public-dashcard-action-test
diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.clj
index 8729de9eb0b..53b740b5a98 100644
--- a/test/metabase/query_processor/api_test.clj
+++ b/test/metabase/query_processor/api_test.clj
@@ -769,6 +769,23 @@
(is (= ["MD" "Twitter" nil 4 16 62] (nth rows 1000)))
(is (= [nil nil nil 7 18760 69540] (last rows)))))))))
+(deftest ^:parallel pivot-dataset-error-response-test
+ (mt/dataset test-data
+ (testing "POST /api/dataset/pivot"
+ (testing "a failed query returns the usual formatted error response, not an empty body"
+ (doseq [[cause query] {"an out-of-range pivot row index"
+ (assoc (qp.pivot.test-util/pivot-query) :pivot_rows [0 1 2 3])
+
+ "a reference to a nonexistent column"
+ (assoc-in (qp.pivot.test-util/pivot-query) [:query :filter]
+ [:= [:field Integer/MAX_VALUE nil] 1])}]
+ (testing cause
+ (let [{:keys [status body]} (mt/user-http-request-full-response :crowberto :post "dataset/pivot" query)]
+ (is (contains? #{400 500} status))
+ (is (=? {:status "failed"
+ :error string?}
+ body)))))))))
+
(deftest ^:parallel pivot-dataset-row-totals-disabled-test
(mt/test-drivers (qp.pivot.test-util/applicable-drivers)
(mt/dataset test-data
diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.clj
index 1eb88887689..0bee9e11807 100644
--- a/test/metabase/query_processor/pivot_test.clj
+++ b/test/metabase/query_processor/pivot_test.clj
@@ -145,6 +145,25 @@
(mt/rows (qp.pivot/run-pivot-query (set/rename-keys (test-query)
{:pivot-rows :pivot_rows, :pivot-cols :pivot_cols})))))))
+(deftest ^:parallel error-generating-pivot-queries-test
+ (testing "An error thrown while generating the pivot sub-queries, i.e. before the QP proper (and its error-handling middleware) runs"
+ ;; the native path silently drops out-of-range pivot indexes rather than throwing, so parity would disagree here
+ (qp.pivot.test-util/without-pivot-parity-check
+ (let [query (assoc (test-query) :pivot-rows [0 1 2 3])]
+ (testing "is thrown as-is for a non-userland query"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Error generating pivot queries"
+ (qp.pivot/run-pivot-query query))))
+ (testing "is returned in the usual formatted error shape for a userland query, like any other QP error"
+ (doseq [userland-query [(qp.core/userland-query query)
+ (assoc query :info {:context :ad-hoc})]]
+ (is (=? {:status :failed
+ :error #"Invalid pivot-rows: specified breakout at index 3, but we only have 3 breakouts"
+ :error_type :invalid-query
+ :json_query map?}
+ (qp.pivot/run-pivot-query userland-query)))))))))
+
(deftest ^:parallel generate-queries-test
(mt/test-drivers (qp.pivot.test-util/applicable-drivers)
(let [metadata-provider (mt/metadata-provider)
diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.clj
index 6b3d8782a96..d83043a7391 100644
--- a/test/metabase/server/streaming_response_test.clj
+++ b/test/metabase/server/streaming_response_test.clj
@@ -413,8 +413,8 @@
"Response should not contain :trace key")
(is (not (contains? error-response :via))
"Response should not contain :via key")
- (is (= "test-value" (get-in error-response [:data :custom-data]))
- "Response should include custom data from ex-info")))))))
+ (is (not (contains? error-response :data))
+ "Response should not contain the ex-data either, like the regular exception middleware")))))))
(deftest write-error-nested-exception-with-stacktraces-disabled-test
(testing "write-error! includes nested exception details when hide-stacktraces is false"
@@ -484,6 +484,120 @@
(is (= "preserve-me" (:custom-data error-response))
"Response should still include custom data")))))))
+(defn- error-response-fn-for-test [error]
+ {:status :failed, :error "generic message", :original-keys (sort (keys error))})
+
+(defn- error-carrying-output-stream
+ "An output stream like the one a streaming response body is handed, carrying `error-response-fn`, that writes to
+ `os`."
+ ^java.io.OutputStream [os error-response-fn]
+ (#'streaming-response/delay-output-stream (delay os) error-response-fn))
+
+(deftest write-error-applies-error-response-fn-test
+ (testing "when the output stream carries an :error-response-fn, write-error! writes what it returns instead of the error itself"
+ (mt/with-temporary-setting-values [hide-stacktraces false]
+ (testing "for a raw exception (formatted to a map first, so the fn always sees a map)"
+ (let [baos (java.io.ByteArrayOutputStream.)]
+ (#'streaming-response/write-error! (error-carrying-output-stream baos error-response-fn-for-test)
+ (ex-info "SENSITIVE MESSAGE" {:query "SENSITIVE QUERY"})
+ :api)
+ (let [output (String. (.toByteArray baos) "UTF-8")]
+ (is (= {:status "failed"
+ :error "generic message"
+ :original-keys ["cause" "data" "trace" "via"]}
+ (json/decode output true)))
+ (is (not (re-find #"SENSITIVE" output))))))
+ (testing "for an already-formatted error map"
+ (let [baos (java.io.ByteArrayOutputStream.)]
+ (#'streaming-response/write-error! (error-carrying-output-stream baos error-response-fn-for-test)
+ {:status :failed, :error "SENSITIVE", :json_query "SENSITIVE"}
+ :api)
+ (let [output (String. (.toByteArray baos) "UTF-8")]
+ (is (= {:status "failed"
+ :error "generic message"
+ :original-keys ["error" "json_query" "status"]}
+ (json/decode output true)))
+ (is (not (re-find #"SENSITIVE" output))))))
+ (testing "an output stream without one writes the error unchanged"
+ (let [baos (java.io.ByteArrayOutputStream.)]
+ (#'streaming-response/write-error! (error-carrying-output-stream baos nil)
+ {:status :failed, :error "visible"}
+ :api)
+ (is (= {:status "failed", :error "visible"}
+ (json/decode (String. (.toByteArray baos) "UTF-8") true))))))))
+
+(defn- respond-and-capture
+ "Respond with `streaming-response` against a fake servlet response. Returns the status code and the body written."
+ [streaming-response]
+ (with-open [os (java.io.ByteArrayOutputStream.)]
+ (let [complete-promise (promise)
+ status (atom nil)]
+ (server.protocols/respond streaming-response
+ {:response (reify HttpServletResponse
+ (isCommitted [_] false)
+ (setStatus [_ new-status] (reset! status new-status))
+ (setContentType [_ _])
+ (setHeader [_ _ _])
+ (getOutputStream [_]
+ (proxy [ServletOutputStream] []
+ (write
+ ([byytes]
+ (.write os ^bytes byytes))
+ ([byytes offset length]
+ (.write os ^bytes byytes offset length))))))
+ :async-context (reify AsyncContext
+ (addListener [_ _])
+ (complete [_]
+ (deliver complete-promise true)))})
+ (is (true? (deref complete-promise 1000 ::timed-out)))
+ {:status @status
+ :body (String. (.toByteArray os) "UTF-8")})))
+
+(deftest with-error-response-fn-test
+ (testing "with-error-response-fn makes a streaming response pass any error its body produces through the fn"
+ (mt/with-temporary-setting-values [hide-stacktraces false]
+ (testing "an exception thrown by the body, on the worker thread"
+ (let [{:keys [status body]} (respond-and-capture
+ (streaming-response/with-error-response-fn
+ (streaming-response/streaming-response {:content-type "application/json"} [_os _]
+ (throw (ex-info "SENSITIVE MESSAGE" {:query "SENSITIVE QUERY"})))
+ error-response-fn-for-test))]
+ (is (= 500 status))
+ (is (= {:status "failed"
+ :error "generic message"
+ :original-keys ["cause" "data" "trace" "via"]}
+ (json/decode body true)))
+ (is (not (re-find #"SENSITIVE" body)))))
+ (testing "an error map the body writes itself with write-error!, as the query processor does"
+ (let [{:keys [status body]} (respond-and-capture
+ (streaming-response/with-error-response-fn
+ (streaming-response/streaming-response {:content-type "application/json"} [os _]
+ (streaming-response/write-error! os {:status :failed, :error "SENSITIVE", :json_query "SENSITIVE"} :api 400))
+ error-response-fn-for-test))]
+ (is (= 400 status))
+ (is (= {:status "failed"
+ :error "generic message"
+ :original-keys ["error" "json_query" "status"]}
+ (json/decode body true)))
+ (is (not (re-find #"SENSITIVE" body))))))))
+
+(deftest with-error-response-fn-response-shapes-test
+ (let [sr (streaming-response/streaming-response {:content-type "application/json"} [_os _])]
+ (testing "a streaming response gets the fn added to its options, which are otherwise kept"
+ (let [sr' (streaming-response/with-error-response-fn sr error-response-fn-for-test)]
+ (is (instance? metabase.server.streaming_response.StreamingResponse sr'))
+ (is (= {:content-type "application/json", :error-response-fn error-response-fn-for-test}
+ (.options ^metabase.server.streaming_response.StreamingResponse sr')))))
+ (testing "a Ring response map whose body is a streaming response gets its body updated"
+ (let [response (streaming-response/with-error-response-fn {:status 202, :body sr} error-response-fn-for-test)]
+ (is (= 202 (:status response)))
+ (is (= error-response-fn-for-test
+ (:error-response-fn (.options ^metabase.server.streaming_response.StreamingResponse (:body response)))))))
+ (testing "any other response is returned unchanged"
+ (is (= {:status 200, :body {:a 1}}
+ (streaming-response/with-error-response-fn {:status 200, :body {:a 1}} error-response-fn-for-test)))
+ (is (nil? (streaming-response/with-error-response-fn nil error-response-fn-for-test))))))
+
(deftest ^:parallel streaming-response-schema-error-test
(testing "streaming-response-schema correctly validates responses"
(let [schema (streaming-response/streaming-response-schema [:map [:data :any]])]
@@ -674,7 +788,7 @@
finished-chan
canceled-chan
(AtomicBoolean. false)
- executor)
+ {:executor executor})
(is (= "custom-streaming-executor" (deref ran-on 5000 ::timed-out)))
(is (= :completed (a/<!! finished-chan))))
(finally
diff --git a/test/metabase/test/http_client.clj b/test/metabase/test/http_client.clj
index 92932644164..a6462e63188 100644
--- a/test/metabase/test/http_client.clj
+++ b/test/metabase/test/http_client.clj
@@ -351,7 +351,12 @@
(setContentType [_ _])
(setHeader [_ _ _]))]
(binding [streaming-response/*response* mock-response]
- (f os canceled-chan))
+ ;; hand `f` the same kind of output stream the Jetty path does, so it carries the response's
+ ;; `:error-response-fn` to `write-error!`
+ (f (#'streaming-response/delay-output-stream
+ (delay os)
+ (:error-response-fn (.options ^StreamingResponse streaming-response)))
+ canceled-chan))
{:body (cond-> (.toByteArray os)
(some #(re-find % content-type) [#"json" #"text"])
(String. "UTF-8"))