The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve error handling in query responses (#82531)
Nathan Voxlandf062ce0832a6
diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex b27d7894ae9..2c11c0fe7be 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -58,6 +58,7 @@[metabase.revisions.api][metabase.search.api][metabase.segments.rest.api]+ [metabase.server.streaming-response :as streaming-response][metabase.session.api][metabase.settings-rest.api][metabase.setup-rest.api]@@ -155,9 +156,29 @@(cond-> x(simple-symbol? x) api.macros/ns-handler))+(defn- sanitize-streaming-errors+ "Middleware that reduces any error written by a streaming response body to+ [[metabase.public-sharing-rest.api/error-response]]. The public and embedding route wrappers catch and genericize+ exceptions thrown by the handler, but a streaming body runs after the handler has returned, so a query failure --+ or an exception attaching the Card's query as `ex-data` -- would otherwise reach an unauthenticated client as-is.++ This lives here rather than next to those wrappers in [[metabase.api.routes.common]] because the `api` module can't+ depend on `server` (which depends on the query processor, which depends on `api`)."+ [handler]+ (fn [request respond raise]+ (handler request+ (fn [response]+ (respond (streaming-response/with-error-response-fn+ response+ metabase.public-sharing-rest.api/error-response)))+ raise)))++(def ^:private ^{:arglists '([handler])} +sanitize-streaming-errors+ (routes.common/wrap-middleware-for-open-api-spec-generation sanitize-streaming-errors))+(defn- +auth [handler] (routes.common/+auth (->handler handler)))-(defn- +message-only-exceptions [handler] (routes.common/+message-only-exceptions (->handler handler)))-(defn- +public-exceptions [handler] (routes.common/+public-exceptions (->handler handler)))+(defn- +message-only-exceptions [handler] (routes.common/+message-only-exceptions (+sanitize-streaming-errors (->handler handler))))+(defn- +public-exceptions [handler] (routes.common/+public-exceptions (+sanitize-streaming-errors (->handler handler))))(declare routes)diff --git a/src/metabase/public_sharing_rest/api.clj b/src/metabase/public_sharing_rest/api.cljindex f8c8ac062fa..f2938a868b9 100644--- a/src/metabase/public_sharing_rest/api.clj+++ b/src/metabase/public_sharing_rest/api.clj@@ -157,15 +157,25 @@[:json_query :parameters]:status]))-(defmethod transform-qp-result :failed- [{error-type :error_type, :as results}]- ;; if the query failed instead, unless the error type is specified and is EXPLICITLY allowed to be shown for embeds,- ;; instead of returning anything about the query just return a generic error message+(defn error-response+ "Reduce a query `error` -- the QP's formatted error response, or the `Throwable->map` of an exception that escaped+ it -- to what public and embedded endpoints are allowed to return: the status and error type, and a generic message+ in place of the original unless the error type is EXPLICITLY allowed to be shown in embeds. Nothing about the query+ itself gets through.++ [[metabase.api-routes.routes]] applies this to every error written by a streaming response under the public and+ embedding routes."+ [{error-type :error_type, :as error}](merge- (select-keys results [:status :error :error_type])+ {:status :failed}+ (select-keys error [:status :error :error_type])(when-not (qp.error-type/show-in-embeds? error-type){:error (tru "An error occurred while running the query.")})))+(defmethod transform-qp-result :failed+ [results]+ (error-response results))+(defn- process-query-for-card-with-id-run-fn"Create the `:make-run` function used for [[process-query-for-card-with-id]] and [[process-query-for-dashcard]]."[qp export-format]diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 738e54c70a4..ff29ee60045 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -256,8 +256,7 @@(update :middleware select-keys [:js-int-to-string? :ignore-cached-results?])):constraints (qp.constraints/default-query-constraints):info info)- rff)- query)))+ rff))))(defn- parameter-field-values[field-ids query]diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex f503f28d975..ad85bf39289 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -29,6 +29,7 @@[metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.metadata :as qp.metadata][metabase.query-processor.middleware.add-remaps :as qp.add-remaps]+ [metabase.query-processor.middleware.catch-exceptions :as qp.catch-exceptions][metabase.query-processor.middleware.drop-fields-in-summaries :as qp.drop-fields-in-summaries][metabase.query-processor.middleware.nest-for-pivot :as qp.nest-for-pivot][metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize]@@ -800,6 +801,23 @@(throw t)(:outcome primary-outcome))))+(mu/defn- run-pivot-query*+ "Impl for [[run-pivot-query]]: pick the pivot implementation for `query` and run it through `rff`."+ [query :- ::qp.schema/any-query+ rff :- ::qp.schema/rff]+ (let [query (-> query+ qp.middleware.normalize/normalize-preprocessing-middleware+ lib/prepare-after-deserialization)+ db (query-database query)+ nativable? (native-path-applicable? db query)+ use-native? (and nativable? (qp.settings/use-native-pivot-tables))+ primary (if use-native? run-native-pivot-query run-pivot-query-multi)+ secondary (if use-native? run-pivot-query-multi run-native-pivot-query)]+ (binding [qp.pipeline/*pivot?* true]+ (if (and nativable? (pivot-parity-enabled?))+ (run-with-parity-check primary secondary query rff use-native?)+ (primary query rff)))))+(mu/defn run-pivot-query"Run the pivot `query` through `rff`.@@ -812,6 +830,9 @@rff, secondary via the default rff for comparison) and disagreement is reported via[[*on-parity-mismatch*]]. Parity checking is on by default in clojure.test tests.+ A query with `:info` is run as a userland query, so any error is+ caught and returned as a formatted error response rather than thrown.+Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself."([query :- ::qp.schema/any-query](run-pivot-query query nil))@@ -824,15 +845,12 @@;; run-pivot-query, so binding it here from the query's :info map would be;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.(qp.setup/with-qp-setup [query query]- (let [query (-> query- qp.middleware.normalize/normalize-preprocessing-middleware- lib/prepare-after-deserialization)- db (query-database query)- nativable? (native-path-applicable? db query)- use-native? (and nativable? (qp.settings/use-native-pivot-tables))- primary (if use-native? run-native-pivot-query run-pivot-query-multi)- secondary (if use-native? run-pivot-query-multi run-native-pivot-query)]- (binding [qp.pipeline/*pivot?* true]- (if (and nativable? (pivot-parity-enabled?))- (run-with-parity-check primary secondary query rff use-native?)- (primary query rff)))))))+ (let [query (cond-> query+ (seq (:info query)) qp/userland-query)+ rff (or rff qp.reducible/default-rff)+ ;; Everything between here and the first `qp/process-query`+ ;; runs outside the QP's own middleware, so wrap it in the same exception-catching middleware+ ;; `qp/process-query` applies to userland queries.+ ;; No-op for non-userland queries.+ qp (qp.catch-exceptions/catch-exceptions run-pivot-query*)]+ (qp query rff)))))diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 3fce2b9c139..2d33fa96255 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -42,7 +42,22 @@(defn- format-exception [e](cond-> (Throwable->map e)- (server.settings/hide-stacktraces) (dissoc :via :trace)))+ ;; `:data` is the ex-data, which for QP exceptions routinely carries the whole query; drop it along with the+ ;; stacktrace and cause chain, as the (non-streaming) exception middleware does.+ (server.settings/hide-stacktraces) (dissoc :via :trace :data)))++;; Implemented by the output stream handed to a streaming response body, so that [[write-error!]] -- called from the+;; body with nothing but that stream -- can find the response's `:error-response-fn`. A function applied to any error+;; body (a formatted error map, or the `Throwable->map` of an exception) right before it is written to the client;+;; whatever it returns is what gets written. See [[with-error-response-fn]].+(definterface ErrorResponseFnProvider+ (errorResponseFn []))++(defn- error-response-fn+ "The `:error-response-fn` carried by `os`, if it is the output stream handed to a streaming response body."+ [os]+ (when (instance? ErrorResponseFnProvider os)+ (.errorResponseFn ^ErrorResponseFnProvider os)))#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}(def ^:dynamic *response*@@ -121,16 +136,18 @@(log/errorf "Async context already completed, cannot write error to client: %s"(if (map? obj) (or (:message obj) (:error obj) (:status obj)) obj))))-(defn- sanitize-error-obj [obj export-format]- (-> (if (not= :api export-format)- (walk/prewalk (fn [x]- (if (map? x)- (apply dissoc x [:json_query :preprocessed])- x))- obj)- obj)- (dissoc :export-format)- (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via))))+(defn- sanitize-error-obj [obj export-format response-fn]+ (let [obj (cond-> obj+ response-fn response-fn)]+ (-> (if (not= :api export-format)+ (walk/prewalk (fn [x]+ (if (map? x)+ (apply dissoc x [:json_query :preprocessed])+ x))+ obj)+ obj)+ (dissoc :export-format)+ (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via)))))(defn- abort-connection!"Abort the underlying Jetty connection for an *already-committed* streaming response, so it@@ -161,7 +178,7 @@(with-open [os os](log/trace "write-error!")(try- (let [obj (sanitize-error-obj obj export-format)]+ (let [obj (sanitize-error-obj obj export-format (error-response-fn os))](with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))](json/encode-to obj writer {})))(catch EofException _)@@ -234,10 +251,11 @@(.cancel fut true)))))))(defn- do-f-async- "Runs `f` asynchronously on `executor` (the shared streaming response `thread-pool` when nil), returning- immediately. When `f` finishes, completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean`+ "Runs `f` asynchronously on the `:executor` in `options` (the shared streaming response `thread-pool` when nil),+ returning immediately. When `f` finishes, completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean`used to coordinate with Jetty's timeout/error callbacks so that only one path calls `.complete`."- [^AsyncContext async-context response ^Request request f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed? executor]+ [^AsyncContext async-context response ^Request request f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?+ {:keys [executor], :as _options}]{:pre [(some? os)]}(let [task (^:once fn* [](binding [*response* response@@ -280,9 +298,11 @@(defn- delay-output-stream"An OutputStream proxy that fetches the actual output stream by dereffing a delay (or other dereffable) before first- use."- [dlay]- (proxy [OutputStream] []+ use. It also carries the response's `error-response-fn` (possibly nil) for [[write-error!]]."+ [dlay error-response-fn]+ (proxy [OutputStream ErrorResponseFnProvider] []+ (errorResponseFn []+ error-response-fn)(close [](.close ^OutputStream @dlay))(flush []@@ -379,7 +399,7 @@(defn- respond[{:keys [^HttpServletResponse response ^AsyncContext async-context request-map response-map request]}- f {:keys [content-type status headers executor], :as _options} finished-chan]+ f {:keys [content-type status headers], :as options} finished-chan](let [canceled-chan (a/promise-chan)completed? (AtomicBoolean. false)](.addListener async-context@@ -416,9 +436,9 @@gzip? (assoc "Content-Encoding" "gzip"))](#'servlet/set-headers response headers)(let [output-stream-delay (output-stream-delay gzip? response)- delay-os (delay-output-stream output-stream-delay)]+ delay-os (delay-output-stream output-stream-delay (:error-response-fn options))](start-async-cancel-loop! request finished-chan canceled-chan)- (do-f-async async-context response request f delay-os finished-chan canceled-chan completed? executor)))+ (do-f-async async-context response request f delay-os finished-chan canceled-chan completed? options)))(catch Throwable e(log/errorf "Unexpected exception in do-f-async: %s" (ex-message e))(try@@ -500,13 +520,39 @@which is a small fixed pool: responses that block for the life of a client connection (rather than for the lifeof a query) must supply their own executor so they cannot exhaust it. The supplied executor's futures mustsupport real interruption (`Future.cancel(true)`) — the hung-request escalation interrupts the worker, and a- ForkJoinPool-backed executor silently ignores it."+ ForkJoinPool-backed executor silently ignores it.+ * `:error-response-fn` -- optional function applied to any error body right before [[write-error!]] writes it to the+ client. Usually added after the fact with [[with-error-response-fn]]."{:style/indent 2, :arglists '([options [os-binding canceled-chan-binding] & body])}[options [os-binding canceled-chan-binding :as bindings] & body]{:pre [(= (count bindings) 2)]}`(-streaming-response (bound-fn [~(vary-meta os-binding assoc :tag 'java.io.OutputStream) ~canceled-chan-binding] ~@body)~options))+(defn with-error-response-fn+ "Make a streaming `response` pass any error its body writes -- a formatted error map, or the `Throwable->map` of an+ exception -- through `error-response-fn` first, writing whatever that returns instead.++ For route middleware sanitizing errors for public and embedded endpoints: their `try`/`catch` around the handler+ cannot see these errors, because a streaming response body runs after the handler has returned.++ `response` may be a [[StreamingResponse]] or a Ring response map whose `:body` is one; anything else is returned+ unchanged."+ [response error-response-fn]+ (cond+ (instance? StreamingResponse response)+ (let [^StreamingResponse response response]+ (->StreamingResponse (.f response)+ (assoc (.options response) :error-response-fn error-response-fn)+ (.donechan response)))++ (and (map? response)+ (instance? StreamingResponse (:body response)))+ (update response :body with-error-response-fn error-response-fn)++ :else+ response))+;;;; Malli schema for StreamingResponse(defn streaming-response-schemadiff --git a/test/metabase/embedding_rest/api/embed_test.clj b/test/metabase/embedding_rest/api/embed_test.cljindex b0992847b4f..565ac690c9e 100644--- a/test/metabase/embedding_rest/api/embed_test.clj+++ b/test/metabase/embedding_rest/api/embed_test.clj@@ -18,6 +18,7 @@[metabase.parameters.custom-values :as custom-values][metabase.public-sharing-rest.api-test :as public-test][metabase.queries-rest.api.card-test :as api.card-test]+ [metabase.query-processor.card :as qp.card][metabase.query-processor.middleware.constraints :as qp.constraints][metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test][metabase.query-processor.pivot.test-util :as api.pivots]@@ -1672,6 +1673,38 @@(is (= "Message seems corrupt or manipulated"(client/client :get 400 (with-new-secret-key! (pivot-card-query-url card ""))))))))))))+(deftest embed-pivot-card-error-does-not-leak-query-test+ (testing "GET /api/embed/pivot/card/:token/query"+ (testing "an error raised while building the pivot sub-queries must not leak the Card's query or a stacktrace"+ (with-embedding-enabled-and-new-secret-key!+ (with-temp-card [card {:enable_embedding true+ :name "EMBED ERROR LEAK CARD NAME CANARY"+ :display :pivot+ :dataset_query {:database (mt/id)+ :type :native+ :native {:query "SELECT * FROM no_such_table -- EMBED_ERROR_LEAK_SQL_CANARY"}}}]+ (let [{:keys [status body]} (client/client-full-response :get (pivot-card-query-url card ""))+ body-str (pr-str body)]+ (is (= 500 status))+ (is (= {:status "failed", :error "An error occurred while running the query.", :error_type "qp"}+ body))+ (is (not (str/includes? body-str "CANARY")))+ (is (not (str/includes? body-str ":trace")))))))))++(deftest embed-card-query-exception-outside-qp-does-not-leak-test+ (testing "GET /api/embed/card/:token/query"+ (testing "an exception that escapes the QP entirely is still reduced to the generic embedding error"+ (with-embedding-enabled-and-new-secret-key!+ (with-temp-card [card {:enable_embedding true, :name "EMBED ERROR LEAK CARD NAME CANARY"}]+ (mt/with-dynamic-fn-redefs [qp.card/process-query-for-card-default-qp+ (fn [query _rff]+ (throw (ex-info "Boom EMBED_ERROR_LEAK_SQL_CANARY" {:query query})))]+ (let [{:keys [status body]} (client/client-full-response :get (card-query-url card ""))]+ (is (= 500 status))+ (is (= {:status "failed", :error "An error occurred while running the query."}+ body))+ (is (not (str/includes? (pr-str body) "CANARY"))))))))))+(defn- pivot-dashcard-url([dashcard] (pivot-dashcard-url dashcard (:dashboard_id dashcard)))([dashcard dashboard-id & [additional-token-keys]]diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.cljindex 713ddd17810..5f66c691dc3 100644--- a/test/metabase/public_sharing_rest/api_test.clj+++ b/test/metabase/public_sharing_rest/api_test.clj@@ -21,6 +21,7 @@[metabase.public-sharing-rest.api :as api.public][metabase.public-sharing.core :as public-sharing][metabase.queries-rest.api.card-test :as api.card-test]+ [metabase.query-processor.card :as qp.card][metabase.query-processor.card-test :as qp.card-test][metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test][metabase.query-processor.pivot.test-util :as api.pivots]@@ -2058,6 +2059,104 @@(is (= "Not found."(client/client :get 404 (dashcard-url dash card dashcard)))))))))))))+(def ^:private error-leak-sql-canary "ERROR_LEAK_SQL_CANARY")+(def ^:private error-leak-card-name-canary "ERROR LEAK CARD NAME CANARY")++(defn- date-param-native-card+ "A healthy native Card with a date field filter, so a caller-supplied parameter value can drive it to an error without+ the Card itself being broken."+ [display]+ {:name error-leak-card-name-canary+ :display display+ :dataset_query {:database (mt/id)+ :type :native+ :native {:query (str "SELECT COUNT(*) AS N FROM ORDERS WHERE {{d}} -- " error-leak-sql-canary)+ :template-tags {"d" {:id "d"+ :name "d"+ :display-name "D"+ :type :dimension+ :widget-type :date/all-options+ :dimension [:field (mt/id :orders :created_at) nil]}}}}+ :parameters [{:id "d"+ :type :date/all-options+ :name "D"+ :slug "d"+ :target [:dimension [:template-tag "d"]]}]})++(def ^:private unparseable-date-param+ "A parameter value that passes endpoint validation but blows up while the query is being built."+ (json/encode [{:id "d"+ :type "date/all-options"+ :target ["dimension" ["template-tag" "d"]]+ :value "NOT-A-DATE"}]))++(defn- assert-generic-query-error+ "Assert that `response` (from [[client/client-full-response]]) is the generic public-endpoint failure body and that+ nothing about the Card leaked into it."+ [{:keys [status body]}]+ (let [body-str (pr-str body)]+ (testing "the query genuinely failed"+ (is (contains? #{400 500} status)))+ (testing "the body is the generic failed-query shape"+ (is (=? {:status "failed"+ :error string?}+ body))+ (is (set/subset? (set (keys body)) #{:status :error :error_type})))+ (testing "the Card's SQL, name, and a stacktrace must not reach an unauthenticated caller"+ (is (not (str/includes? body-str error-leak-sql-canary)))+ (is (not (str/includes? body-str error-leak-card-name-canary)))+ (is (not (str/includes? body-str ":trace")))+ (is (not (str/includes? body-str ":via"))))))++(deftest public-pivot-card-error-does-not-leak-query-test+ (testing "GET /api/public/pivot/card/:uuid/query"+ (testing "an error raised while building the pivot sub-queries must not leak the Card's query or a stacktrace"+ (mt/dataset test-data+ (mt/with-temporary-setting-values [enable-public-sharing true]+ (with-temp-public-card [{uuid :public_uuid} (date-param-native-card :pivot)]+ (let [url (format "public/pivot/card/%s/query" uuid)]+ (testing "sanity check: the Card is healthy without the bad parameter"+ (is (= 202 (:status (client/client-full-response :get url)))))+ (assert-generic-query-error+ (client/client-full-response :get url :parameters unparseable-date-param)))))))))++(deftest public-card-with-pivot-display-error-does-not-leak-query-test+ (testing "GET /api/public/card/:uuid/query"+ (testing "a Card with :display :pivot takes the pivot path on its ordinary public link too"+ (mt/dataset test-data+ (mt/with-temporary-setting-values [enable-public-sharing true]+ (with-temp-public-card [{uuid :public_uuid} (date-param-native-card :pivot)]+ (assert-generic-query-error+ (client/client-full-response :get (format "public/card/%s/query" uuid)+ :parameters unparseable-date-param))))))))++(deftest public-pivot-dashcard-error-does-not-leak-query-test+ (testing "GET /api/public/pivot/dashboard/:uuid/dashcard/:dashcard-id/card/:card-id"+ (testing "an error raised before the QP runs must not leak the query of a Card that is not itself public"+ (mt/with-temporary-setting-values [enable-public-sharing true]+ (with-temp-public-dashboard [dash]+ (mt/with-temp [:model/Card card {:name error-leak-card-name-canary+ :display :pivot+ :dataset_query {:database (mt/id)+ :type :native+ :native {:query (str "SELECT * FROM no_such_table -- "+ error-leak-sql-canary)}}}]+ (let [dashcard (add-card-to-dashboard! card dash)]+ (is (nil? (:public_uuid card)))+ (assert-generic-query-error+ (client/client-full-response :get (pivot-dashcard-url dash card dashcard))))))))))++(deftest public-card-query-exception-outside-qp-does-not-leak-test+ (testing "GET /api/public/card/:uuid/query"+ (testing "an exception that escapes the QP entirely (thrown outside its error-handling middleware) is still sanitized"+ (mt/with-temporary-setting-values [enable-public-sharing true]+ (with-temp-public-card [{uuid :public_uuid} {:name error-leak-card-name-canary}]+ (mt/with-dynamic-fn-redefs [qp.card/process-query-for-card-default-qp+ (fn [query _rff]+ (throw (ex-info (str "Boom " error-leak-sql-canary) {:query query})))]+ (assert-generic-query-error+ (client/client-full-response :get (format "public/card/%s/query" uuid)))))))))+;;; ------------------------- POST /api/public/dashboard/:dashboard-uuid/dashcard/:uuid/execute ------------------------------(deftest execute-public-dashcard-action-testdiff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 8729de9eb0b..53b740b5a98 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -769,6 +769,23 @@(is (= ["MD" "Twitter" nil 4 16 62] (nth rows 1000)))(is (= [nil nil nil 7 18760 69540] (last rows)))))))))+(deftest ^:parallel pivot-dataset-error-response-test+ (mt/dataset test-data+ (testing "POST /api/dataset/pivot"+ (testing "a failed query returns the usual formatted error response, not an empty body"+ (doseq [[cause query] {"an out-of-range pivot row index"+ (assoc (qp.pivot.test-util/pivot-query) :pivot_rows [0 1 2 3])++ "a reference to a nonexistent column"+ (assoc-in (qp.pivot.test-util/pivot-query) [:query :filter]+ [:= [:field Integer/MAX_VALUE nil] 1])}]+ (testing cause+ (let [{:keys [status body]} (mt/user-http-request-full-response :crowberto :post "dataset/pivot" query)]+ (is (contains? #{400 500} status))+ (is (=? {:status "failed"+ :error string?}+ body)))))))))+(deftest ^:parallel pivot-dataset-row-totals-disabled-test(mt/test-drivers (qp.pivot.test-util/applicable-drivers)(mt/dataset test-datadiff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex 1eb88887689..0bee9e11807 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -145,6 +145,25 @@(mt/rows (qp.pivot/run-pivot-query (set/rename-keys (test-query){:pivot-rows :pivot_rows, :pivot-cols :pivot_cols})))))))+(deftest ^:parallel error-generating-pivot-queries-test+ (testing "An error thrown while generating the pivot sub-queries, i.e. before the QP proper (and its error-handling middleware) runs"+ ;; the native path silently drops out-of-range pivot indexes rather than throwing, so parity would disagree here+ (qp.pivot.test-util/without-pivot-parity-check+ (let [query (assoc (test-query) :pivot-rows [0 1 2 3])]+ (testing "is thrown as-is for a non-userland query"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Error generating pivot queries"+ (qp.pivot/run-pivot-query query))))+ (testing "is returned in the usual formatted error shape for a userland query, like any other QP error"+ (doseq [userland-query [(qp.core/userland-query query)+ (assoc query :info {:context :ad-hoc})]]+ (is (=? {:status :failed+ :error #"Invalid pivot-rows: specified breakout at index 3, but we only have 3 breakouts"+ :error_type :invalid-query+ :json_query map?}+ (qp.pivot/run-pivot-query userland-query)))))))))+(deftest ^:parallel generate-queries-test(mt/test-drivers (qp.pivot.test-util/applicable-drivers)(let [metadata-provider (mt/metadata-provider)diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex 6b3d8782a96..d83043a7391 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -413,8 +413,8 @@"Response should not contain :trace key")(is (not (contains? error-response :via))"Response should not contain :via key")- (is (= "test-value" (get-in error-response [:data :custom-data]))- "Response should include custom data from ex-info")))))))+ (is (not (contains? error-response :data))+ "Response should not contain the ex-data either, like the regular exception middleware")))))))(deftest write-error-nested-exception-with-stacktraces-disabled-test(testing "write-error! includes nested exception details when hide-stacktraces is false"@@ -484,6 +484,120 @@(is (= "preserve-me" (:custom-data error-response))"Response should still include custom data")))))))+(defn- error-response-fn-for-test [error]+ {:status :failed, :error "generic message", :original-keys (sort (keys error))})++(defn- error-carrying-output-stream+ "An output stream like the one a streaming response body is handed, carrying `error-response-fn`, that writes to+ `os`."+ ^java.io.OutputStream [os error-response-fn]+ (#'streaming-response/delay-output-stream (delay os) error-response-fn))++(deftest write-error-applies-error-response-fn-test+ (testing "when the output stream carries an :error-response-fn, write-error! writes what it returns instead of the error itself"+ (mt/with-temporary-setting-values [hide-stacktraces false]+ (testing "for a raw exception (formatted to a map first, so the fn always sees a map)"+ (let [baos (java.io.ByteArrayOutputStream.)]+ (#'streaming-response/write-error! (error-carrying-output-stream baos error-response-fn-for-test)+ (ex-info "SENSITIVE MESSAGE" {:query "SENSITIVE QUERY"})+ :api)+ (let [output (String. (.toByteArray baos) "UTF-8")]+ (is (= {:status "failed"+ :error "generic message"+ :original-keys ["cause" "data" "trace" "via"]}+ (json/decode output true)))+ (is (not (re-find #"SENSITIVE" output))))))+ (testing "for an already-formatted error map"+ (let [baos (java.io.ByteArrayOutputStream.)]+ (#'streaming-response/write-error! (error-carrying-output-stream baos error-response-fn-for-test)+ {:status :failed, :error "SENSITIVE", :json_query "SENSITIVE"}+ :api)+ (let [output (String. (.toByteArray baos) "UTF-8")]+ (is (= {:status "failed"+ :error "generic message"+ :original-keys ["error" "json_query" "status"]}+ (json/decode output true)))+ (is (not (re-find #"SENSITIVE" output))))))+ (testing "an output stream without one writes the error unchanged"+ (let [baos (java.io.ByteArrayOutputStream.)]+ (#'streaming-response/write-error! (error-carrying-output-stream baos nil)+ {:status :failed, :error "visible"}+ :api)+ (is (= {:status "failed", :error "visible"}+ (json/decode (String. (.toByteArray baos) "UTF-8") true))))))))++(defn- respond-and-capture+ "Respond with `streaming-response` against a fake servlet response. Returns the status code and the body written."+ [streaming-response]+ (with-open [os (java.io.ByteArrayOutputStream.)]+ (let [complete-promise (promise)+ status (atom nil)]+ (server.protocols/respond streaming-response+ {:response (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ new-status] (reset! status new-status))+ (setContentType [_ _])+ (setHeader [_ _ _])+ (getOutputStream [_]+ (proxy [ServletOutputStream] []+ (write+ ([byytes]+ (.write os ^bytes byytes))+ ([byytes offset length]+ (.write os ^bytes byytes offset length))))))+ :async-context (reify AsyncContext+ (addListener [_ _])+ (complete [_]+ (deliver complete-promise true)))})+ (is (true? (deref complete-promise 1000 ::timed-out)))+ {:status @status+ :body (String. (.toByteArray os) "UTF-8")})))++(deftest with-error-response-fn-test+ (testing "with-error-response-fn makes a streaming response pass any error its body produces through the fn"+ (mt/with-temporary-setting-values [hide-stacktraces false]+ (testing "an exception thrown by the body, on the worker thread"+ (let [{:keys [status body]} (respond-and-capture+ (streaming-response/with-error-response-fn+ (streaming-response/streaming-response {:content-type "application/json"} [_os _]+ (throw (ex-info "SENSITIVE MESSAGE" {:query "SENSITIVE QUERY"})))+ error-response-fn-for-test))]+ (is (= 500 status))+ (is (= {:status "failed"+ :error "generic message"+ :original-keys ["cause" "data" "trace" "via"]}+ (json/decode body true)))+ (is (not (re-find #"SENSITIVE" body)))))+ (testing "an error map the body writes itself with write-error!, as the query processor does"+ (let [{:keys [status body]} (respond-and-capture+ (streaming-response/with-error-response-fn+ (streaming-response/streaming-response {:content-type "application/json"} [os _]+ (streaming-response/write-error! os {:status :failed, :error "SENSITIVE", :json_query "SENSITIVE"} :api 400))+ error-response-fn-for-test))]+ (is (= 400 status))+ (is (= {:status "failed"+ :error "generic message"+ :original-keys ["error" "json_query" "status"]}+ (json/decode body true)))+ (is (not (re-find #"SENSITIVE" body))))))))++(deftest with-error-response-fn-response-shapes-test+ (let [sr (streaming-response/streaming-response {:content-type "application/json"} [_os _])]+ (testing "a streaming response gets the fn added to its options, which are otherwise kept"+ (let [sr' (streaming-response/with-error-response-fn sr error-response-fn-for-test)]+ (is (instance? metabase.server.streaming_response.StreamingResponse sr'))+ (is (= {:content-type "application/json", :error-response-fn error-response-fn-for-test}+ (.options ^metabase.server.streaming_response.StreamingResponse sr')))))+ (testing "a Ring response map whose body is a streaming response gets its body updated"+ (let [response (streaming-response/with-error-response-fn {:status 202, :body sr} error-response-fn-for-test)]+ (is (= 202 (:status response)))+ (is (= error-response-fn-for-test+ (:error-response-fn (.options ^metabase.server.streaming_response.StreamingResponse (:body response)))))))+ (testing "any other response is returned unchanged"+ (is (= {:status 200, :body {:a 1}}+ (streaming-response/with-error-response-fn {:status 200, :body {:a 1}} error-response-fn-for-test)))+ (is (nil? (streaming-response/with-error-response-fn nil error-response-fn-for-test))))))+(deftest ^:parallel streaming-response-schema-error-test(testing "streaming-response-schema correctly validates responses"(let [schema (streaming-response/streaming-response-schema [:map [:data :any]])]@@ -674,7 +788,7 @@finished-chancanceled-chan(AtomicBoolean. false)- executor)+ {:executor executor})(is (= "custom-streaming-executor" (deref ran-on 5000 ::timed-out)))(is (= :completed (a/<!! finished-chan))))(finallydiff --git a/test/metabase/test/http_client.clj b/test/metabase/test/http_client.cljindex 92932644164..a6462e63188 100644--- a/test/metabase/test/http_client.clj+++ b/test/metabase/test/http_client.clj@@ -351,7 +351,12 @@(setContentType [_ _])(setHeader [_ _ _]))](binding [streaming-response/*response* mock-response]- (f os canceled-chan))+ ;; hand `f` the same kind of output stream the Jetty path does, so it carries the response's+ ;; `:error-response-fn` to `write-error!`+ (f (#'streaming-response/delay-output-stream+ (delay os)+ (:error-response-fn (.options ^StreamingResponse streaming-response)))+ canceled-chan)){:body (cond-> (.toByteArray os)(some #(re-find % content-type) [#"json" #"text"])(String. "UTF-8"))