Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve query error responses and logs (#84585)

Nathan Voxland1bf41894f504
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsx b/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsx
index fc35ef3ff12..b9aa81a4882 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsx
+++ b/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsx
@@ -77,7 +77,6 @@ export function QueryVisualization(props: QueryVisualizationProps) {
className={S.spread}
error={result.error}
errorType={result.error_type}
- via={result.via ?? []}
question={question}
duration={result.duration ?? 0}
errorAction={errorAction}
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx
index fc763241ca1..c9ea21b3ecc 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx
@@ -1,5 +1,4 @@
import cx from "classnames";
-import { getIn } from "icepick";
import type { ReactNode } from "react";
import { t } from "ttag";
@@ -23,11 +22,10 @@ import { VISUALIZATION_SLOW_TIMEOUT } from "../../../constants";
import VisErrorS from "./VisualizationError.module.css";
import { AdminEmail } from "./components";
-import { adjustPositions, getDatasetErrorMessage, stripRemarks } from "./utils";
+import { getDatasetErrorMessage, stripRemarks } from "./utils";
interface VisualizationErrorProps {
className?: string;
- via: Record<string, any>[];
question: Question;
duration: number;
error: DatasetError;
@@ -37,7 +35,6 @@ interface VisualizationErrorProps {
export function VisualizationError({
className,
- via,
question,
duration,
error,
@@ -139,12 +136,7 @@ export function VisualizationError({
if (isNative) {
// always show errors for native queries
- let processedError = errorMessage;
- const origSql = getIn(via, [(via || "").length - 1, "ex-data", "sql"]);
- if (typeof origSql === "string" && processedError) {
- processedError = adjustPositions(processedError, origSql);
- }
- processedError = stripRemarks(processedError);
+ const processedError = stripRemarks(errorMessage);
const database = question.database();
const isSql = database && getEngineNativeType(database.engine) === "sql";
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx
index d39d27fb200..4aeb26f679a 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx
@@ -76,7 +76,6 @@ export const setup = ({
error={error as DatasetError}
errorType={errorType}
errorAction={errorAction}
- via={[]}
/>,
{ storeInitialState: state },
);
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts
index ef5b4ce0d65..01af90a32fd 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts
@@ -1,42 +1,4 @@
-import { adjustPositions, stripRemarks } from "../utils";
-
-describe("adjustPositions", () => {
- const remarkedQuery =
- "-- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a\nfwefwef";
- const remarkedRedshiftQuery =
- "/* anything before the remarks in multiline is redshift */-- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a\nfwefwef";
- const unremarkedQuery = "fwefwef";
-
- const unadjustedError = `boopy boop boopy boop fake error message Position: 1000;`;
- const adjustedError = `boopy boop boopy boop fake error message Position: 881;`;
- const redshiftAdjustedError = `boopy boop boopy boop fake error message Position: 823;`;
-
- const errorLineNumbers = "boopy boop boopy boop fake error message Line: 2";
-
- it("error adjusted pg", () => {
- expect(adjustPositions(unadjustedError, remarkedQuery)).toEqual(
- adjustedError,
- );
- });
-
- it("error adjusted redshift", () => {
- expect(adjustPositions(unadjustedError, remarkedRedshiftQuery)).toEqual(
- redshiftAdjustedError,
- );
- });
-
- it("unremarked query should be a noop", () => {
- expect(adjustPositions(unadjustedError, unremarkedQuery)).toEqual(
- unadjustedError,
- );
- });
-
- it("error adjusted line numbers should be a noop", () => {
- expect(adjustPositions(errorLineNumbers, remarkedQuery)).toEqual(
- errorLineNumbers,
- );
- });
-});
+import { stripRemarks } from "../utils";
describe("stripRemarks", () => {
const errorH2Unstripped = `
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts
index 81dc9342f29..aa2dc000136 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts
@@ -13,41 +13,6 @@ export function getDatasetErrorMessage(error: DatasetError): string {
: "";
}
-export function adjustPositions(error: string, origSql: string): string {
- /* Positions in error messages are borked coming in for Postgres errors.
- * Previously, you would see "blahblahblah bombed out, Position: 119" in a 10-character invalid query.
- * This is because MB shoves in 'remarks' into the original query and we get the exception from the query with remarks.
- * This function adjusts the value of the positions in the exception message to account for this.
- * This is done in mildly scary kludge here in frontend after everything,
- * because the alternative of doing it in backend
- * is an absolutely terrifying kludge involving messing with exceptions.
- */
- let adjustmentLength = 0;
-
- // redshift remarks use c-style multiline comments...
- const multiLineBeginPos = origSql.search("/\\*");
- const multiLineEndPos = origSql.search("\\*/");
- // if multiLineBeginPos is 0 then we know it's a redshift remark
- if (multiLineBeginPos === 0 && multiLineEndPos !== -1) {
- adjustmentLength += multiLineEndPos + 2; // 2 for */ in itself
- }
-
- const chompedSql = origSql.substr(adjustmentLength);
- // there also seem to be cases where remarks don't get in...
- const commentPos = chompedSql.search("--");
- const newLinePos = chompedSql.search("\n");
- // 5 is a heuristic: this indicates that this is almost certainly an initial remark comment
- if (commentPos !== -1 && commentPos < 5) {
- // There will be a \n after the redshift comment,
- // which is why there needs to be a 2 added
- adjustmentLength += newLinePos + 2;
- }
-
- return error.replace(/Position: (\d+)/, function (_, p1) {
- return "Position: " + (parseInt(p1) - adjustmentLength);
- });
-}
-
export function stripRemarks(error: string | undefined): string {
/* SQL snippets in error messages are borked coming in for errors in many DBs.
* You're expecting something with just your sql in the message,
diff --git a/src/metabase/query_processor/middleware/catch_exceptions.clj b/src/metabase/query_processor/middleware/catch_exceptions.clj
index 35f7dfc9e7c..7d2a5e7cd79 100644
--- a/src/metabase/query_processor/middleware/catch_exceptions.clj
+++ b/src/metabase/query_processor/middleware/catch_exceptions.clj
@@ -1,9 +1,19 @@
(ns metabase.query-processor.middleware.catch-exceptions
- "Middleware for catching exceptions thrown by the query processor and returning them in a friendlier format."
- (:refer-clojure :exclude [some get-in])
+ "Middleware for catching exceptions thrown by the query processor and returning them in a friendlier format.
+
+ The response carries only the client-facing part of each exception's ex-data (see [[metabase.util.api-error]]). The
+ error log line carries just the message, plus the ex-data keys listed under `:log/keys`, as log context:
+
+ (throw (ex-info (tru \"Field is not available\")
+ {:status-code 400, :field-id id, :field-name field-name, :log/keys #{:field-id :field-name}}))
+
+ responds without either key and logs `field-id` and `field-name` alongside the message. Don't list the query or
+ values from it."
+ (:refer-clojure :exclude [some get-in not-empty select-keys])
(:require
[clojure.string :as str]
[metabase.analytics-interface.core :as analytics]
+ [metabase.config.core :as config]
[metabase.driver :as driver]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.queries.schema :as queries.schema]
@@ -14,11 +24,12 @@
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.schema :as qp.schema]
[metabase.util :as u]
+ [metabase.util.api-error :as api-error]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.util.performance :refer [some get-in]])
+ [metabase.util.performance :refer [some get-in not-empty select-keys]])
(:import
(clojure.lang ExceptionInfo)
(java.sql SQLException)))
@@ -41,19 +52,31 @@
[^InterruptedException _e]
{:status :interrupted})
+(defn- response-ex-data
+ "The part of an exception's `data` sent to the client: what [[api-error/response-data]] sends, and `:status-code`,
+ which [[metabase.query-processor.streaming]] turns into the HTTP status."
+ [data]
+ (not-empty (merge (api-error/response-data data)
+ (select-keys data [:status-code]))))
+
(defmethod format-exception ExceptionInfo
[e]
;; `:is-curated` is a flag that signals whether the error message in `e` was approved by product
;; to be shown to the user. It is used by FE.
- (let [{error-type :type, is-curated :is-curated, :as data} (ex-data e)]
+ (let [{error-type :type, is-curated :is-curated, :as data} (ex-data e)
+ response-data (response-ex-data data)]
+ (when-let [dropped (and config/is-dev?
+ (not-empty (apply dissoc data (keys response-data))))]
+ (log/debugf "Query error response leaves out ex-data: %s" (pr-str dropped)))
(merge
((get-method format-exception Throwable) e)
(when (qp.error-type/known-error-type? error-type)
{:error_type error-type})
(when is-curated
{:error_is_curated is-curated})
- ;; TODO - we should probably change this key to `:data` so we're not mixing lisp-case and snake_case keys
- {:ex-data data})))
+ (when response-data
+ ;; TODO - we should probably change this key to `:data` so we're not mixing lisp-case and snake_case keys
+ {:ex-data response-data}))))
(defmethod format-exception SQLException
[^SQLException e]
@@ -151,6 +174,19 @@
(catch Throwable e
(assoc (Throwable->map e) :status :failed))))
+(defn- log-context
+ "The ex-data entries each exception in `e`'s chain lists under `:log/keys`, for the error log line. An inner
+ exception's entries win over an outer one's. Empty when a `:log/keys` is malformed, so that it can't mask `e`."
+ [e]
+ (try
+ (into {}
+ (map (fn [ex]
+ (let [data (ex-data ex)]
+ (select-keys data (:log/keys data)))))
+ (u/full-exception-chain e))
+ (catch Throwable _
+ {})))
+
(mu/defn catch-exceptions :- ::qp.schema/qp
"Middleware for catching exceptions thrown by the query processor and returning them in a 'normal' format. Forwards
exceptions to the `result-chan`."
@@ -175,8 +211,9 @@
pool-saturated? (qp.error-type/connection-pool-saturated?
(:error_type formatted-exception))]
(when-not (or query-canceled? pool-saturated?)
- (log/errorf "Error processing query: %s"
- (or (:error formatted-exception) "Error running query")))
+ (log/with-thread-context (log-context e)
+ (log/errorf "Error processing query: %s"
+ (or (:error formatted-exception) "Error running query"))))
;; ensure always a message on the error otherwise FE thinks query was successful. (#23258, #23281)
(let [result (update formatted-exception
:error (fnil identity (trs "Error running query")))]
diff --git a/test/metabase/documents/api/document_test.clj b/test/metabase/documents/api/document_test.clj
index ca22899ef42..df38bb8ba7f 100644
--- a/test/metabase/documents/api/document_test.clj
+++ b/test/metabase/documents/api/document_test.clj
@@ -2335,9 +2335,7 @@
(data-perms/set-database-permission! all-users-group (mt/id) :perms/download-results :no)
(is (malli= [:map
[:status [:= "failed"]]
- [:error_type [:= "missing-required-permissions"]]
- [:ex-data [:map
- [:permissions-error? [:= true]]]]]
+ [:error_type [:= "missing-required-permissions"]]]
(mt/user-http-request :rasta
:post 403
(format "document/%s/card/%s/query/csv" doc-id card-id)
diff --git a/test/metabase/query_processor/middleware/catch_exceptions_test.clj b/test/metabase/query_processor/middleware/catch_exceptions_test.clj
index 2595b34ecd5..da26990e71a 100644
--- a/test/metabase/query_processor/middleware/catch_exceptions_test.clj
+++ b/test/metabase/query_processor/middleware/catch_exceptions_test.clj
@@ -3,6 +3,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.catch-exceptions-test]}}}}}}
(:require
[clojure.test :refer :all]
+ [metabase.config.core :as config]
[metabase.driver :as driver]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -17,7 +18,8 @@
[metabase.query-processor.reducible :as qp.reducible]
[metabase.query-processor.test :as qp]
[metabase.test :as mt]
- [metabase.test.data.users :as test.users])
+ [metabase.test.data.users :as test.users]
+ [metabase.util.log :as log])
(:import
(java.sql SQLException)))
@@ -32,41 +34,40 @@
(deftest ^:parallel exception-response-test
(testing "Should nicely format a chain of exceptions, with the top-level Exception appearing first"
(testing "lowest-level error `:type` should be pulled up to the top-level"
- (let [e1 (ex-info "1" {:level 1})
+ (let [e1 (ex-info "1" {:level 1, :error-code "one"})
e2 (ex-info "2" {:level 2, :type qp.error-type/qp :is-curated true} e1)
- e3 (ex-info "3" {:level 3} e2)]
+ e3 (ex-info "3" {:level 3, :error-code "three"} e2)]
(is (= {:status :failed
:class clojure.lang.ExceptionInfo
:error "1"
:stacktrace true
:error_type :qp
- :ex-data {:level 1}
+ :ex-data {:error-code "one"}
:via [{:status :failed
:class clojure.lang.ExceptionInfo
:error "2"
:stacktrace true
- :ex-data {:level 2, :type :qp, :is-curated true}
:error_type :qp
:error_is_curated true}
{:status :failed
:class clojure.lang.ExceptionInfo
:error "3"
:stacktrace true
- :ex-data {:level 3}}]}
+ :ex-data {:error-code "three"}}]}
(-> (#'catch-exceptions/exception-response e3)
(update :stacktrace sequential?)
(update :via (fn [causes]
(for [cause causes]
(update cause :stacktrace sequential?))))))))))
(testing "SQLExceptions that include stack traces should have them removed"
- (let [e1 (ex-info "mock databricks jdbc driver exception" {:level 1})
+ (let [e1 (ex-info "mock databricks jdbc driver exception" {:level 1, :error-code "one"})
e2 (SQLException. "mock sql exception\n\tat line1\n\tat line2\n\tat line3" ^Throwable e1)
- e3 (ex-info "mock exception" {:level 3} e2)]
+ e3 (ex-info "mock exception" {:level 3, :error-code "three"} e2)]
(is (= {:status :failed
:class clojure.lang.ExceptionInfo
:error "mock sql exception"
:stacktrace true
- :ex-data {:level 1}
+ :ex-data {:error-code "one"}
:via [{:status :failed
:class java.sql.SQLException,
:error "mock sql exception\n\tat line1\n\tat line2\n\tat line3",
@@ -76,7 +77,7 @@
:class clojure.lang.ExceptionInfo
:error "mock exception"
:stacktrace true
- :ex-data {:level 3}}]}
+ :ex-data {:error-code "three"}}]}
(-> (#'catch-exceptions/exception-response e3)
(update :stacktrace sequential?)
(update :via (fn [causes]
@@ -158,13 +159,74 @@
query)]
(is (=? [{:level :error, :message "Error processing query: boom"}]
(messages)))
- (testing "the userland response still has the full detail"
+ (testing "the userland response has the detail, but not the server-side ex-data"
(is (=? {:status :failed
:error "boom"
:stacktrace vector?
- :ex-data {:secret-in-ex-data "hunter2-ex-data"}
:json_query map?}
- result)))))))
+ result))
+ (is (not (contains? result :ex-data))))))))
+
+(deftest ^:parallel response-ex-data-test
+ (testing "the response carries only the client-facing ex-data: the keys `metabase.util.api-error` sends, and `:status-code`"
+ (is (= {:status-code 400, :error-code "some-code", :field 1}
+ (:ex-data (catch-exceptions
+ (fn [] (throw (ex-info "boom" {:status-code 400
+ :error-code "some-code"
+ :field 1
+ :response/keys #{:field}
+ :card-id 2
+ :field-name "PASSWORD"
+ :log/keys #{:field-name}}))))))))
+ (testing "the ex-data of each exception in the chain is filtered"
+ (let [result (catch-exceptions
+ (fn [] (throw (ex-info "outer" {:secret 1}
+ (ex-info "inner" {:secret 2, :error-code "inner"})))))]
+ (is (= {:error-code "inner"} (:ex-data result)))
+ (is (= [nil] (map :ex-data (:via result)))))))
+
+(deftest ^:synchronized dropped-ex-data-debug-log-test
+ (with-redefs [config/is-dev? true]
+ (testing "in dev mode, the ex-data entries left out of the response are logged at debug level, for debugging"
+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :debug]]
+ (catch-exceptions (fn [] (throw (ex-info "boom" {:status-code 400, :card-id 2}))))
+ (is (=? [{:level :debug, :message #"(?s).*:card-id 2.*"}]
+ (filter (comp #{:debug} :level) (messages))))
+ (is (not (re-find #"status-code" (pr-str (messages)))))))
+ (testing "nothing is logged when no entries are left out"
+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :debug]]
+ (catch-exceptions (fn [] (throw (ex-info "boom" {:status-code 400}))))
+ (is (empty? (filter (comp #{:debug} :level) (messages)))))))
+ (with-redefs [config/is-dev? false]
+ (testing "outside dev mode, nothing is logged"
+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :debug]]
+ (catch-exceptions (fn [] (throw (ex-info "boom" {:status-code 400, :card-id 2}))))
+ (is (empty? (filter (comp #{:debug} :level) (messages))))))))
+
+(deftest ^:synchronized log-ex-data-test
+ (testing "the ex-data keys listed under `:log/keys` go to the error log line's context, from anywhere in the chain, and not to the response"
+ (let [contexts (atom [])]
+ (mt/with-dynamic-fn-redefs [log/with-thread-context-fn (fn [context-map f]
+ (swap! contexts conj context-map)
+ (f))]
+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :error]]
+ (let [result (catch-exceptions
+ (fn [] (throw (ex-info "outer" {:card-id 2, :secret "hunter2", :log/keys #{:card-id}}
+ (ex-info "inner" {:field-id 1
+ :field-name "PASSWORD"
+ :log/keys #{:field-id :field-name}})))))]
+ (is (=? [{:level :error, :message "Error processing query: inner"}]
+ (messages)))
+ (is (contains? (set @contexts) {:card-id 2, :field-id 1, :field-name "PASSWORD"}))
+ (is (not (re-find #"hunter2" (pr-str @contexts))))
+ (is (not (re-find #"PASSWORD|field-id|card-id" (pr-str (dissoc result :stacktrace :via)))))
+ (is (not-any? :ex-data (:via result))))))))
+ (testing "a malformed `:log/keys` doesn't mask the original error"
+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :error]]
+ (is (=? {:status :failed, :error "boom"}
+ (catch-exceptions (fn [] (throw (ex-info "boom" {:field-id 1, :log/keys :field-id}))))))
+ (is (=? [{:level :error, :message "Error processing query: boom"}]
+ (messages))))))
(deftest ^:parallel catch-exceptions-test
(testing "include-query-execution-info-test"
@@ -214,20 +276,26 @@
(data-perms/set-table-permission! (perms-group/all-users) (mt/id :venues) :perms/create-queries :query-builder)
(testing (str "If someone doesn't have native query execution permissions, they shouldn't see the native version of "
"the query in the error response")
- (is (=? {:native nil, :preprocessed map?}
- (test.users/with-test-user :rasta
- (qp/process-query
- (qp/userland-query
- (mt/mbql-query venues {:fields [!month.id]})))))))
+ (let [result (test.users/with-test-user :rasta
+ (qp/process-query
+ (qp/userland-query
+ (mt/mbql-query venues {:fields [!month.id]}))))]
+ (is (=? {:native nil, :preprocessed map?}
+ result))
+ (testing "nor the SQL a driver put in its ex-data"
+ (is (not-any? (comp :sql :ex-data) (cons result (:via result)))))))
(testing "They should see it if they have ad-hoc native query perms"
(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)
(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/create-queries :query-builder-and-native)
;; this is not actually a valid query
- (is (=? {:native {:query (str "SELECT DATE_TRUNC('month', \"PUBLIC\".\"VENUES\".\"ID\") AS \"ID\""
- " FROM \"PUBLIC\".\"VENUES\" LIMIT 1048575")
- :params nil}
- :preprocessed map?}
- (test.users/with-test-user :rasta
- (qp/process-query
- (qp/userland-query
- (mt/mbql-query venues {:fields [!month.id]}))))))))))
+ (let [result (test.users/with-test-user :rasta
+ (qp/process-query
+ (qp/userland-query
+ (mt/mbql-query venues {:fields [!month.id]}))))]
+ (is (=? {:native {:query (str "SELECT DATE_TRUNC('month', \"PUBLIC\".\"VENUES\".\"ID\") AS \"ID\""
+ " FROM \"PUBLIC\".\"VENUES\" LIMIT 1048575")
+ :params nil}
+ :preprocessed map?}
+ result))
+ (testing "but not the SQL a driver put in its ex-data, which is server-side detail"
+ (is (not-any? (comp :sql :ex-data) (cons result (:via result))))))))))