The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve query error responses and logs (#84585)
Nathan Voxland1bf41894f504
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsx b/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsxindex fc35ef3ff12..b9aa81a4882 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsx+++ b/frontend/src/metabase/querying/components/QueryVisualization/QueryVisualization.tsx@@ -77,7 +77,6 @@ export function QueryVisualization(props: QueryVisualizationProps) {className={S.spread}error={result.error}errorType={result.error_type}- via={result.via ?? []}question={question}duration={result.duration ?? 0}errorAction={errorAction}diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsxindex fc763241ca1..c9ea21b3ecc 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx@@ -1,5 +1,4 @@import cx from "classnames";-import { getIn } from "icepick";import type { ReactNode } from "react";import { t } from "ttag";@@ -23,11 +22,10 @@ import { VISUALIZATION_SLOW_TIMEOUT } from "../../../constants";import VisErrorS from "./VisualizationError.module.css";import { AdminEmail } from "./components";-import { adjustPositions, getDatasetErrorMessage, stripRemarks } from "./utils";+import { getDatasetErrorMessage, stripRemarks } from "./utils";interface VisualizationErrorProps {className?: string;- via: Record<string, any>[];question: Question;duration: number;error: DatasetError;@@ -37,7 +35,6 @@ interface VisualizationErrorProps {export function VisualizationError({className,- via,question,duration,error,@@ -139,12 +136,7 @@ export function VisualizationError({if (isNative) {// always show errors for native queries- let processedError = errorMessage;- const origSql = getIn(via, [(via || "").length - 1, "ex-data", "sql"]);- if (typeof origSql === "string" && processedError) {- processedError = adjustPositions(processedError, origSql);- }- processedError = stripRemarks(processedError);+ const processedError = stripRemarks(errorMessage);const database = question.database();const isSql = database && getEngineNativeType(database.engine) === "sql";diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsxindex d39d27fb200..4aeb26f679a 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx@@ -76,7 +76,6 @@ export const setup = ({error={error as DatasetError}errorType={errorType}errorAction={errorAction}- via={[]}/>,{ storeInitialState: state },);diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.tsindex ef5b4ce0d65..01af90a32fd 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts@@ -1,42 +1,4 @@-import { adjustPositions, stripRemarks } from "../utils";--describe("adjustPositions", () => {- const remarkedQuery =- "-- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a\nfwefwef";- const remarkedRedshiftQuery =- "/* anything before the remarks in multiline is redshift */-- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a\nfwefwef";- const unremarkedQuery = "fwefwef";-- const unadjustedError = `boopy boop boopy boop fake error message Position: 1000;`;- const adjustedError = `boopy boop boopy boop fake error message Position: 881;`;- const redshiftAdjustedError = `boopy boop boopy boop fake error message Position: 823;`;-- const errorLineNumbers = "boopy boop boopy boop fake error message Line: 2";-- it("error adjusted pg", () => {- expect(adjustPositions(unadjustedError, remarkedQuery)).toEqual(- adjustedError,- );- });-- it("error adjusted redshift", () => {- expect(adjustPositions(unadjustedError, remarkedRedshiftQuery)).toEqual(- redshiftAdjustedError,- );- });-- it("unremarked query should be a noop", () => {- expect(adjustPositions(unadjustedError, unremarkedQuery)).toEqual(- unadjustedError,- );- });-- it("error adjusted line numbers should be a noop", () => {- expect(adjustPositions(errorLineNumbers, remarkedQuery)).toEqual(- errorLineNumbers,- );- });-});+import { stripRemarks } from "../utils";describe("stripRemarks", () => {const errorH2Unstripped = `diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.tsindex 81dc9342f29..aa2dc000136 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts@@ -13,41 +13,6 @@ export function getDatasetErrorMessage(error: DatasetError): string {: "";}-export function adjustPositions(error: string, origSql: string): string {- /* Positions in error messages are borked coming in for Postgres errors.- * Previously, you would see "blahblahblah bombed out, Position: 119" in a 10-character invalid query.- * This is because MB shoves in 'remarks' into the original query and we get the exception from the query with remarks.- * This function adjusts the value of the positions in the exception message to account for this.- * This is done in mildly scary kludge here in frontend after everything,- * because the alternative of doing it in backend- * is an absolutely terrifying kludge involving messing with exceptions.- */- let adjustmentLength = 0;-- // redshift remarks use c-style multiline comments...- const multiLineBeginPos = origSql.search("/\\*");- const multiLineEndPos = origSql.search("\\*/");- // if multiLineBeginPos is 0 then we know it's a redshift remark- if (multiLineBeginPos === 0 && multiLineEndPos !== -1) {- adjustmentLength += multiLineEndPos + 2; // 2 for */ in itself- }-- const chompedSql = origSql.substr(adjustmentLength);- // there also seem to be cases where remarks don't get in...- const commentPos = chompedSql.search("--");- const newLinePos = chompedSql.search("\n");- // 5 is a heuristic: this indicates that this is almost certainly an initial remark comment- if (commentPos !== -1 && commentPos < 5) {- // There will be a \n after the redshift comment,- // which is why there needs to be a 2 added- adjustmentLength += newLinePos + 2;- }-- return error.replace(/Position: (\d+)/, function (_, p1) {- return "Position: " + (parseInt(p1) - adjustmentLength);- });-}-export function stripRemarks(error: string | undefined): string {/* SQL snippets in error messages are borked coming in for errors in many DBs.* You're expecting something with just your sql in the message,diff --git a/src/metabase/query_processor/middleware/catch_exceptions.clj b/src/metabase/query_processor/middleware/catch_exceptions.cljindex 35f7dfc9e7c..7d2a5e7cd79 100644--- a/src/metabase/query_processor/middleware/catch_exceptions.clj+++ b/src/metabase/query_processor/middleware/catch_exceptions.clj@@ -1,9 +1,19 @@(ns metabase.query-processor.middleware.catch-exceptions- "Middleware for catching exceptions thrown by the query processor and returning them in a friendlier format."- (:refer-clojure :exclude [some get-in])+ "Middleware for catching exceptions thrown by the query processor and returning them in a friendlier format.++ The response carries only the client-facing part of each exception's ex-data (see [[metabase.util.api-error]]). The+ error log line carries just the message, plus the ex-data keys listed under `:log/keys`, as log context:++ (throw (ex-info (tru \"Field is not available\")+ {:status-code 400, :field-id id, :field-name field-name, :log/keys #{:field-id :field-name}}))++ responds without either key and logs `field-id` and `field-name` alongside the message. Don't list the query or+ values from it."+ (:refer-clojure :exclude [some get-in not-empty select-keys])(:require[clojure.string :as str][metabase.analytics-interface.core :as analytics]+ [metabase.config.core :as config][metabase.driver :as driver][metabase.lib.schema.common :as lib.schema.common][metabase.queries.schema :as queries.schema]@@ -14,11 +24,12 @@[metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.schema :as qp.schema][metabase.util :as u]+ [metabase.util.api-error :as api-error][metabase.util.i18n :refer [trs]][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.performance :refer [some get-in]])+ [metabase.util.performance :refer [some get-in not-empty select-keys]])(:import(clojure.lang ExceptionInfo)(java.sql SQLException)))@@ -41,19 +52,31 @@[^InterruptedException _e]{:status :interrupted})+(defn- response-ex-data+ "The part of an exception's `data` sent to the client: what [[api-error/response-data]] sends, and `:status-code`,+ which [[metabase.query-processor.streaming]] turns into the HTTP status."+ [data]+ (not-empty (merge (api-error/response-data data)+ (select-keys data [:status-code]))))+(defmethod format-exception ExceptionInfo[e];; `:is-curated` is a flag that signals whether the error message in `e` was approved by product;; to be shown to the user. It is used by FE.- (let [{error-type :type, is-curated :is-curated, :as data} (ex-data e)]+ (let [{error-type :type, is-curated :is-curated, :as data} (ex-data e)+ response-data (response-ex-data data)]+ (when-let [dropped (and config/is-dev?+ (not-empty (apply dissoc data (keys response-data))))]+ (log/debugf "Query error response leaves out ex-data: %s" (pr-str dropped)))(merge((get-method format-exception Throwable) e)(when (qp.error-type/known-error-type? error-type){:error_type error-type})(when is-curated{:error_is_curated is-curated})- ;; TODO - we should probably change this key to `:data` so we're not mixing lisp-case and snake_case keys- {:ex-data data})))+ (when response-data+ ;; TODO - we should probably change this key to `:data` so we're not mixing lisp-case and snake_case keys+ {:ex-data response-data}))))(defmethod format-exception SQLException[^SQLException e]@@ -151,6 +174,19 @@(catch Throwable e(assoc (Throwable->map e) :status :failed))))+(defn- log-context+ "The ex-data entries each exception in `e`'s chain lists under `:log/keys`, for the error log line. An inner+ exception's entries win over an outer one's. Empty when a `:log/keys` is malformed, so that it can't mask `e`."+ [e]+ (try+ (into {}+ (map (fn [ex]+ (let [data (ex-data ex)]+ (select-keys data (:log/keys data)))))+ (u/full-exception-chain e))+ (catch Throwable _+ {})))+(mu/defn catch-exceptions :- ::qp.schema/qp"Middleware for catching exceptions thrown by the query processor and returning them in a 'normal' format. Forwardsexceptions to the `result-chan`."@@ -175,8 +211,9 @@pool-saturated? (qp.error-type/connection-pool-saturated?(:error_type formatted-exception))](when-not (or query-canceled? pool-saturated?)- (log/errorf "Error processing query: %s"- (or (:error formatted-exception) "Error running query")))+ (log/with-thread-context (log-context e)+ (log/errorf "Error processing query: %s"+ (or (:error formatted-exception) "Error running query"))));; ensure always a message on the error otherwise FE thinks query was successful. (#23258, #23281)(let [result (update formatted-exception:error (fnil identity (trs "Error running query")))]diff --git a/test/metabase/documents/api/document_test.clj b/test/metabase/documents/api/document_test.cljindex ca22899ef42..df38bb8ba7f 100644--- a/test/metabase/documents/api/document_test.clj+++ b/test/metabase/documents/api/document_test.clj@@ -2335,9 +2335,7 @@(data-perms/set-database-permission! all-users-group (mt/id) :perms/download-results :no)(is (malli= [:map[:status [:= "failed"]]- [:error_type [:= "missing-required-permissions"]]- [:ex-data [:map- [:permissions-error? [:= true]]]]]+ [:error_type [:= "missing-required-permissions"]]](mt/user-http-request :rasta:post 403(format "document/%s/card/%s/query/csv" doc-id card-id)diff --git a/test/metabase/query_processor/middleware/catch_exceptions_test.clj b/test/metabase/query_processor/middleware/catch_exceptions_test.cljindex 2595b34ecd5..da26990e71a 100644--- a/test/metabase/query_processor/middleware/catch_exceptions_test.clj+++ b/test/metabase/query_processor/middleware/catch_exceptions_test.clj@@ -3,6 +3,7 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.catch-exceptions-test]}}}}}}(:require[clojure.test :refer :all]+ [metabase.config.core :as config][metabase.driver :as driver][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -17,7 +18,8 @@[metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.test :as qp][metabase.test :as mt]- [metabase.test.data.users :as test.users])+ [metabase.test.data.users :as test.users]+ [metabase.util.log :as log])(:import(java.sql SQLException)))@@ -32,41 +34,40 @@(deftest ^:parallel exception-response-test(testing "Should nicely format a chain of exceptions, with the top-level Exception appearing first"(testing "lowest-level error `:type` should be pulled up to the top-level"- (let [e1 (ex-info "1" {:level 1})+ (let [e1 (ex-info "1" {:level 1, :error-code "one"})e2 (ex-info "2" {:level 2, :type qp.error-type/qp :is-curated true} e1)- e3 (ex-info "3" {:level 3} e2)]+ e3 (ex-info "3" {:level 3, :error-code "three"} e2)](is (= {:status :failed:class clojure.lang.ExceptionInfo:error "1":stacktrace true:error_type :qp- :ex-data {:level 1}+ :ex-data {:error-code "one"}:via [{:status :failed:class clojure.lang.ExceptionInfo:error "2":stacktrace true- :ex-data {:level 2, :type :qp, :is-curated true}:error_type :qp:error_is_curated true}{:status :failed:class clojure.lang.ExceptionInfo:error "3":stacktrace true- :ex-data {:level 3}}]}+ :ex-data {:error-code "three"}}]}(-> (#'catch-exceptions/exception-response e3)(update :stacktrace sequential?)(update :via (fn [causes](for [cause causes](update cause :stacktrace sequential?))))))))))(testing "SQLExceptions that include stack traces should have them removed"- (let [e1 (ex-info "mock databricks jdbc driver exception" {:level 1})+ (let [e1 (ex-info "mock databricks jdbc driver exception" {:level 1, :error-code "one"})e2 (SQLException. "mock sql exception\n\tat line1\n\tat line2\n\tat line3" ^Throwable e1)- e3 (ex-info "mock exception" {:level 3} e2)]+ e3 (ex-info "mock exception" {:level 3, :error-code "three"} e2)](is (= {:status :failed:class clojure.lang.ExceptionInfo:error "mock sql exception":stacktrace true- :ex-data {:level 1}+ :ex-data {:error-code "one"}:via [{:status :failed:class java.sql.SQLException,:error "mock sql exception\n\tat line1\n\tat line2\n\tat line3",@@ -76,7 +77,7 @@:class clojure.lang.ExceptionInfo:error "mock exception":stacktrace true- :ex-data {:level 3}}]}+ :ex-data {:error-code "three"}}]}(-> (#'catch-exceptions/exception-response e3)(update :stacktrace sequential?)(update :via (fn [causes]@@ -158,13 +159,74 @@query)](is (=? [{:level :error, :message "Error processing query: boom"}](messages)))- (testing "the userland response still has the full detail"+ (testing "the userland response has the detail, but not the server-side ex-data"(is (=? {:status :failed:error "boom":stacktrace vector?- :ex-data {:secret-in-ex-data "hunter2-ex-data"}:json_query map?}- result)))))))+ result))+ (is (not (contains? result :ex-data))))))))++(deftest ^:parallel response-ex-data-test+ (testing "the response carries only the client-facing ex-data: the keys `metabase.util.api-error` sends, and `:status-code`"+ (is (= {:status-code 400, :error-code "some-code", :field 1}+ (:ex-data (catch-exceptions+ (fn [] (throw (ex-info "boom" {:status-code 400+ :error-code "some-code"+ :field 1+ :response/keys #{:field}+ :card-id 2+ :field-name "PASSWORD"+ :log/keys #{:field-name}}))))))))+ (testing "the ex-data of each exception in the chain is filtered"+ (let [result (catch-exceptions+ (fn [] (throw (ex-info "outer" {:secret 1}+ (ex-info "inner" {:secret 2, :error-code "inner"})))))]+ (is (= {:error-code "inner"} (:ex-data result)))+ (is (= [nil] (map :ex-data (:via result)))))))++(deftest ^:synchronized dropped-ex-data-debug-log-test+ (with-redefs [config/is-dev? true]+ (testing "in dev mode, the ex-data entries left out of the response are logged at debug level, for debugging"+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :debug]]+ (catch-exceptions (fn [] (throw (ex-info "boom" {:status-code 400, :card-id 2}))))+ (is (=? [{:level :debug, :message #"(?s).*:card-id 2.*"}]+ (filter (comp #{:debug} :level) (messages))))+ (is (not (re-find #"status-code" (pr-str (messages)))))))+ (testing "nothing is logged when no entries are left out"+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :debug]]+ (catch-exceptions (fn [] (throw (ex-info "boom" {:status-code 400}))))+ (is (empty? (filter (comp #{:debug} :level) (messages)))))))+ (with-redefs [config/is-dev? false]+ (testing "outside dev mode, nothing is logged"+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :debug]]+ (catch-exceptions (fn [] (throw (ex-info "boom" {:status-code 400, :card-id 2}))))+ (is (empty? (filter (comp #{:debug} :level) (messages))))))))++(deftest ^:synchronized log-ex-data-test+ (testing "the ex-data keys listed under `:log/keys` go to the error log line's context, from anywhere in the chain, and not to the response"+ (let [contexts (atom [])]+ (mt/with-dynamic-fn-redefs [log/with-thread-context-fn (fn [context-map f]+ (swap! contexts conj context-map)+ (f))]+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :error]]+ (let [result (catch-exceptions+ (fn [] (throw (ex-info "outer" {:card-id 2, :secret "hunter2", :log/keys #{:card-id}}+ (ex-info "inner" {:field-id 1+ :field-name "PASSWORD"+ :log/keys #{:field-id :field-name}})))))]+ (is (=? [{:level :error, :message "Error processing query: inner"}]+ (messages)))+ (is (contains? (set @contexts) {:card-id 2, :field-id 1, :field-name "PASSWORD"}))+ (is (not (re-find #"hunter2" (pr-str @contexts))))+ (is (not (re-find #"PASSWORD|field-id|card-id" (pr-str (dissoc result :stacktrace :via)))))+ (is (not-any? :ex-data (:via result))))))))+ (testing "a malformed `:log/keys` doesn't mask the original error"+ (mt/with-log-messages-for-level [messages [metabase.query-processor.middleware.catch-exceptions :error]]+ (is (=? {:status :failed, :error "boom"}+ (catch-exceptions (fn [] (throw (ex-info "boom" {:field-id 1, :log/keys :field-id}))))))+ (is (=? [{:level :error, :message "Error processing query: boom"}]+ (messages))))))(deftest ^:parallel catch-exceptions-test(testing "include-query-execution-info-test"@@ -214,20 +276,26 @@(data-perms/set-table-permission! (perms-group/all-users) (mt/id :venues) :perms/create-queries :query-builder)(testing (str "If someone doesn't have native query execution permissions, they shouldn't see the native version of ""the query in the error response")- (is (=? {:native nil, :preprocessed map?}- (test.users/with-test-user :rasta- (qp/process-query- (qp/userland-query- (mt/mbql-query venues {:fields [!month.id]})))))))+ (let [result (test.users/with-test-user :rasta+ (qp/process-query+ (qp/userland-query+ (mt/mbql-query venues {:fields [!month.id]}))))]+ (is (=? {:native nil, :preprocessed map?}+ result))+ (testing "nor the SQL a driver put in its ex-data"+ (is (not-any? (comp :sql :ex-data) (cons result (:via result)))))))(testing "They should see it if they have ad-hoc native query perms"(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/create-queries :query-builder-and-native);; this is not actually a valid query- (is (=? {:native {:query (str "SELECT DATE_TRUNC('month', \"PUBLIC\".\"VENUES\".\"ID\") AS \"ID\""- " FROM \"PUBLIC\".\"VENUES\" LIMIT 1048575")- :params nil}- :preprocessed map?}- (test.users/with-test-user :rasta- (qp/process-query- (qp/userland-query- (mt/mbql-query venues {:fields [!month.id]}))))))))))+ (let [result (test.users/with-test-user :rasta+ (qp/process-query+ (qp/userland-query+ (mt/mbql-query venues {:fields [!month.id]}))))]+ (is (=? {:native {:query (str "SELECT DATE_TRUNC('month', \"PUBLIC\".\"VENUES\".\"ID\") AS \"ID\""+ " FROM \"PUBLIC\".\"VENUES\" LIMIT 1048575")+ :params nil}+ :preprocessed map?}+ result))+ (testing "but not the SQL a driver put in its ex-data, which is server-side detail"+ (is (not-any? (comp :sql :ex-data) (cons result (:via result))))))))))