The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Don't hand raw driver errors to the model when a Slack upload fails (#81202)
Andrei Simionescu02f5855a35d7
diff --git a/src/metabase/slackbot/uploads.clj b/src/metabase/slackbot/uploads.cljindex 5de6dab641a..94c8eb47794 100644--- a/src/metabase/slackbot/uploads.clj+++ b/src/metabase/slackbot/uploads.clj@@ -57,6 +57,24 @@(.write out buf 0 n)(recur total))))))))+(def ^:private generic-upload-error+ ;; Handed a bare internal-error note, the model told the user no file had been attached.+ "Metabase hit an internal error while saving the file, so the upload didn't finish. Ask a Metabase admin to check the server logs.")++(defn- upload-error-message+ "Returns failure text that is safe to hand to the model.+ The upload layer's own 4xx errors carry a message written for the user and have no cause. Anything else,+ including the 4xx that relays a raw driver error along with its cause, gets [[generic-upload-error]], since+ driver and JDBC messages can name hosts or accounts."+ [e]+ (let [{:keys [status-code]} (ex-data e)]+ (if (and (integer? status-code)+ (<= 400 status-code 499)+ (nil? (ex-cause e))+ (not (str/blank? (ex-message e))))+ (ex-message e)+ generic-upload-error)))+(defn- upload-settings"Get upload settings map. Returns nil if uploads are not enabled."[]@@ -90,9 +108,9 @@:model-id (:id result):model-name (:name result)}))(catch Exception e- (log/warnf "[slackbot] File upload failed: error=%s" (ex-message e))+ (log/warnf e "[slackbot] File upload failed: error=%s" (ex-message e))(analytics/inc! :metabase-slackbot/file-uploads {:result "error"})- {:error (ex-message e) :filename name})+ {:error (upload-error-message e) :filename name})(finally(io/delete-file temp-file true))))))diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.cljindex 7b5ced39679..6fbdfffd52b 100644--- a/src/metabase/upload/impl.clj+++ b/src/metabase/upload/impl.clj@@ -221,12 +221,13 @@(for [[value parser] (u/map-all vector row parsers)](do(when-not parser- (throw (ex-info (format "Column count in data (%s) exceeds the number of in the header (%s)"- (count rows)- (count parsers))- {:settings settings- :col-upload-types rows- :row row})))+ (throw (ex-info (tru "Column count in data ({0}) exceeds the number of columns in the header ({1})"+ (count row)+ (count parsers))+ {:status-code 422+ :settings settings+ :col-upload-types col-upload-types+ :row row})))(when-not (str/blank? value)(parser value)))))))@@ -320,7 +321,7 @@(InputStreamReader. charset))))(defn- assert-separator-chosen [s]- (or s (throw (IllegalArgumentException. "Unable to determine separator"))))+ (or s (throw (ex-info (tru "Unable to determine separator") {:status-code 422}))))(defn- infer-separator"Guess at what symbol is being used as a separator in the given CSV-like file.@@ -341,6 +342,18 @@ffirstassert-separator-chosen)))+(defn- tag-csv-errors+ "Rethrows the plain exceptions data.csv uses for malformed input as upload errors, so their message reaches the user."+ [rows]+ (lazy-seq+ (try+ (when-let [s (seq rows)]+ (cons (first s) (tag-csv-errors (rest s))))+ (catch Exception e+ (if (str/starts-with? (str (ex-message e)) "CSV error")+ (throw (ex-info (ex-message e) {:status-code 422}))+ (throw e))))))+(defn- infer-parser"Currently this only infers the separator, but in future it may also handle different quoting options."[filename ^File file]@@ -348,7 +361,7 @@\tab(infer-separator file))](fn [stream]- (csv/read-csv stream :separator s))))+ (tag-csv-errors (csv/read-csv stream :separator s)))))(defn- columns-with-auto-pk [columns](merge (ordered-map/ordered-map auto-pk-column-keyword ::upload-types/auto-incrementing-int-pk) columns))@@ -440,7 +453,9 @@:size-mb (file-size-mb csv-file)}}(catch Throwable e(driver/drop-table! driver (:id db) table-name)- (throw (ex-info (ex-message e) {:status-code 400} e))))))))+ (throw (if (:status-code (ex-data e))+ e+ (ex-info (ex-message e) {:status-code 400} e)))))))));;;; +------------------+;;;; | Create uploaddiff --git a/src/metabase/upload/parsing.clj b/src/metabase/upload/parsing.cljindex 212189b12f2..2055f82f93f 100644--- a/src/metabase/upload/parsing.clj+++ b/src/metabase/upload/parsing.clj@@ -28,8 +28,8 @@(cond(re-matches #"(?i)true|t|yes|y|1" s) true(re-matches #"(?i)false|f|no|n|0" s) false- :else (throw (IllegalArgumentException.- (tru "''{0}'' is not a recognizable boolean" s)))))+ :else (throw (ex-info (tru "''{0}'' is not a recognizable boolean" s)+ {:status-code 422}))))(def local-date-patterns"patterns used to generate the local date formatter. Excludes ISO_LOCAL_DATE (uuuu-MM-dd) because there's@@ -78,8 +78,8 @@(try(LocalDate/parse s local-date-formatter)(catch Exception _- (throw (IllegalArgumentException.- (tru "''{0}'' is not a recognizable date" s))))))+ (throw (ex-info (tru "''{0}'' is not a recognizable date" s)+ {:status-code 422})))))(defn parse-local-datetime"Parses a string representing a local datetime into a LocalDateTime.@@ -103,8 +103,8 @@(try(parse-local-datetime s)(catch Exception _- (throw (IllegalArgumentException.- (tru "''{0}'' is not a recognizable datetime" s))))))))+ (throw (ex-info (tru "''{0}'' is not a recognizable datetime" s)+ {:status-code 422})))))))(def ^:private auxillary-offset-datetime-formatter(-> (DateTimeFormatterBuilder.)@@ -150,7 +150,8 @@(catch DateTimeParseException _(OffsetDateTime/parse ss auxillary-offset-datetime-formatter)))(catch Exception _- (throw (IllegalArgumentException. (tru "''{0}'' is not a recognizable zoned datetime" s)))))))+ (throw (ex-info (tru "''{0}'' is not a recognizable zoned datetime" s)+ {:status-code 422}))))))(defn- remove-currency-signs"Remove any recognized currency signs from the string (c.f. [[currency-regex]])."@@ -201,15 +202,17 @@(str/trim)(remove-currency-signs)(parse-plain-number number-separators))- (catch Exception e- (throw (IllegalArgumentException. (tru "''{0}'' is not a recognizable number" s) e)))))+ (catch Exception _+ (throw (ex-info (tru "''{0}'' is not a recognizable number" s)+ {:status-code 422})))))(defn- parse-as-biginteger"Parses a string representing a number as a java.math.BigInteger, rounding down if necessary."[number-separators s](let [n (parse-number number-separators s)](when-not (zero? (mod n 1))- (throw (IllegalArgumentException. (tru "''{0}'' is not an integer" s))))+ (throw (ex-info (tru "''{0}'' is not an integer" s)+ {:status-code 422})))(biginteger n)))(defmulti upload-type->parserdiff --git a/test/metabase/slackbot/uploads_test.clj b/test/metabase/slackbot/uploads_test.cljindex abd9873367a..eaf3db9e3ca 100644--- a/test/metabase/slackbot/uploads_test.clj+++ b/test/metabase/slackbot/uploads_test.clj@@ -54,6 +54,33 @@(body-fn {:upload-calls upload-calls:download-calls download-calls}))))+(defn- failed-upload-result!+ "Run one CSV file through [[slackbot.uploads/handle-file-uploads]] with+ `create-csv-upload!` throwing `thrown`."+ [thrown]+ (with-upload-mocks!+ {:uploads-enabled? true}+ (fn [_]+ (mt/with-dynamic-fn-redefs [upload.impl/create-csv-upload! (fn [_] (throw thrown))]+ (slackbot.uploads/handle-file-uploads [tu/slack-csv-file])))))++(deftest csv-upload-failure-message-test+ (testing "a deliberate upload error keeps its message, filename attached"+ (let [result (failed-upload-result! (ex-info "Uploads are not enabled." {:status-code 422}))]+ (is (= "Uploads are not enabled."+ (get-in result [:upload-result :results 0 :error])))+ (is (str/includes? (-> result :system-messages first :content)+ "data.csv: Uploads are not enabled."))))+ (testing "a driver error rewrapped with its raw message is replaced with the generic line"+ (let [raw "Connection to db.internal.example.com:5432 refused. Check that the hostname and port are correct."+ result (failed-upload-result! (ex-info raw {:status-code 400} (java.sql.SQLException. raw)))]+ (is (= @#'slackbot.uploads/generic-upload-error+ (get-in result [:upload-result :results 0 :error])))))+ (testing "a raw driver exception is replaced with the generic line"+ (let [result (failed-upload-result! (java.sql.SQLException. "FATAL: password authentication failed for user \"metabase\""))]+ (is (= @#'slackbot.uploads/generic-upload-error+ (get-in result [:upload-result :results 0 :error]))))))+(deftest ^:synchronized csv-upload-disabled-test(testing "POST /events with file upload when uploads are disabled"(tu/with-slackbot-setupdiff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.cljindex ac3a4127310..fee348c9562 100644--- a/test/metabase/upload/impl_test.clj+++ b/test/metabase/upload/impl_test.clj@@ -1361,6 +1361,26 @@(= :schema-filters (keyword (:type conn-prop))))(driver/connection-properties driver))))+(deftest create-csv-upload!-malformed-csv-test+ (mt/test-drivers (mt/normal-drivers-with-feature :uploads)+ (doseq [[description options message]+ [["a row wider than the header"+ {:file (csv-file-with ["a,b" "1,2,3"])}+ #"^Column count in data \(3\) exceeds the number of columns in the header \(2\)$"]+ ["no separator that parses the first lines"+ {:file (csv-file-with ["\"a" "1"])}+ #"^Unable to determine separator"]+ ["a syntax error past the lines used to pick the separator"+ {:file (csv-file-with (concat ["a,b"] (repeat 10 "1,2") ["\"3,4"]))}+ #"^CSV error"]+ ["a syntax error in a TSV, which skips separator inference"+ {:file (doto (tmp-file "test" ".tsv") (spit "a\tb\n\"1\t2")), :csv-file-prefix "data.tsv"}+ #"^CSV error"]]]+ (testing (str "Upload fails with a 422 and the message the user can act on, given " description)+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo message+ (do-with-uploaded-example-csv! options identity)))]+ (is (= 422 (:status-code (ex-data e)))))))))+(deftest create-csv-upload!-schema-does-not-sync-test;; We only need to test this for a single driver, and the way this test has been written is coupled to Postgres(mt/test-driver :postgres