Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Don't hand raw driver errors to the model when a Slack upload fails (#81202)

Andrei Simionescu02f5855a35d7
diff --git a/src/metabase/slackbot/uploads.clj b/src/metabase/slackbot/uploads.clj
index 5de6dab641a..94c8eb47794 100644
--- a/src/metabase/slackbot/uploads.clj
+++ b/src/metabase/slackbot/uploads.clj
@@ -57,6 +57,24 @@
(.write out buf 0 n)
(recur total))))))))
+(def ^:private generic-upload-error
+ ;; Handed a bare internal-error note, the model told the user no file had been attached.
+ "Metabase hit an internal error while saving the file, so the upload didn't finish. Ask a Metabase admin to check the server logs.")
+
+(defn- upload-error-message
+ "Returns failure text that is safe to hand to the model.
+ The upload layer's own 4xx errors carry a message written for the user and have no cause. Anything else,
+ including the 4xx that relays a raw driver error along with its cause, gets [[generic-upload-error]], since
+ driver and JDBC messages can name hosts or accounts."
+ [e]
+ (let [{:keys [status-code]} (ex-data e)]
+ (if (and (integer? status-code)
+ (<= 400 status-code 499)
+ (nil? (ex-cause e))
+ (not (str/blank? (ex-message e))))
+ (ex-message e)
+ generic-upload-error)))
+
(defn- upload-settings
"Get upload settings map. Returns nil if uploads are not enabled."
[]
@@ -90,9 +108,9 @@
:model-id (:id result)
:model-name (:name result)}))
(catch Exception e
- (log/warnf "[slackbot] File upload failed: error=%s" (ex-message e))
+ (log/warnf e "[slackbot] File upload failed: error=%s" (ex-message e))
(analytics/inc! :metabase-slackbot/file-uploads {:result "error"})
- {:error (ex-message e) :filename name})
+ {:error (upload-error-message e) :filename name})
(finally
(io/delete-file temp-file true))))))
diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.clj
index 7b5ced39679..6fbdfffd52b 100644
--- a/src/metabase/upload/impl.clj
+++ b/src/metabase/upload/impl.clj
@@ -221,12 +221,13 @@
(for [[value parser] (u/map-all vector row parsers)]
(do
(when-not parser
- (throw (ex-info (format "Column count in data (%s) exceeds the number of in the header (%s)"
- (count rows)
- (count parsers))
- {:settings settings
- :col-upload-types rows
- :row row})))
+ (throw (ex-info (tru "Column count in data ({0}) exceeds the number of columns in the header ({1})"
+ (count row)
+ (count parsers))
+ {:status-code 422
+ :settings settings
+ :col-upload-types col-upload-types
+ :row row})))
(when-not (str/blank? value)
(parser value)))))))
@@ -320,7 +321,7 @@
(InputStreamReader. charset))))
(defn- assert-separator-chosen [s]
- (or s (throw (IllegalArgumentException. "Unable to determine separator"))))
+ (or s (throw (ex-info (tru "Unable to determine separator") {:status-code 422}))))
(defn- infer-separator
"Guess at what symbol is being used as a separator in the given CSV-like file.
@@ -341,6 +342,18 @@
ffirst
assert-separator-chosen)))
+(defn- tag-csv-errors
+ "Rethrows the plain exceptions data.csv uses for malformed input as upload errors, so their message reaches the user."
+ [rows]
+ (lazy-seq
+ (try
+ (when-let [s (seq rows)]
+ (cons (first s) (tag-csv-errors (rest s))))
+ (catch Exception e
+ (if (str/starts-with? (str (ex-message e)) "CSV error")
+ (throw (ex-info (ex-message e) {:status-code 422}))
+ (throw e))))))
+
(defn- infer-parser
"Currently this only infers the separator, but in future it may also handle different quoting options."
[filename ^File file]
@@ -348,7 +361,7 @@
\tab
(infer-separator file))]
(fn [stream]
- (csv/read-csv stream :separator s))))
+ (tag-csv-errors (csv/read-csv stream :separator s)))))
(defn- columns-with-auto-pk [columns]
(merge (ordered-map/ordered-map auto-pk-column-keyword ::upload-types/auto-incrementing-int-pk) columns))
@@ -440,7 +453,9 @@
:size-mb (file-size-mb csv-file)}}
(catch Throwable e
(driver/drop-table! driver (:id db) table-name)
- (throw (ex-info (ex-message e) {:status-code 400} e))))))))
+ (throw (if (:status-code (ex-data e))
+ e
+ (ex-info (ex-message e) {:status-code 400} e)))))))))
;;;; +------------------+
;;;; | Create upload
diff --git a/src/metabase/upload/parsing.clj b/src/metabase/upload/parsing.clj
index 212189b12f2..2055f82f93f 100644
--- a/src/metabase/upload/parsing.clj
+++ b/src/metabase/upload/parsing.clj
@@ -28,8 +28,8 @@
(cond
(re-matches #"(?i)true|t|yes|y|1" s) true
(re-matches #"(?i)false|f|no|n|0" s) false
- :else (throw (IllegalArgumentException.
- (tru "''{0}'' is not a recognizable boolean" s)))))
+ :else (throw (ex-info (tru "''{0}'' is not a recognizable boolean" s)
+ {:status-code 422}))))
(def local-date-patterns
"patterns used to generate the local date formatter. Excludes ISO_LOCAL_DATE (uuuu-MM-dd) because there's
@@ -78,8 +78,8 @@
(try
(LocalDate/parse s local-date-formatter)
(catch Exception _
- (throw (IllegalArgumentException.
- (tru "''{0}'' is not a recognizable date" s))))))
+ (throw (ex-info (tru "''{0}'' is not a recognizable date" s)
+ {:status-code 422})))))
(defn parse-local-datetime
"Parses a string representing a local datetime into a LocalDateTime.
@@ -103,8 +103,8 @@
(try
(parse-local-datetime s)
(catch Exception _
- (throw (IllegalArgumentException.
- (tru "''{0}'' is not a recognizable datetime" s))))))))
+ (throw (ex-info (tru "''{0}'' is not a recognizable datetime" s)
+ {:status-code 422})))))))
(def ^:private auxillary-offset-datetime-formatter
(-> (DateTimeFormatterBuilder.)
@@ -150,7 +150,8 @@
(catch DateTimeParseException _
(OffsetDateTime/parse ss auxillary-offset-datetime-formatter)))
(catch Exception _
- (throw (IllegalArgumentException. (tru "''{0}'' is not a recognizable zoned datetime" s)))))))
+ (throw (ex-info (tru "''{0}'' is not a recognizable zoned datetime" s)
+ {:status-code 422}))))))
(defn- remove-currency-signs
"Remove any recognized currency signs from the string (c.f. [[currency-regex]])."
@@ -201,15 +202,17 @@
(str/trim)
(remove-currency-signs)
(parse-plain-number number-separators))
- (catch Exception e
- (throw (IllegalArgumentException. (tru "''{0}'' is not a recognizable number" s) e)))))
+ (catch Exception _
+ (throw (ex-info (tru "''{0}'' is not a recognizable number" s)
+ {:status-code 422})))))
(defn- parse-as-biginteger
"Parses a string representing a number as a java.math.BigInteger, rounding down if necessary."
[number-separators s]
(let [n (parse-number number-separators s)]
(when-not (zero? (mod n 1))
- (throw (IllegalArgumentException. (tru "''{0}'' is not an integer" s))))
+ (throw (ex-info (tru "''{0}'' is not an integer" s)
+ {:status-code 422})))
(biginteger n)))
(defmulti upload-type->parser
diff --git a/test/metabase/slackbot/uploads_test.clj b/test/metabase/slackbot/uploads_test.clj
index abd9873367a..eaf3db9e3ca 100644
--- a/test/metabase/slackbot/uploads_test.clj
+++ b/test/metabase/slackbot/uploads_test.clj
@@ -54,6 +54,33 @@
(body-fn {:upload-calls upload-calls
:download-calls download-calls}))))
+(defn- failed-upload-result!
+ "Run one CSV file through [[slackbot.uploads/handle-file-uploads]] with
+ `create-csv-upload!` throwing `thrown`."
+ [thrown]
+ (with-upload-mocks!
+ {:uploads-enabled? true}
+ (fn [_]
+ (mt/with-dynamic-fn-redefs [upload.impl/create-csv-upload! (fn [_] (throw thrown))]
+ (slackbot.uploads/handle-file-uploads [tu/slack-csv-file])))))
+
+(deftest csv-upload-failure-message-test
+ (testing "a deliberate upload error keeps its message, filename attached"
+ (let [result (failed-upload-result! (ex-info "Uploads are not enabled." {:status-code 422}))]
+ (is (= "Uploads are not enabled."
+ (get-in result [:upload-result :results 0 :error])))
+ (is (str/includes? (-> result :system-messages first :content)
+ "data.csv: Uploads are not enabled."))))
+ (testing "a driver error rewrapped with its raw message is replaced with the generic line"
+ (let [raw "Connection to db.internal.example.com:5432 refused. Check that the hostname and port are correct."
+ result (failed-upload-result! (ex-info raw {:status-code 400} (java.sql.SQLException. raw)))]
+ (is (= @#'slackbot.uploads/generic-upload-error
+ (get-in result [:upload-result :results 0 :error])))))
+ (testing "a raw driver exception is replaced with the generic line"
+ (let [result (failed-upload-result! (java.sql.SQLException. "FATAL: password authentication failed for user \"metabase\""))]
+ (is (= @#'slackbot.uploads/generic-upload-error
+ (get-in result [:upload-result :results 0 :error]))))))
+
(deftest ^:synchronized csv-upload-disabled-test
(testing "POST /events with file upload when uploads are disabled"
(tu/with-slackbot-setup
diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.clj
index ac3a4127310..fee348c9562 100644
--- a/test/metabase/upload/impl_test.clj
+++ b/test/metabase/upload/impl_test.clj
@@ -1361,6 +1361,26 @@
(= :schema-filters (keyword (:type conn-prop))))
(driver/connection-properties driver))))
+(deftest create-csv-upload!-malformed-csv-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+ (doseq [[description options message]
+ [["a row wider than the header"
+ {:file (csv-file-with ["a,b" "1,2,3"])}
+ #"^Column count in data \(3\) exceeds the number of columns in the header \(2\)$"]
+ ["no separator that parses the first lines"
+ {:file (csv-file-with ["\"a" "1"])}
+ #"^Unable to determine separator"]
+ ["a syntax error past the lines used to pick the separator"
+ {:file (csv-file-with (concat ["a,b"] (repeat 10 "1,2") ["\"3,4"]))}
+ #"^CSV error"]
+ ["a syntax error in a TSV, which skips separator inference"
+ {:file (doto (tmp-file "test" ".tsv") (spit "a\tb\n\"1\t2")), :csv-file-prefix "data.tsv"}
+ #"^CSV error"]]]
+ (testing (str "Upload fails with a 422 and the message the user can act on, given " description)
+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo message
+ (do-with-uploaded-example-csv! options identity)))]
+ (is (= 422 (:status-code (ex-data e)))))))))
+
(deftest create-csv-upload!-schema-does-not-sync-test
;; We only need to test this for a single driver, and the way this test has been written is coupled to Postgres
(mt/test-driver :postgres