Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Various Slackbot upload improvements (#82216)

Chris Trutere28ae43d1d07
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 8ba10782b85..87ddb095670 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1594,10 +1594,7 @@
system
upload
util}
- :model-imports #{:model/AuthIdentity
- :model/Card
- :model/Database
- :model/MetabotMessage}}
+ :model-imports #{:model/AuthIdentity :model/Card :model/MetabotMessage}}
metrics
{:team "UX West"
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 9932b16f88a..d2904c2029f 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -732,7 +732,7 @@
(prometheus/counter :metabase-slackbot/responses-deleted
{:description "Number of Slack bot responses deleted by users."})
(prometheus/counter :metabase-slackbot/file-uploads
- {:description "Number of file uploads via the Slack bot."
+ {:description "Number of files attached to Slack bot messages, by what became of each."
:labels [:result]})
(prometheus/counter :metabase-slackbot/responses-truncated
{:description (str "Number of Slack bot responses truncated because they exceeded "
diff --git a/src/metabase/slackbot/api.clj b/src/metabase/slackbot/api.clj
index 7f74f65d3af..ceff2622cf5 100644
--- a/src/metabase/slackbot/api.clj
+++ b/src/metabase/slackbot/api.clj
@@ -163,54 +163,17 @@
:headers {"Content-Type" "text/plain"}
:body "ok"})
-(defn- all-files-skipped?
- "Returns true if nothing was uploaded and at least one file was refused."
- [{:keys [upload-result]}]
- (let [{:keys [results skipped remote]} upload-result]
- (and (empty? results)
- (or (seq skipped) (seq remote)))))
-
-(defn- refused-files-message
- "Reply text for a share where nothing could be uploaded, naming why for each file."
- [{:keys [skipped remote]}]
- (str/join " "
- (cond-> []
- (seq skipped)
- (conj (format "I can only process CSV and TSV files. The following files were skipped: %s"
- (str/join ", " skipped)))
-
- (seq remote)
- (conj (format "These files are stored outside Slack, so I can't upload them: %s"
- (str/join ", " remote))))))
-
(mu/defn- handle-message-file-share
- "Process a file_share message - handles CSV uploads"
+ "Handle a Slack message with file attachments."
[client :- slackbot.client/SlackClient
event :- slackbot.events/SlackMessageFileShareEvent]
- (let [files (:files event)
- text (:text event)
- has-text? (not (str/blank? text))
- file-handling (when (seq files)
- (slackbot.uploads/handle-file-uploads files))
- extra-history (cond
- ;; Pre-flight error (uploads disabled, no permission)
- (:error file-handling)
- [{:role :assistant
- :content (:error file-handling)}]
-
- ;; Upload results to communicate to AI
- (:system-messages file-handling)
- (:system-messages file-handling))
- all-skipped? (all-files-skipped? file-handling)
- should-skip-ai? (and (not has-text?)
- (not (:error file-handling))
- all-skipped?)]
- ;; If nothing could be uploaded and there's no text, respond directly
- ;; without calling the AI to avoid sending an empty prompt
- (if should-skip-ai?
- (slackbot.client/post-message client
- (merge (slackbot.events/event->reply-context event)
- {:text (refused-files-message (:upload-result file-handling))}))
+ (let [extra-history (slackbot.uploads/handle-file-uploads! client (:files event))]
+ ;; When a message contains only attachments, reply directly instead of sending an empty prompt to the AI.
+ ;; A message carrying neither text nor a file leaves nothing to reply with, so say nothing at all.
+ (if (str/blank? (:text event))
+ (when-let [text (not-empty (str/join "\n\n" (map :content extra-history)))]
+ (slackbot.client/post-message client
+ (assoc (slackbot.events/event->reply-context event) :text text)))
(slackbot.streaming/send-response client event extra-history))))
(defmethod analytics.core/known-labels :metabase-slackbot/responses-generated [_]
@@ -220,7 +183,12 @@
{:source "channel" :result "error"}])
(defmethod analytics.core/known-labels :metabase-slackbot/file-uploads [_]
- [{:result "success"} {:result "error"}])
+ [{:result "success"}
+ {:result "error"}
+ {:result "too-large"}
+ {:result "unsupported"}
+ {:result "remote"}
+ {:result "unavailable"}])
(defn- event-source
"Return the source label for a Slack event: \"dm\" or \"channel\"."
diff --git a/src/metabase/slackbot/client.clj b/src/metabase/slackbot/client.clj
index 11190ee2339..eae8782dc45 100644
--- a/src/metabase/slackbot/client.clj
+++ b/src/metabase/slackbot/client.clj
@@ -237,12 +237,19 @@
#{301 302 303 307 308})
(defn- redirect-target
- "The absolute URL `resp` redirects to, resolved against the `url` that produced it, or nil if it is not a redirect."
+ "The absolute URL `resp` redirects to, resolved against the `url` that produced it.
+ Nil if `resp` is not a redirect, carries no `Location`, or its `Location` does not resolve to a URL."
[url resp]
(when (contains? redirect-statuses (:status resp))
(when-let [location (or (get-in resp [:headers "location"])
(get-in resp [:headers "Location"]))]
- (str (.resolve (URI. (str url)) (str location))))))
+ ;; The caller reads this with the response body still open, and only its own branches close it.
+ ;; An unresolvable `Location` therefore has to read as no redirect rather than throw.
+ (try
+ (str (.resolve (URI. (str url)) (str location)))
+ (catch Exception e
+ (log/debugf "[slackbot] Ignoring an unresolvable redirect Location: %s" (ex-message e))
+ nil)))))
(defn download-file-stream
"Download a file from Slack, returning an InputStream instead of buffering in memory.
diff --git a/src/metabase/slackbot/db.clj b/src/metabase/slackbot/db.clj
index 2d6c400cb74..4c43e950cf4 100644
--- a/src/metabase/slackbot/db.clj
+++ b/src/metabase/slackbot/db.clj
@@ -67,8 +67,3 @@
"The Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
(t2/select-one :model/Card :id card-id))
-
-(mu/defn database
- "The Database with `database-id`, or nil."
- [database-id :- ::lib.schema.id/database]
- (t2/select-one :model/Database :id database-id))
diff --git a/src/metabase/slackbot/uploads.clj b/src/metabase/slackbot/uploads.clj
index 94c8eb47794..cc76b61733a 100644
--- a/src/metabase/slackbot/uploads.clj
+++ b/src/metabase/slackbot/uploads.clj
@@ -1,179 +1,219 @@
(ns metabase.slackbot.uploads
- "CSV upload handling for slackbot."
+ "CSV and TSV upload handling for Slack."
(:require
[clojure.java.io :as io]
[clojure.string :as str]
[metabase.analytics-interface.core :as analytics]
- [metabase.channel.settings :as channel.settings]
[metabase.slackbot.client :as slackbot.client]
- [metabase.slackbot.db :as slackbot.db]
[metabase.upload.core :as upload]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log])
+ (:import
+ (java.io File InputStream OutputStream)))
(set! *warn-on-reflection* true)
-(def ^:private max-file-size-bytes
- "Maximum file size for CSV uploads (200MB)"
+(def ^:private max-slack-upload-size-bytes
+ "Maximum file size that the bot will download from Slack."
(* 200 1024 1024))
-(def ^:private allowed-csv-filetypes
- "File types that are allowed for CSV uploads"
+(def ^:private supported-filetypes
+ "Slack file types that can be uploaded."
#{"csv" "tsv"})
-(defn- csv-file?
- "Check if a Slack file is a CSV/TSV based on filetype."
+(defn- supported-file?
+ "Whether Slack reports `file` as a CSV or TSV."
[{:keys [filetype]}]
- (contains? allowed-csv-filetypes filetype))
+ (contains? supported-filetypes filetype))
(defn- remote-file?
- "Whether a Slack file is stored outside Slack, so its URL and size come from the app that registered it rather
- than from Slack. Those are never downloaded, whatever filetype they claim."
+ "Whether Slack reports `file` as stored outside Slack.
+ A remote file's URL and size come from the app that registered it, so it is never downloaded."
[{:keys [mode]}]
(= "external" mode))
(defn- size-limit-message
- "The user-facing message for a file over [[max-file-size-bytes]]."
+ "The message for a file over [[max-slack-upload-size-bytes]]."
[filename]
- (format "File '%s' exceeds %dMB size limit" filename (quot max-file-size-bytes (* 1024 1024))))
+ (format "I couldn't upload %s because it is larger than the %d MB limit."
+ filename
+ (quot max-slack-upload-size-bytes (* 1024 1024))))
-(defn- validate-file-size
- "Returns nil if valid, error string if too large."
+(defn- file-size-error
+ "Why `file` is too large to upload, or nil if it is not."
[{:keys [name size]}]
- (when (> size max-file-size-bytes)
+ (when (> size max-slack-upload-size-bytes)
(size-limit-message name)))
(defn- copy-to-file!
- "Copy `in` into `file`, refusing more than [[max-file-size-bytes]]. The size on the event is only what the sender
- declared, so the limit has to hold as the bytes arrive."
- [^java.io.InputStream in ^java.io.File file filename]
+ "Copy `in` into `file`, refusing more than [[max-slack-upload-size-bytes]]."
+ [^InputStream in ^File file filename]
+ ;; The size on the event is only what the sender declared, so the limit has to hold as the bytes arrive.
(let [buf (byte-array 8192)]
- (with-open [^java.io.OutputStream out (io/output-stream file)]
+ (with-open [^OutputStream out (io/output-stream file)]
(loop [written 0]
(let [n (.read in buf)]
(when-not (neg? n)
(let [total (+ written n)]
- (when (> total max-file-size-bytes)
- (throw (ex-info (size-limit-message filename) {:status-code 400})))
+ (when (> total max-slack-upload-size-bytes)
+ (throw (ex-info (size-limit-message filename) {:status-code 400, ::over-size-limit true})))
(.write out buf 0 n)
(recur total))))))))
-(def ^:private generic-upload-error
- ;; Handed a bare internal-error note, the model told the user no file had been attached.
- "Metabase hit an internal error while saving the file, so the upload didn't finish. Ask a Metabase admin to check the server logs.")
-
-(defn- upload-error-message
- "Returns failure text that is safe to hand to the model.
- The upload layer's own 4xx errors carry a message written for the user and have no cause. Anything else,
- including the 4xx that relays a raw driver error along with its cause, gets [[generic-upload-error]], since
- driver and JDBC messages can name hosts or accounts."
- [e]
- (let [{:keys [status-code]} (ex-data e)]
- (if (and (integer? status-code)
- (<= 400 status-code 499)
- (nil? (ex-cause e))
- (not (str/blank? (ex-message e))))
- (ex-message e)
- generic-upload-error)))
-
-(defn- upload-settings
- "Get upload settings map. Returns nil if uploads are not enabled."
+(defn- count-files!
+ "Count `n` attached files under `result` in the Slack upload counter."
+ [result n]
+ (analytics/inc! :metabase-slackbot/file-uploads {:result result} n))
+
+(defn- upload-target
+ "The upload database, schema, and table prefix, or nil when uploads are not configured."
[]
(when-let [db (upload/current-database)]
- {:db_id (:id db)
- :schema_name (:uploads_schema_name db)
- :table_prefix (:uploads_table_prefix db)}))
-
-(defn- process-csv-file
- "Process a single CSV file upload. Returns a result map with either
- :model-id/:model-name (success) or :error (failure)."
- [{:keys [db_id schema_name table_prefix]} {:keys [name url_private] :as file}]
- (if-let [size-error (validate-file-size file)]
+ {:db db
+ :schema-name (:uploads_schema_name db)
+ :table-prefix (:uploads_table_prefix db)}))
+
+(defn- create-temp-file!
+ "Create a temporary file using the validated Slack file type as its suffix."
+ [filetype]
+ (File/createTempFile "slack-upload-" (str "." filetype)))
+
+(defn- upload-file!
+ "Upload one CSV or TSV file.
+ Returns `:filename`, `:model-id`, and `:model-name` on success, or `:filename` and a safe `:error` on failure."
+ [client {:keys [db schema-name table-prefix]} {:keys [name filetype url_private] :as file}]
+ (if-let [size-error (file-size-error file)]
(do
(log/warnf "[slackbot] File exceeds size limit: error=%s" size-error)
- {:error size-error :filename name})
- (let [temp-file (java.io.File/createTempFile "slack-upload-" (str "-" name))]
- (try
- (with-open [^java.io.InputStream stream (slackbot.client/download-file-stream {:token (channel.settings/unobfuscated-slack-app-token)} url_private)]
- (copy-to-file! stream temp-file name)
+ (count-files! "too-large" 1)
+ {:error size-error, :filename name})
+ (try
+ ;; [[upload/create-csv-upload!]] reads the file more than once, so download it to disk first.
+ (let [temp-file (create-temp-file! filetype)]
+ (try
+ (with-open [^InputStream stream (slackbot.client/download-file-stream client url_private)]
+ (copy-to-file! stream temp-file name))
(let [result (upload/create-csv-upload!
{:filename name
:file temp-file
- :db-id db_id
- :schema-name schema_name
- :table-prefix table_prefix
+ :db-id (:id db)
+ :schema-name schema-name
+ :table-prefix table-prefix
:collection-id nil})]
- (log/infof "[slackbot] File uploaded: model_id=%d" (:id result))
- (analytics/inc! :metabase-slackbot/file-uploads {:result "success"})
- {:filename name
- :model-id (:id result)
- :model-name (:name result)}))
- (catch Exception e
- (log/warnf e "[slackbot] File upload failed: error=%s" (ex-message e))
- (analytics/inc! :metabase-slackbot/file-uploads {:result "error"})
- {:error (upload-error-message e) :filename name})
- (finally
- (io/delete-file temp-file true))))))
-
-(defn- process-file-uploads
- "Process all files from a Slack event. Returns a map with:
- :results - seq of individual file results
- :skipped - seq of non-CSV filenames that were skipped
- :remote - seq of filenames refused for being stored outside Slack"
- [settings files]
- (let [{remote-files true local-files false} (group-by remote-file? files)
- {csv-files true other-files false} (group-by csv-file? local-files)
- skipped (mapv :name other-files)
- remote (mapv :name remote-files)]
+ (log/infof "[slackbot] File uploaded: model_id=%s" (:id result))
+ (count-files! "success" 1)
+ {:filename name
+ :model-id (:id result)
+ :model-name (:name result)})
+ (finally
+ (io/delete-file temp-file true))))
+ (catch Exception e
+ (log/warn e "[slackbot] File upload failed" {:filename name})
+ (count-files! (if (::over-size-limit (ex-data e)) "too-large" "error") 1)
+ ;; Repeat back only what was written for the user: the size limit, the file type, and the upload module's
+ ;; own 4xx errors, which describe a problem in the file (a row with too many columns, say).
+ ;; The upload module rethrows driver and JDBC errors as a 4xx carrying a cause, and those can name hosts.
+ {:error (cond
+ (::over-size-limit (ex-data e))
+ (ex-message e)
+
+ (= 415 (:status-code (ex-data e)))
+ (format "I couldn't upload %s because it doesn't look like a CSV or TSV file." name)
+
+ (and (let [status (:status-code (ex-data e))]
+ (and (integer? status) (<= 400 status 499)))
+ (nil? (ex-cause e))
+ (not (str/blank? (ex-message e))))
+ (format "I couldn't upload %s: %s" name (ex-message e))
+
+ :else
+ (format "I couldn't upload %s because something went wrong. Ask your Metabase admin to check the server logs." name))
+ :filename name}))))
+
+(defn- upload-files!
+ "Upload supported files and return their results with the names of unsupported and remote files."
+ [client target files]
+ (let [{remote-files true local-files false} (group-by remote-file? files)
+ {supported-files true unsupported-files false} (group-by supported-file? local-files)
+ skipped (mapv :name unsupported-files)
+ remote (mapv :name remote-files)]
(when (seq skipped)
- (log/debugf "[slackbot] Skipping %d non-CSV files" (count skipped)))
+ (log/debugf "[slackbot] Skipping %d unsupported files" (count skipped))
+ (count-files! "unsupported" (count skipped)))
(when (seq remote)
- (log/debugf "[slackbot] Refusing %d files stored outside Slack" (count remote)))
- {:results (mapv (partial process-csv-file settings) csv-files)
+ (log/debugf "[slackbot] Refusing %d files stored outside Slack" (count remote))
+ (count-files! "remote" (count remote)))
+ {:results (mapv #(upload-file! client target %) supported-files)
:skipped skipped
:remote remote}))
-(defn- build-upload-system-messages
- "Build system messages to inject into AI request about uploads."
+(defn- assistant-history-message
+ "A Metabot message that can be sent directly or added to the AI history."
+ [content]
+ {:role :assistant
+ :content content})
+
+(def ^:private uploads-not-configured-message
+ "Uploads aren't configured yet. Ask your Metabase admin to choose an upload database in Admin > Settings > Uploads.")
+
+(def ^:private upload-unavailable-message
+ "I can't upload files to the configured database. Ask your Metabase admin to check the upload settings and your permissions.")
+
+(defn- uploaded-files-message
+ "A Metabot message describing successful uploads."
+ [successes]
+ (if (= 1 (count successes))
+ (let [{:keys [filename model-id model-name]} (first successes)]
+ (format "I uploaded %s as the Metabase model %s (ID %d). I can help you query it." filename model-name model-id))
+ (format "I uploaded these files as Metabase models: %s. I can help you query them."
+ (str/join ", " (map #(format "%s as %s (ID %d)"
+ (:filename %)
+ (:model-name %)
+ (:model-id %))
+ successes)))))
+
+(defn- failed-files-message
+ "A Metabot message describing failed uploads."
+ [failures]
+ (str/join " " (map :error failures)))
+
+(defn- skipped-files-message
+ "A Metabot message describing unsupported attachments."
+ [skipped]
+ (format "I can only upload CSV and TSV files, so I skipped the following: %s." (str/join ", " skipped)))
+
+(defn- remote-files-message
+ "A Metabot message describing attachments stored outside Slack."
+ [remote]
+ (format "I can only upload files stored in Slack itself, so I skipped the following: %s."
+ (str/join ", " remote)))
+
+(defn- build-upload-history
+ "Metabot messages describing what became of each attached file."
[{:keys [results skipped remote]}]
(let [successes (filter :model-id results)
- failures (filter :error results)]
+ failures (filter :error results)]
(cond-> []
(seq successes)
- (conj {:role :assistant
- :content (format "The following message included 1 or more attached CSV files which are now available as models in Metabase: %s. You can help them query this data."
- (str/join ", " (map #(format "%s (model ID: %d)"
- (:filename %)
- (:model-id %))
- successes)))})
+ (conj (assistant-history-message (uploaded-files-message successes)))
(seq failures)
- (conj {:role :assistant
- :content (format "The following message included 1 or more attached CSV files. Some file uploads failed: %s. Explain these errors to the user."
- (str/join ", " (map #(format "%s: %s"
- (:filename %)
- (:error %))
- failures)))})
+ (conj (assistant-history-message (failed-files-message failures)))
(seq skipped)
- (conj {:role :assistant
- :content (format "The following message included 1 or more non-CSV files which are not supported: %s. Let them know only CSV files can be uploaded."
- (str/join ", " skipped))})
+ (conj (assistant-history-message (skipped-files-message skipped)))
(seq remote)
- (conj {:role :assistant
- :content (format "The following message included 1 or more files stored outside Slack, which Metabase cannot upload: %s. Let them know the file has to be uploaded to Slack itself."
- (str/join ", " remote))}))))
-
-(defn handle-file-uploads
- "Handle file uploads if present. Returns nil if no files, otherwise
- returns upload result map and messages to inject into AI request."
- [files]
+ (conj (assistant-history-message (remote-files-message remote))))))
+
+(defn handle-file-uploads!
+ "Handle attached CSV and TSV files with `client`.
+ Returns nil when no files are attached, otherwise Metabot messages describing what became of them."
+ [client files]
(when (seq files)
- (if-let [{:keys [db_id schema_name] :as settings} (upload-settings)]
- (let [db (slackbot.db/database db_id)]
- (if-not (upload/can-create-upload? db schema_name)
- {:error "You don't have permission to upload files. Contact your Metabase administrator."}
- (let [result (process-file-uploads settings files)]
- {:upload-result result
- :system-messages (build-upload-system-messages result)})))
- {:error "CSV uploads are not enabled. An administrator needs to configure a database for uploads in Admin > Settings > Uploads."})))
+ (if-let [{:keys [db schema-name] :as target} (upload-target)]
+ (if-not (upload/can-create-upload? db schema-name)
+ (do (count-files! "unavailable" (count files))
+ [(assistant-history-message upload-unavailable-message)])
+ (build-upload-history (upload-files! client target files)))
+ (do (count-files! "unavailable" (count files))
+ [(assistant-history-message uploads-not-configured-message)]))))
diff --git a/test/metabase/slackbot/client_test.clj b/test/metabase/slackbot/client_test.clj
index 5dc1694ec69..48b244d73f7 100644
--- a/test/metabase/slackbot/client_test.clj
+++ b/test/metabase/slackbot/client_test.clj
@@ -80,6 +80,20 @@
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"(?i)unexpected response"
(slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))
(is (= 2 @calls) "one hop only"))))
+ (testing "a Location that will not resolve is no redirect, and the response body is still closed"
+ (let [calls (atom 0)
+ closed (atom false)]
+ (with-redefs [http/get (fn [_url _options]
+ (swap! calls inc)
+ {:status 302
+ :headers {"location" "http://[not a uri"}
+ :body (proxy [java.io.ByteArrayInputStream] [(.getBytes "")]
+ (close []
+ (reset! closed true)))})]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"(?i)unexpected response"
+ (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))
+ (is (= 1 @calls) "the malformed target is never requested")
+ (is (true? @closed) "the response stream is closed"))))
(testing "behind a JVM proxy the dns-resolver is omitted, leaving the target to the proxy"
(let [opts (atom nil)]
(with-redefs [u.http/jvm-proxied-url? (constantly true)
diff --git a/test/metabase/slackbot/uploads_test.clj b/test/metabase/slackbot/uploads_test.clj
index eaf3db9e3ca..79c808a5cb6 100644
--- a/test/metabase/slackbot/uploads_test.clj
+++ b/test/metabase/slackbot/uploads_test.clj
@@ -4,15 +4,19 @@
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.app-db.encryption-test-util :as encryption-tu]
- [metabase.channel.settings :as channel.settings]
+ [metabase.slackbot.api :as slackbot.api]
[metabase.slackbot.client :as slackbot.client]
[metabase.slackbot.test-util :as tu]
[metabase.slackbot.uploads :as slackbot.uploads]
[metabase.test :as mt]
+ [metabase.test.data.sql :as sql.tx]
[metabase.test.fixtures :as fixtures]
[metabase.upload.db :as upload.db]
[metabase.upload.impl :as upload.impl]
- [metabase.util :as u]))
+ [metabase.util :as u]
+ [toucan2.core :as t2])
+ (:import
+ (java.io ByteArrayInputStream)))
(set! *warn-on-reflection* true)
@@ -21,19 +25,11 @@
(encryption-tu/with-encrypted-app-db-fixture tu/test-encryption-key))
(defn- with-upload-mocks!
- "Helper to set up common mocks for CSV upload tests.
- Options:
- - :uploads-enabled? - Whether uploads are enabled (default false)
- - :can-create-upload? - Whether user can create uploads (default true)
- - :upload-result - Result from create-csv-upload! (default success with id 123)
- - :download-content - Content returned by download-slack-file (default valid CSV)
-
- Calls body-fn with a map containing tracking atoms:
- {:upload-calls, :download-calls}"
- [{:keys [uploads-enabled? can-create-upload? upload-result download-content db-id]
+ "Run `body-fn` with configurable upload mocks and atoms that track upload and download calls."
+ [{:keys [uploads-enabled? can-create-upload? upload-result upload-error download-content db-id]
:or {uploads-enabled? false
can-create-upload? true
- upload-result {:id 123 :name "uploaded_data"}
+ upload-result {:id 123, :name "uploaded_data"}
download-content (.getBytes "col1,col2\nval1,val2")
db-id 1}}
body-fn]
@@ -47,40 +43,15 @@
upload.impl/can-create-upload? (constantly can-create-upload?)
upload.impl/create-csv-upload! (fn [params]
(swap! upload-calls conj params)
- upload-result)
- slackbot.client/download-file-stream (fn [_client url]
- (swap! download-calls conj url)
- (io/input-stream download-content))]
+ (if upload-error
+ (throw upload-error)
+ upload-result))
+ slackbot.client/download-file-stream (fn [_client url]
+ (swap! download-calls conj url)
+ (io/input-stream download-content))]
(body-fn {:upload-calls upload-calls
:download-calls download-calls}))))
-(defn- failed-upload-result!
- "Run one CSV file through [[slackbot.uploads/handle-file-uploads]] with
- `create-csv-upload!` throwing `thrown`."
- [thrown]
- (with-upload-mocks!
- {:uploads-enabled? true}
- (fn [_]
- (mt/with-dynamic-fn-redefs [upload.impl/create-csv-upload! (fn [_] (throw thrown))]
- (slackbot.uploads/handle-file-uploads [tu/slack-csv-file])))))
-
-(deftest csv-upload-failure-message-test
- (testing "a deliberate upload error keeps its message, filename attached"
- (let [result (failed-upload-result! (ex-info "Uploads are not enabled." {:status-code 422}))]
- (is (= "Uploads are not enabled."
- (get-in result [:upload-result :results 0 :error])))
- (is (str/includes? (-> result :system-messages first :content)
- "data.csv: Uploads are not enabled."))))
- (testing "a driver error rewrapped with its raw message is replaced with the generic line"
- (let [raw "Connection to db.internal.example.com:5432 refused. Check that the hostname and port are correct."
- result (failed-upload-result! (ex-info raw {:status-code 400} (java.sql.SQLException. raw)))]
- (is (= @#'slackbot.uploads/generic-upload-error
- (get-in result [:upload-result :results 0 :error])))))
- (testing "a raw driver exception is replaced with the generic line"
- (let [result (failed-upload-result! (java.sql.SQLException. "FATAL: password authentication failed for user \"metabase\""))]
- (is (= @#'slackbot.uploads/generic-upload-error
- (get-in result [:upload-result :results 0 :error]))))))
-
(deftest ^:synchronized csv-upload-disabled-test
(testing "POST /events with file upload when uploads are disabled"
(tu/with-slackbot-setup
@@ -92,7 +63,7 @@
{:uploads-enabled? false}
(fn [{:keys [upload-calls]}]
(tu/with-slackbot-mocks
- {:ai-text "CSV uploads are not enabled on this Metabase instance."}
+ {:ai-text "Uploads aren't configured yet. Ask your Metabase admin to choose an upload database in Admin > Settings > Uploads."}
(fn [{:keys [stop-stream-calls append-text-calls]}]
(let [response (mt/client :post 200 "metabot/slack/events"
(tu/slack-request-options event-body)
@@ -104,7 +75,7 @@
(testing "no upload was attempted"
(is (= 0 (count @upload-calls))))
(testing "AI responds with error message"
- (is (some #(= "CSV uploads are not enabled on this Metabase instance." %)
+ (is (some #(= "Uploads aren't configured yet. Ask your Metabase admin to choose an upload database in Admin > Settings > Uploads." %)
@append-text-calls))))))))))))
(deftest ^:synchronized csv-upload-success-test
@@ -117,11 +88,11 @@
(with-upload-mocks!
{:uploads-enabled? true
:can-create-upload? true
- :upload-result {:id 456 :name "Data"}}
+ :upload-result {:id 456, :name "Data"}}
(fn [{:keys [upload-calls download-calls]}]
(tu/with-slackbot-mocks
{:ai-text "Your CSV has been uploaded successfully as a model."}
- (fn [{:keys [stop-stream-calls append-text-calls]}]
+ (fn [{:keys [stop-stream-calls append-text-calls ai-request-calls]}]
(let [response (mt/client :post 200 "metabot/slack/events"
(tu/slack-request-options event-body)
event-body)]
@@ -138,10 +109,63 @@
(is (= (:name tu/slack-csv-file) (:filename call)))))
(testing "AI responds with success message"
(is (some #(= "Your CSV has been uploaded successfully as a model." %)
- @append-text-calls))))))))))))
+ @append-text-calls)))
+ (testing "AI receives the upload details"
+ (is (some #(= {:role :assistant
+ :content "I uploaded data.csv as the Metabase model Data (ID 456). I can help you query it."}
+ %)
+ (:messages (first @ai-request-calls))))))))))))))
-(deftest ^:synchronized csv-upload-non-csv-skipped-test
- (testing "POST /events with non-CSV file is skipped"
+(deftest ^:synchronized csv-upload-no-text-test
+ (testing "POST /events with a CSV file and no text responds directly without AI"
+ (tu/with-slackbot-setup
+ (let [event-body (-> tu/base-dm-event
+ (update :event merge {:subtype "file_share", :files [tu/slack-csv-file]})
+ (update :event dissoc :text))]
+ (with-upload-mocks!
+ {:uploads-enabled? true
+ :upload-result {:id 456, :name "Data"}}
+ (fn [_]
+ (tu/with-slackbot-mocks
+ {}
+ (fn [{:keys [post-calls ai-request-calls]}]
+ (let [response (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)]
+ (is (= "ok" response))
+ (u/poll {:thunk #(= 1 (count @post-calls))
+ :done? true?
+ :timeout-ms 5000})
+ (is (empty? @ai-request-calls))
+ (is (= "I uploaded data.csv as the Metabase model Data (ID 456). I can help you query it."
+ (:text (first @post-calls)))))))))))))
+
+(deftest ^:synchronized csv-upload-no-text-failure-test
+ (testing "POST /events with a failed CSV upload and no text responds directly without exposing the backend error"
+ (tu/with-slackbot-setup
+ (let [event-body (-> tu/base-dm-event
+ (update :event merge {:subtype "file_share", :files [tu/slack-csv-file]})
+ (update :event dissoc :text))]
+ (with-upload-mocks!
+ {:uploads-enabled? true
+ :upload-error (ex-info "sensitive database details" {})}
+ (fn [_]
+ (tu/with-slackbot-mocks
+ {}
+ (fn [{:keys [post-calls ai-request-calls]}]
+ (let [response (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)]
+ (is (= "ok" response))
+ (u/poll {:thunk #(= 1 (count @post-calls))
+ :done? true?
+ :timeout-ms 5000})
+ (is (empty? @ai-request-calls))
+ (is (= "I couldn't upload data.csv because something went wrong. Ask your Metabase admin to check the server logs."
+ (:text (first @post-calls)))))))))))))
+
+(deftest ^:synchronized unsupported-file-skipped-test
+ (testing "POST /events skips an unsupported file"
(tu/with-slackbot-setup
(let [event-body (update tu/base-dm-event :event merge
{:subtype "file_share"
@@ -155,7 +179,7 @@
{:uploads-enabled? true}
(fn [{:keys [upload-calls download-calls]}]
(tu/with-slackbot-mocks
- {:ai-text "Only CSV files can be uploaded."}
+ {:ai-text "I can only upload CSV and TSV files, so I skipped document.pdf."}
(fn [{:keys [stop-stream-calls append-text-calls]}]
(let [response (mt/client :post 200 "metabot/slack/events"
(tu/slack-request-options event-body)
@@ -169,13 +193,13 @@
(testing "no upload was attempted"
(is (= 0 (count @upload-calls))))
(testing "AI responds explaining PDF not supported"
- (is (some #(= "Only CSV files can be uploaded." %)
+ (is (some #(= "I can only upload CSV and TSV files, so I skipped document.pdf." %)
@append-text-calls))))))))))))
-(deftest ^:synchronized csv-upload-non-csv-no-text-test
- (testing "POST /events with non-CSV file and no text responds directly without AI"
+(deftest ^:synchronized unsupported-file-no-text-test
+ (testing "POST /events with an unsupported file and no text responds directly without AI"
(tu/with-slackbot-setup
- ;; No :text field - user uploaded file without typing anything
+ ;; No `:text` field because the user uploaded a file without typing anything.
(let [event-body (-> tu/base-dm-event
(update :event merge
{:subtype "file_share"
@@ -190,7 +214,7 @@
(fn [{:keys [upload-calls download-calls]}]
(tu/with-slackbot-mocks
{:ai-text "This should not be called"}
- (fn [{:keys [post-calls delete-calls]}]
+ (fn [{:keys [post-calls ai-request-calls]}]
(let [response (mt/client :post 200 "metabot/slack/events"
(tu/slack-request-options event-body)
event-body)]
@@ -198,22 +222,71 @@
(u/poll {:thunk #(>= (count @post-calls) 1)
:done? true?
:timeout-ms 5000})
- ;; When AI is called, there are 2 posts (Thinking... then response) and 1 delete.
- ;; When AI is skipped, there is just 1 post and no deletes.
- (testing "AI was not called (only 1 post, no deletes)"
- (is (= 1 (count @post-calls)))
- (is (= 0 (count @delete-calls))))
+ (testing "AI was not called"
+ (is (empty? @ai-request-calls)))
(testing "no download was attempted"
(is (= 0 (count @download-calls))))
(testing "no upload was attempted"
(is (= 0 (count @upload-calls))))
(testing "responds directly with skip message"
- (let [message-text (:text (first @post-calls))]
- (is (str/includes? message-text "can only process CSV and TSV"))
- (is (str/includes? message-text "query_result.xlsx")))))))))))))
+ (is (= "I can only upload CSV and TSV files, so I skipped the following: query_result.xlsx."
+ (:text (first @post-calls))))))))))))))
+
+(deftest ^:synchronized remote-file-no-text-test
+ (testing "POST /events with a file stored outside Slack and no text responds directly without fetching it"
+ (tu/with-slackbot-setup
+ ;; No `:text` field because the user shared a file without typing anything.
+ (let [event-body (-> tu/base-dm-event
+ (update :event merge
+ {:subtype "file_share"
+ :files [{:id "F123"
+ :name "remote.csv"
+ :filetype "csv"
+ :mode "external"
+ :url_private "https://evil.test/remote.csv"
+ :size 100}]})
+ (update :event dissoc :text))]
+ (with-upload-mocks!
+ {:uploads-enabled? true}
+ (fn [{:keys [upload-calls download-calls]}]
+ (tu/with-slackbot-mocks
+ {:ai-text "This should not be called"}
+ (fn [{:keys [post-calls ai-request-calls]}]
+ (let [response (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)]
+ (is (= "ok" response))
+ (u/poll {:thunk #(>= (count @post-calls) 1)
+ :done? true?
+ :timeout-ms 5000})
+ (testing "AI was not called"
+ (is (empty? @ai-request-calls)))
+ (testing "the URL the registering app chose is never fetched"
+ (is (= 0 (count @download-calls))))
+ (testing "no upload was attempted"
+ (is (= 0 (count @upload-calls))))
+ (testing "responds directly explaining the file has to live in Slack"
+ (is (= "I can only upload files stored in Slack itself, so I skipped the following: remote.csv."
+ (:text (first @post-calls))))))))))))))
+
+;; Driving the handler directly rather than the route: this asserts that nothing is posted, which the route's
+;; async dispatch cannot settle without racing it.
+(deftest ^:synchronized no-text-and-no-files-test
+ (testing "a file_share message carrying neither text nor files says nothing rather than posting an empty message"
+ (tu/with-slackbot-setup
+ (tu/with-slackbot-mocks
+ {:ai-text "This should not be called"}
+ (fn [{:keys [post-calls ai-request-calls]}]
+ (#'slackbot.api/handle-message-file-share
+ {:token "xoxb-test-token"}
+ (-> (:event tu/base-dm-event)
+ (merge {:subtype "file_share", :files []})
+ (dissoc :text)))
+ (is (empty? @post-calls))
+ (is (empty? @ai-request-calls)))))))
-(deftest ^:synchronized csv-upload-mixed-files-test
- (testing "POST /events with mix of CSV and non-CSV files"
+(deftest ^:synchronized mixed-file-upload-test
+ (testing "POST /events with supported and unsupported files"
(tu/with-slackbot-setup
(let [event-body (update tu/base-dm-event :event merge
{:subtype "file_share"
@@ -231,10 +304,10 @@
:size 150}]})]
(with-upload-mocks!
{:uploads-enabled? true
- :upload-result {:id 789 :name "Uploaded Data"}}
+ :upload-result {:id 789, :name "Uploaded Data"}}
(fn [{:keys [upload-calls download-calls]}]
(tu/with-slackbot-mocks
- {:ai-text "Uploaded 2 files, skipped 1 non-CSV file."}
+ {:ai-text "I uploaded data.csv and more_data.tsv, and skipped report.pdf because I can only upload CSV and TSV files."}
(fn [{:keys [stop-stream-calls]}]
(let [response (mt/client :post 200 "metabot/slack/events"
(tu/slack-request-options event-body)
@@ -254,7 +327,7 @@
(deftest ^:synchronized csv-upload-file-too-large-test
(testing "POST /events with file exceeding size limit"
(tu/with-slackbot-setup
- (let [too-large-size (inc (* 200 1024 1024)) ;; Just over 200MB
+ (let [too-large-size (inc (* 200 1024 1024)) ;; Just over 200 MB.
event-body (update tu/base-dm-event :event merge
{:subtype "file_share"
:text "Here's my huge file"
@@ -266,7 +339,7 @@
{:uploads-enabled? true}
(fn [{:keys [upload-calls download-calls]}]
(tu/with-slackbot-mocks
- {:ai-text "The file exceeds the 200MB size limit."}
+ {:ai-text "I couldn't upload huge_data.csv because it is larger than the 200 MB limit."}
(fn [{:keys [stop-stream-calls]}]
(let [response (mt/client :post 200 "metabot/slack/events"
(tu/slack-request-options event-body)
@@ -292,7 +365,7 @@
:can-create-upload? false}
(fn [{:keys [upload-calls]}]
(tu/with-slackbot-mocks
- {:ai-text "You don't have permission to upload files."}
+ {:ai-text "I can't upload files to the configured database. Ask your Metabase admin to check the upload settings and your permissions."}
(fn [{:keys [stop-stream-calls append-text-calls]}]
(let [response (mt/client :post 200 "metabot/slack/events"
(tu/slack-request-options event-body)
@@ -304,62 +377,222 @@
(testing "no upload was attempted"
(is (= 0 (count @upload-calls))))
(testing "AI responds with permission error"
- (is (some #(= "You don't have permission to upload files." %)
+ (is (some #(= "I can't upload files to the configured database. Ask your Metabase admin to check the upload settings and your permissions." %)
@append-text-calls))))))))))))
-(deftest process-csv-file-streams-to-temp-file-test
- (testing "process-csv-file streams download content through a temp file to the upload fn"
+(def ^:private test-client {:token "xoxb-fake"})
+
+(def ^:private test-target
+ {:db {:id 1}, :schema-name nil, :table-prefix nil})
+
+(deftest upload-file-streams-to-temp-file-test
+ (testing "upload-file! closes the download before uploading and removes the temporary file"
(let [csv-content "col1,col2\nfoo,bar\nbaz,qux"
- uploaded-file (atom nil)]
+ uploaded-file (atom nil)
+ temp-file (atom nil)
+ stream-closed (atom false)]
(mt/with-dynamic-fn-redefs
- [slackbot.client/download-file-stream (fn [_client _url]
- (io/input-stream (.getBytes csv-content)))
+ [slackbot.client/download-file-stream (fn [client _url]
+ (is (= test-client client))
+ (proxy [ByteArrayInputStream] [(.getBytes csv-content)]
+ (close []
+ (reset! stream-closed true))))
upload.impl/create-csv-upload! (fn [{:keys [file] :as _params}]
+ (is @stream-closed)
+ (reset! temp-file file)
(reset! uploaded-file (slurp file))
- {:id 1 :name "test"})
- ;; stub out the token lookup
- channel.settings/unobfuscated-slack-app-token (constantly "xoxb-fake")]
- (let [result (#'slackbot.uploads/process-csv-file
- {:db_id 1 :schema_name nil :table_prefix nil}
- {:name "test.csv" :url_private "https://example.com/test.csv" :size 100})]
- (is (= 1 (:model-id result)))
- (testing "file content was streamed correctly through temp file"
- (is (= csv-content @uploaded-file))))))))
+ {:id 1, :name "test"})]
+ (is (=? {:filename "test.csv"
+ :model-id 1
+ :model-name "test"}
+ (#'slackbot.uploads/upload-file!
+ test-client
+ test-target
+ {:name "test.csv", :filetype "csv", :url_private "https://example.com/test.csv", :size 100})))
+ (testing "file content was streamed correctly through temp file"
+ (is (= csv-content @uploaded-file)))
+ (testing "the validated file type is used as the suffix"
+ (is (str/ends-with? (.getName ^java.io.File @temp-file) ".csv")))
+ (testing "the temp file is cleaned up"
+ (is (false? (.exists ^java.io.File @temp-file))))))))
+
+(deftest upload-file-temp-file-failure-test
+ (testing "a file that cannot be staged on disk is reported as a failed file, not thrown"
+ (mt/with-dynamic-fn-redefs
+ [slackbot.uploads/create-temp-file! (fn [_filetype]
+ (throw (ex-info "sensitive filesystem details" {})))
+ slackbot.client/download-file-stream (fn [_client _url]
+ (throw (ex-info "download should not start" {})))]
+ (is (= {:filename "data.csv"
+ :error "I couldn't upload data.csv because something went wrong. Ask your Metabase admin to check the server logs."}
+ (#'slackbot.uploads/upload-file!
+ test-client
+ test-target
+ {:name "data.csv", :filetype "csv", :url_private "https://example.com/x.csv", :size 100}))))))
+
+(deftest upload-file-cleans-up-after-upload-failure-test
+ (testing "the temporary file is removed and the backend error is not returned"
+ (let [temp-file (atom nil)]
+ (mt/with-dynamic-fn-redefs
+ [slackbot.client/download-file-stream (fn [_client _url]
+ (io/input-stream (.getBytes "col1,col2\nval1,val2")))
+ upload.impl/create-csv-upload! (fn [{:keys [file]}]
+ (reset! temp-file file)
+ (throw (ex-info "sensitive database details" {})))]
+ (is (= {:filename "data.csv"
+ :error "I couldn't upload data.csv because something went wrong. Ask your Metabase admin to check the server logs."}
+ (#'slackbot.uploads/upload-file!
+ test-client
+ test-target
+ {:name "data.csv", :filetype "csv", :url_private "https://example.com/x.csv", :size 100})))
+ (is (false? (.exists ^java.io.File @temp-file)))))))
+
+(deftest upload-file-unsupported-file-type-test
+ (testing "a file that does not look like a CSV or TSV says so"
+ (mt/with-dynamic-fn-redefs
+ [slackbot.client/download-file-stream (fn [_client _url]
+ (io/input-stream (.getBytes "col1,col2\nval1,val2")))
+ upload.impl/create-csv-upload! (fn [_params]
+ (throw (ex-info "Unsupported File Type" {:status-code 415})))]
+ (is (= {:filename "data.csv"
+ :error "I couldn't upload data.csv because it doesn't look like a CSV or TSV file."}
+ (#'slackbot.uploads/upload-file!
+ test-client
+ test-target
+ {:name "data.csv", :filetype "csv", :url_private "https://example.com/x.csv", :size 100}))))))
+
+;; Runs the real upload module, since the regression was in what it throws, not in how the bot words it.
+(deftest upload-file-malformed-csv-test
+ (testing "a row with more columns than the header tells the user what is wrong with their file"
+ (mt/test-driver :h2
+ (mt/with-empty-db
+ (mt/with-discard-model-updates! [:model/Database]
+ (t2/update! :model/Database (mt/id) {:uploads_enabled true})
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-dynamic-fn-redefs
+ [slackbot.client/download-file-stream (fn [_client _url]
+ (io/input-stream (.getBytes "a,b\n1,2\n3,4,5\n")))]
+ (is (=? {:filename "malformed.csv"
+ :error #"I couldn't upload malformed\.csv: Column count in data .*"}
+ (#'slackbot.uploads/upload-file!
+ test-client
+ {:db (t2/select-one :model/Database (mt/id))
+ :schema-name (sql.tx/session-schema :h2)
+ :table-prefix "slackbot_"}
+ {:name "malformed.csv", :filetype "csv", :url_private "https://example.com/x.csv", :size 100}))))))))))
+
+(deftest upload-file-failure-message-test
+ (doseq [[description thrown expected]
+ [["a deliberate upload error keeps its message, filename attached"
+ (ex-info "Uploads are not enabled." {:status-code 422})
+ "I couldn't upload data.csv: Uploads are not enabled."]
+ ["a driver error rewrapped as a 4xx with its raw message is replaced with the generic line"
+ (let [raw "Connection to db.internal.example.com:5432 refused. Check that the hostname and port are correct."]
+ (ex-info raw {:status-code 400} (java.sql.SQLException. raw)))
+ "I couldn't upload data.csv because something went wrong. Ask your Metabase admin to check the server logs."]
+ ["a raw driver exception is replaced with the generic line"
+ (java.sql.SQLException. "FATAL: password authentication failed for user \"metabase\"")
+ "I couldn't upload data.csv because something went wrong. Ask your Metabase admin to check the server logs."]]]
+ (testing description
+ (mt/with-dynamic-fn-redefs
+ [slackbot.client/download-file-stream (fn [_client _url]
+ (io/input-stream (.getBytes "col1,col2\nval1,val2")))
+ upload.impl/create-csv-upload! (fn [_params] (throw thrown))]
+ (is (= {:filename "data.csv", :error expected}
+ (#'slackbot.uploads/upload-file!
+ test-client
+ test-target
+ {:name "data.csv", :filetype "csv", :url_private "https://example.com/x.csv", :size 100})))))))
-(deftest ^:parallel csv-file-detection-test
- (testing "csv-file? correctly identifies CSV/TSV files"
- (is (true? (#'slackbot.uploads/csv-file? {:filetype "csv"})))
- (is (true? (#'slackbot.uploads/csv-file? {:filetype "tsv"})))
- (is (false? (#'slackbot.uploads/csv-file? {:filetype "pdf"})))
- (is (false? (#'slackbot.uploads/csv-file? {:filetype "xlsx"})))
- (is (false? (#'slackbot.uploads/csv-file? {:filetype nil})))
- (is (false? (#'slackbot.uploads/csv-file? {}))))
+(deftest build-upload-history-test
+ (is (= [{:role :assistant
+ :content "I uploaded these files as Metabase models: data.csv as Data (ID 1), data.tsv as More Data (ID 2). I can help you query them."}
+ {:role :assistant
+ :content "I couldn't upload broken.csv because something went wrong. Ask your Metabase admin to check the server logs."}
+ {:role :assistant
+ :content "I can only upload CSV and TSV files, so I skipped the following: notes.txt."}]
+ (#'slackbot.uploads/build-upload-history
+ {:results [{:filename "data.csv", :model-id 1, :model-name "Data"}
+ {:filename "data.tsv", :model-id 2, :model-name "More Data"}
+ {:filename "broken.csv", :error "I couldn't upload broken.csv because something went wrong. Ask your Metabase admin to check the server logs."}]
+ :skipped ["notes.txt"]}))))
+
+(deftest handle-file-uploads!-nothing-attempted-test
+ (testing "no files at all"
+ (is (nil? (slackbot.uploads/handle-file-uploads! test-client []))))
+ (testing "no database is configured for uploads"
+ (mt/with-dynamic-fn-redefs [upload.db/current-database (constantly nil)]
+ (is (= [{:role :assistant
+ :content "Uploads aren't configured yet. Ask your Metabase admin to choose an upload database in Admin > Settings > Uploads."}]
+ (slackbot.uploads/handle-file-uploads! test-client [tu/slack-csv-file])))))
+ (testing "the configured upload target is unavailable"
+ (mt/with-dynamic-fn-redefs [upload.db/current-database (constantly {:id 1})
+ upload.impl/can-create-upload? (constantly false)]
+ (is (= [{:role :assistant
+ :content "I can't upload files to the configured database. Ask your Metabase admin to check the upload settings and your permissions."}]
+ (slackbot.uploads/handle-file-uploads! test-client [tu/slack-csv-file]))))))
+
+(deftest ^:parallel supported-file?-test
+ (testing "only CSV and TSV files are supported"
+ (is (true? (#'slackbot.uploads/supported-file? {:filetype "csv"})))
+ (is (true? (#'slackbot.uploads/supported-file? {:filetype "tsv"})))
+ (is (false? (#'slackbot.uploads/supported-file? {:filetype "pdf"})))
+ (is (false? (#'slackbot.uploads/supported-file? {:filetype "xlsx"})))
+ (is (false? (#'slackbot.uploads/supported-file? {:filetype "txt"})))
+ (is (false? (#'slackbot.uploads/supported-file? {:filetype nil})))
+ (is (false? (#'slackbot.uploads/supported-file? {})))))
+
+(deftest ^:parallel remote-file?-test
(testing "a file stored outside Slack is refused whatever filetype it claims, ordinary upload modes are kept"
(is (true? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "external"})))
(is (false? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "snippet"})))
(is (false? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "hosted"})))
(is (false? (#'slackbot.uploads/remote-file? {:filetype "csv"})))))
-(deftest ^:parallel remote-files-are-reported-separately-test
+(deftest remote-files-are-reported-separately-test
(testing "a remote CSV is refused for being remote, not reported as an unsupported filetype"
- (let [{:keys [skipped remote results]}
- (#'slackbot.uploads/process-file-uploads
- {:db_id 1}
- [{:name "evil.csv" :filetype "csv" :mode "external" :url_private "https://evil.test/x.csv" :size 0}
- {:name "notes.pdf" :filetype "pdf" :mode "hosted" :url_private "https://files.slack.com/notes.pdf" :size 10}])]
- (is (= ["evil.csv"] remote))
- (is (= ["notes.pdf"] skipped))
- (is (empty? results)))))
+ (is (=? {:remote ["evil.csv"]
+ :skipped ["notes.pdf"]
+ :results empty?}
+ (#'slackbot.uploads/upload-files!
+ nil
+ {:db {:id 1}}
+ [{:name "evil.csv", :filetype "csv", :mode "external", :url_private "https://evil.test/x.csv", :size 0}
+ {:name "notes.pdf", :filetype "pdf", :mode "hosted", :url_private "https://files.slack.com/notes.pdf", :size 10}])))))
(deftest copy-to-file!-enforces-the-size-limit-test
(testing "a stream longer than the limit is refused, since the event only carries the size the sender declared"
(let [file (java.io.File/createTempFile "slackbot-cap-" ".csv")]
(try
- (with-redefs-fn {#'slackbot.uploads/max-file-size-bytes 8}
+ (with-redefs-fn {#'slackbot.uploads/max-slack-upload-size-bytes 8}
(fn []
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"exceeds"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"larger than"
(#'slackbot.uploads/copy-to-file!
(io/input-stream (.getBytes "0123456789abcdefghij"))
file
"big.csv")))))
(finally (io/delete-file file true))))))
+
+(deftest ^:synchronized file-uploads-metric-test
+ (testing "every attached file is counted once, under what became of it"
+ (mt/with-prometheus-system! [_ system]
+ (with-upload-mocks!
+ {:uploads-enabled? true}
+ (fn [_]
+ (slackbot.uploads/handle-file-uploads!
+ test-client
+ [tu/slack-csv-file
+ (assoc tu/slack-csv-file :name "huge.csv", :size (inc (* 200 1024 1024)))
+ {:name "notes.pdf", :filetype "pdf", :url_private "https://files.slack.com/notes.pdf", :size 10}
+ {:name "remote.csv", :filetype "csv", :mode "external", :url_private "https://example.com/x.csv", :size 10}])))
+ (mt/with-dynamic-fn-redefs [upload.db/current-database (constantly nil)]
+ (slackbot.uploads/handle-file-uploads! test-client [tu/slack-csv-file tu/slack-csv-file]))
+ (is (= {"success" 1.0
+ "error" 0.0
+ "too-large" 1.0
+ "unsupported" 1.0
+ "remote" 1.0
+ "unavailable" 2.0}
+ (into {}
+ (for [result ["success" "error" "too-large" "unsupported" "remote" "unavailable"]]
+ [result (mt/metric-value system :metabase-slackbot/file-uploads {:result result})])))))))