Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

SEC-1218: Only download slackbot files from Slack hosts (#82320)

Andrei Simionescu4d0744a8202f
diff --git a/src/metabase/slackbot/api.clj b/src/metabase/slackbot/api.clj
index 40b0078ec08..7f74f65d3af 100644
--- a/src/metabase/slackbot/api.clj
+++ b/src/metabase/slackbot/api.clj
@@ -164,11 +164,24 @@
:body "ok"})
(defn- all-files-skipped?
- "Returns true if all files were skipped (none were CSV/TSV)."
+ "Returns true if nothing was uploaded and at least one file was refused."
[{:keys [upload-result]}]
- (let [{:keys [results skipped]} upload-result]
- (and (seq skipped)
- (empty? results))))
+ (let [{:keys [results skipped remote]} upload-result]
+ (and (empty? results)
+ (or (seq skipped) (seq remote)))))
+
+(defn- refused-files-message
+ "Reply text for a share where nothing could be uploaded, naming why for each file."
+ [{:keys [skipped remote]}]
+ (str/join " "
+ (cond-> []
+ (seq skipped)
+ (conj (format "I can only process CSV and TSV files. The following files were skipped: %s"
+ (str/join ", " skipped)))
+
+ (seq remote)
+ (conj (format "These files are stored outside Slack, so I can't upload them: %s"
+ (str/join ", " remote))))))
(mu/defn- handle-message-file-share
"Process a file_share message - handles CSV uploads"
@@ -192,14 +205,12 @@
should-skip-ai? (and (not has-text?)
(not (:error file-handling))
all-skipped?)]
- ;; If all files were skipped (non-CSV) and there's no text, respond directly
+ ;; If nothing could be uploaded and there's no text, respond directly
;; without calling the AI to avoid sending an empty prompt
(if should-skip-ai?
- (let [skipped-files (get-in file-handling [:upload-result :skipped])]
- (slackbot.client/post-message client
- (merge (slackbot.events/event->reply-context event)
- {:text (format "I can only process CSV and TSV files. The following files were skipped: %s"
- (str/join ", " skipped-files))})))
+ (slackbot.client/post-message client
+ (merge (slackbot.events/event->reply-context event)
+ {:text (refused-files-message (:upload-result file-handling))}))
(slackbot.streaming/send-response client event extra-history))))
(defmethod analytics.core/known-labels :metabase-slackbot/responses-generated [_]
diff --git a/src/metabase/slackbot/client.clj b/src/metabase/slackbot/client.clj
index a6fb8b6b7a2..11190ee2339 100644
--- a/src/metabase/slackbot/client.clj
+++ b/src/metabase/slackbot/client.clj
@@ -1,9 +1,13 @@
(ns metabase.slackbot.client
"Slack API client functions for Metabot slackbot."
(:import
- (java.io InputStream))
+ (java.io InputStream)
+ (java.net URI))
(:require
[clj-http.client :as http]
+ [medley.core :as m]
+ [metabase.util :as u]
+ [metabase.util.http :as u.http]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
[metabase.util.log :as log]
@@ -19,6 +23,10 @@
"Socket (read) timeout for streaming API calls (start/append/stop)."
5000)
+(def ^:private download-timeout-ms
+ "Socket and connection timeout for a Slack file download."
+ 10000)
+
(def SlackClient
"Malli schema for a Slack client."
[:map {:closed true}
@@ -193,14 +201,75 @@
(:body (slack-post-json client "/views.open" {:trigger_id trigger_id
:view view})))
+(def ^:private slack-file-hosts
+ "The hosts Slack serves file content from. The rest of `slack.com` is deliberately excluded: a remote file could
+ otherwise name a Slack API endpoint, and the response to that token-bearing request is saved as a model."
+ #{"files.slack.com" "files-origin.slack.com"})
+
+(defn- slack-file-url?
+ "Whether `url` is safe to fetch with the bot token attached: it clears the generic SSRF pre-check
+ ([[metabase.util.http/safe-url?]]: https, no userinfo, a real external hostname) and its host is one of
+ [[slack-file-hosts]]."
+ [url]
+ (and (u.http/safe-url? url)
+ (boolean
+ (when-let [host (some-> (u.http/->hostname url) u/lower-case-en)]
+ (contains? slack-file-hosts host)))))
+
+(defn- slack-file-request!
+ "GET `url` with the bot token attached, leaving any redirect for the caller to vet."
+ [client url]
+ (http/get url (m/assoc-some
+ {:headers {"Authorization" (str "Bearer " (:token client))}
+ :as :stream
+ :redirect-strategy :none
+ :socket-timeout download-timeout-ms
+ :connection-timeout download-timeout-ms
+ :throw-exceptions false}
+ ;; Under a JVM proxy clj-http resolves the proxy, not the target, so the
+ ;; policy resolver would judge the wrong host; omit it and rely on the
+ ;; Slack host allowlist above.
+ :dns-resolver (when-not (u.http/jvm-proxied-url? url)
+ (u.http/network-policy-dns-resolver :external-only)))))
+
+(def ^:private redirect-statuses
+ "Statuses whose `Location` is worth following."
+ #{301 302 303 307 308})
+
+(defn- redirect-target
+ "The absolute URL `resp` redirects to, resolved against the `url` that produced it, or nil if it is not a redirect."
+ [url resp]
+ (when (contains? redirect-statuses (:status resp))
+ (when-let [location (or (get-in resp [:headers "location"])
+ (get-in resp [:headers "Location"]))]
+ (str (.resolve (URI. (str url)) (str location))))))
+
(defn download-file-stream
"Download a file from Slack, returning an InputStream instead of buffering in memory.
- Caller is responsible for closing the stream (e.g. via `with-open`)."
+ Caller is responsible for closing the stream (e.g. via `with-open`).
+
+ Refuses a URL that is not one of Slack's file hosts: `url` arrives on a `file_share` event, and on a Slack
+ remote file it is whatever address the app that registered the file picked. Slack may answer with a redirect
+ to another of its file hosts, so one hop is followed, checked the same way before the token goes out again."
^InputStream
[client url]
- (-> (http/get url {:headers {"Authorization" (str "Bearer " (:token client))}
- :as :stream})
- :body))
+ (loop [url url, hops-left 1]
+ (when-not (slack-file-url? url)
+ (throw (ex-info (tru "Refusing to download a file from a non-Slack host.")
+ {:status-code 400, :host (u.http/->hostname url)})))
+ (let [resp (slack-file-request! client url)
+ status (:status resp)
+ ^InputStream body (:body resp)
+ target (when (pos? hops-left) (redirect-target url resp))]
+ (cond
+ (<= 200 status 299) body
+
+ target (do (some-> body .close)
+ (recur target (dec hops-left)))
+
+ :else (do (some-> body .close)
+ (throw (ex-info (tru "Unexpected response downloading file from Slack (status {0})." status)
+ {:status-code 502, :upstream-status status})))))))
;; -------------------- SLACK STREAMING API --------------------
;; These functions implement Slack's chat streaming API for progressive AI responses.
diff --git a/src/metabase/slackbot/events.clj b/src/metabase/slackbot/events.clj
index cb773d5d095..1f167c93df7 100644
--- a/src/metabase/slackbot/events.clj
+++ b/src/metabase/slackbot/events.clj
@@ -25,6 +25,7 @@
[:name :string]
[:mimetype {:optional true} [:maybe :string]]
[:filetype {:optional true} [:maybe :string]]
+ [:mode {:optional true} [:maybe :string]]
[:url_private :string]
[:size :int]])
diff --git a/src/metabase/slackbot/uploads.clj b/src/metabase/slackbot/uploads.clj
index 900ef0e8989..5de6dab641a 100644
--- a/src/metabase/slackbot/uploads.clj
+++ b/src/metabase/slackbot/uploads.clj
@@ -25,11 +25,37 @@
[{:keys [filetype]}]
(contains? allowed-csv-filetypes filetype))
+(defn- remote-file?
+ "Whether a Slack file is stored outside Slack, so its URL and size come from the app that registered it rather
+ than from Slack. Those are never downloaded, whatever filetype they claim."
+ [{:keys [mode]}]
+ (= "external" mode))
+
+(defn- size-limit-message
+ "The user-facing message for a file over [[max-file-size-bytes]]."
+ [filename]
+ (format "File '%s' exceeds %dMB size limit" filename (quot max-file-size-bytes (* 1024 1024))))
+
(defn- validate-file-size
"Returns nil if valid, error string if too large."
[{:keys [name size]}]
(when (> size max-file-size-bytes)
- (format "File '%s' exceeds %dMB size limit" name (quot max-file-size-bytes (* 1024 1024)))))
+ (size-limit-message name)))
+
+(defn- copy-to-file!
+ "Copy `in` into `file`, refusing more than [[max-file-size-bytes]]. The size on the event is only what the sender
+ declared, so the limit has to hold as the bytes arrive."
+ [^java.io.InputStream in ^java.io.File file filename]
+ (let [buf (byte-array 8192)]
+ (with-open [^java.io.OutputStream out (io/output-stream file)]
+ (loop [written 0]
+ (let [n (.read in buf)]
+ (when-not (neg? n)
+ (let [total (+ written n)]
+ (when (> total max-file-size-bytes)
+ (throw (ex-info (size-limit-message filename) {:status-code 400})))
+ (.write out buf 0 n)
+ (recur total))))))))
(defn- upload-settings
"Get upload settings map. Returns nil if uploads are not enabled."
@@ -50,7 +76,7 @@
(let [temp-file (java.io.File/createTempFile "slack-upload-" (str "-" name))]
(try
(with-open [^java.io.InputStream stream (slackbot.client/download-file-stream {:token (channel.settings/unobfuscated-slack-app-token)} url_private)]
- (io/copy stream temp-file)
+ (copy-to-file! stream temp-file name)
(let [result (upload/create-csv-upload!
{:filename name
:file temp-file
@@ -73,18 +99,24 @@
(defn- process-file-uploads
"Process all files from a Slack event. Returns a map with:
:results - seq of individual file results
- :skipped - seq of non-CSV filenames that were skipped"
+ :skipped - seq of non-CSV filenames that were skipped
+ :remote - seq of filenames refused for being stored outside Slack"
[settings files]
- (let [{csv-files true other-files false} (group-by csv-file? files)
- skipped (mapv :name other-files)]
+ (let [{remote-files true local-files false} (group-by remote-file? files)
+ {csv-files true other-files false} (group-by csv-file? local-files)
+ skipped (mapv :name other-files)
+ remote (mapv :name remote-files)]
(when (seq skipped)
(log/debugf "[slackbot] Skipping %d non-CSV files" (count skipped)))
+ (when (seq remote)
+ (log/debugf "[slackbot] Refusing %d files stored outside Slack" (count remote)))
{:results (mapv (partial process-csv-file settings) csv-files)
- :skipped skipped}))
+ :skipped skipped
+ :remote remote}))
(defn- build-upload-system-messages
"Build system messages to inject into AI request about uploads."
- [{:keys [results skipped]}]
+ [{:keys [results skipped remote]}]
(let [successes (filter :model-id results)
failures (filter :error results)]
(cond-> []
@@ -107,7 +139,12 @@
(seq skipped)
(conj {:role :assistant
:content (format "The following message included 1 or more non-CSV files which are not supported: %s. Let them know only CSV files can be uploaded."
- (str/join ", " skipped))}))))
+ (str/join ", " skipped))})
+
+ (seq remote)
+ (conj {:role :assistant
+ :content (format "The following message included 1 or more files stored outside Slack, which Metabase cannot upload: %s. Let them know the file has to be uploaded to Slack itself."
+ (str/join ", " remote))}))))
(defn handle-file-uploads
"Handle file uploads if present. Returns nil if no files, otherwise
diff --git a/test/metabase/slackbot/client_test.clj b/test/metabase/slackbot/client_test.clj
new file mode 100644
index 00000000000..5dc1694ec69
--- /dev/null
+++ b/test/metabase/slackbot/client_test.clj
@@ -0,0 +1,91 @@
+(ns metabase.slackbot.client-test
+ (:require
+ [clj-http.client :as http]
+ [clojure.java.io :as io]
+ [clojure.test :refer :all]
+ [metabase.slackbot.client :as slackbot.client]
+ [metabase.util.http :as u.http]))
+
+(set! *warn-on-reflection* true)
+
+(defn- ok-stream
+ "A clj-http 200 response whose body is a small CSV stream."
+ []
+ {:status 200 :body (io/input-stream (.getBytes "a,b"))})
+
+(defn- redirect-to
+ "A clj-http redirect response pointing at `location`."
+ [location]
+ {:status 302 :headers {"location" location} :body (io/input-stream (.getBytes ""))})
+
+(deftest download-file-stream-test
+ (let [client {:token "xoxb-secret"}]
+ (testing "an unsafe, non-Slack or non-file-host URL is refused before any request is made"
+ (doseq [url ["http://169.254.169.254/latest/meta-data/"
+ "https://files.slack.com.evil.test/x.csv"
+ "https://evilslack.com/x.csv"
+ "http://files.slack.com/files-pri/T1-F1/data.csv"
+ ;; the rest of slack.com must not receive the token: the response is saved as a model
+ "https://slack.com/api/conversations.list"
+ "https://metaboat.slack.com/files/U1/F1/x.csv"]]
+ (let [called (atom false)]
+ (with-redefs [http/get (fn [& _] (reset! called true) (ok-stream))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"non-Slack host"
+ (slackbot.client/download-file-stream client url)))
+ (is (false? @called) (str "no request for " url))))))
+ (testing "a Slack URL is fetched with the token, without following redirects, under the network policy"
+ (let [opts (atom nil)]
+ (with-redefs [http/get (fn [_url options]
+ (reset! opts options)
+ (ok-stream))]
+ (with-open [stream (slackbot.client/download-file-stream client "https://FILES.slack.com/files-pri/T1-F1/data.csv")]
+ (is (= "a,b" (slurp stream))))
+ (is (= "Bearer xoxb-secret" (get-in @opts [:headers "Authorization"])))
+ (is (= :none (:redirect-strategy @opts)))
+ (is (some? (:dns-resolver @opts))))))
+ (testing "the CDN file host is accepted too"
+ (with-redefs [http/get (fn [_url _options] (ok-stream))]
+ (with-open [stream (slackbot.client/download-file-stream client "https://files-origin.slack.com/files-pri/T1-F1/data.csv")]
+ (is (= "a,b" (slurp stream))))))
+ (testing "a non-2xx response throws instead of streaming the body"
+ (with-redefs [http/get (fn [_url _options] {:status 500 :body (io/input-stream (.getBytes "<html>"))})]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"(?i)unexpected response"
+ (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))))
+ (testing "a redirect to another Slack file host is followed once, with the token reattached"
+ (let [calls (atom [])]
+ (with-redefs [http/get (fn [url options]
+ (swap! calls conj [url (get-in options [:headers "Authorization"])])
+ (if (= url "https://files.slack.com/files-pri/T1-F1/data.csv")
+ (redirect-to "https://files-origin.slack.com/files-pri/T1-F1/data.csv")
+ (ok-stream)))]
+ (with-open [stream (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")]
+ (is (= "a,b" (slurp stream))))
+ (is (= ["https://files.slack.com/files-pri/T1-F1/data.csv"
+ "https://files-origin.slack.com/files-pri/T1-F1/data.csv"]
+ (mapv first @calls)))
+ (is (every? (fn [[_ auth]] (= "Bearer xoxb-secret" auth)) @calls)))))
+ (testing "a redirect off Slack is refused rather than followed"
+ (let [calls (atom 0)]
+ (with-redefs [http/get (fn [_url _options]
+ (swap! calls inc)
+ (redirect-to "https://evil.test/x.csv"))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"non-Slack host"
+ (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))
+ (is (= 1 @calls) "the redirect target is never requested"))))
+ (testing "a second redirect is refused instead of followed"
+ (let [calls (atom 0)]
+ (with-redefs [http/get (fn [_url _options]
+ (swap! calls inc)
+ (redirect-to "https://files-origin.slack.com/files-pri/T1-F1/data.csv"))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"(?i)unexpected response"
+ (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))
+ (is (= 2 @calls) "one hop only"))))
+ (testing "behind a JVM proxy the dns-resolver is omitted, leaving the target to the proxy"
+ (let [opts (atom nil)]
+ (with-redefs [u.http/jvm-proxied-url? (constantly true)
+ http/get (fn [_url options]
+ (reset! opts options)
+ (ok-stream))]
+ (with-open [stream (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")]
+ (is (= "a,b" (slurp stream))))
+ (is (not (contains? @opts :dns-resolver))))))))
diff --git a/test/metabase/slackbot/uploads_test.clj b/test/metabase/slackbot/uploads_test.clj
index 6c43b7af9f7..abd9873367a 100644
--- a/test/metabase/slackbot/uploads_test.clj
+++ b/test/metabase/slackbot/uploads_test.clj
@@ -306,4 +306,33 @@
(is (false? (#'slackbot.uploads/csv-file? {:filetype "pdf"})))
(is (false? (#'slackbot.uploads/csv-file? {:filetype "xlsx"})))
(is (false? (#'slackbot.uploads/csv-file? {:filetype nil})))
- (is (false? (#'slackbot.uploads/csv-file? {})))))
+ (is (false? (#'slackbot.uploads/csv-file? {}))))
+ (testing "a file stored outside Slack is refused whatever filetype it claims, ordinary upload modes are kept"
+ (is (true? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "external"})))
+ (is (false? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "snippet"})))
+ (is (false? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "hosted"})))
+ (is (false? (#'slackbot.uploads/remote-file? {:filetype "csv"})))))
+
+(deftest ^:parallel remote-files-are-reported-separately-test
+ (testing "a remote CSV is refused for being remote, not reported as an unsupported filetype"
+ (let [{:keys [skipped remote results]}
+ (#'slackbot.uploads/process-file-uploads
+ {:db_id 1}
+ [{:name "evil.csv" :filetype "csv" :mode "external" :url_private "https://evil.test/x.csv" :size 0}
+ {:name "notes.pdf" :filetype "pdf" :mode "hosted" :url_private "https://files.slack.com/notes.pdf" :size 10}])]
+ (is (= ["evil.csv"] remote))
+ (is (= ["notes.pdf"] skipped))
+ (is (empty? results)))))
+
+(deftest copy-to-file!-enforces-the-size-limit-test
+ (testing "a stream longer than the limit is refused, since the event only carries the size the sender declared"
+ (let [file (java.io.File/createTempFile "slackbot-cap-" ".csv")]
+ (try
+ (with-redefs-fn {#'slackbot.uploads/max-file-size-bytes 8}
+ (fn []
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"exceeds"
+ (#'slackbot.uploads/copy-to-file!
+ (io/input-stream (.getBytes "0123456789abcdefghij"))
+ file
+ "big.csv")))))
+ (finally (io/delete-file file true))))))