The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
SEC-1218: Only download slackbot files from Slack hosts (#82320)
Andrei Simionescu4d0744a8202f
diff --git a/src/metabase/slackbot/api.clj b/src/metabase/slackbot/api.cljindex 40b0078ec08..7f74f65d3af 100644--- a/src/metabase/slackbot/api.clj+++ b/src/metabase/slackbot/api.clj@@ -164,11 +164,24 @@:body "ok"})(defn- all-files-skipped?- "Returns true if all files were skipped (none were CSV/TSV)."+ "Returns true if nothing was uploaded and at least one file was refused."[{:keys [upload-result]}]- (let [{:keys [results skipped]} upload-result]- (and (seq skipped)- (empty? results))))+ (let [{:keys [results skipped remote]} upload-result]+ (and (empty? results)+ (or (seq skipped) (seq remote)))))++(defn- refused-files-message+ "Reply text for a share where nothing could be uploaded, naming why for each file."+ [{:keys [skipped remote]}]+ (str/join " "+ (cond-> []+ (seq skipped)+ (conj (format "I can only process CSV and TSV files. The following files were skipped: %s"+ (str/join ", " skipped)))++ (seq remote)+ (conj (format "These files are stored outside Slack, so I can't upload them: %s"+ (str/join ", " remote))))))(mu/defn- handle-message-file-share"Process a file_share message - handles CSV uploads"@@ -192,14 +205,12 @@should-skip-ai? (and (not has-text?)(not (:error file-handling))all-skipped?)]- ;; If all files were skipped (non-CSV) and there's no text, respond directly+ ;; If nothing could be uploaded and there's no text, respond directly;; without calling the AI to avoid sending an empty prompt(if should-skip-ai?- (let [skipped-files (get-in file-handling [:upload-result :skipped])]- (slackbot.client/post-message client- (merge (slackbot.events/event->reply-context event)- {:text (format "I can only process CSV and TSV files. The following files were skipped: %s"- (str/join ", " skipped-files))})))+ (slackbot.client/post-message client+ (merge (slackbot.events/event->reply-context event)+ {:text (refused-files-message (:upload-result file-handling))}))(slackbot.streaming/send-response client event extra-history))))(defmethod analytics.core/known-labels :metabase-slackbot/responses-generated [_]diff --git a/src/metabase/slackbot/client.clj b/src/metabase/slackbot/client.cljindex a6fb8b6b7a2..11190ee2339 100644--- a/src/metabase/slackbot/client.clj+++ b/src/metabase/slackbot/client.clj@@ -1,9 +1,13 @@(ns metabase.slackbot.client"Slack API client functions for Metabot slackbot."(:import- (java.io InputStream))+ (java.io InputStream)+ (java.net URI))(:require[clj-http.client :as http]+ [medley.core :as m]+ [metabase.util :as u]+ [metabase.util.http :as u.http][metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log]@@ -19,6 +23,10 @@"Socket (read) timeout for streaming API calls (start/append/stop)."5000)+(def ^:private download-timeout-ms+ "Socket and connection timeout for a Slack file download."+ 10000)+(def SlackClient"Malli schema for a Slack client."[:map {:closed true}@@ -193,14 +201,75 @@(:body (slack-post-json client "/views.open" {:trigger_id trigger_id:view view})))+(def ^:private slack-file-hosts+ "The hosts Slack serves file content from. The rest of `slack.com` is deliberately excluded: a remote file could+ otherwise name a Slack API endpoint, and the response to that token-bearing request is saved as a model."+ #{"files.slack.com" "files-origin.slack.com"})++(defn- slack-file-url?+ "Whether `url` is safe to fetch with the bot token attached: it clears the generic SSRF pre-check+ ([[metabase.util.http/safe-url?]]: https, no userinfo, a real external hostname) and its host is one of+ [[slack-file-hosts]]."+ [url]+ (and (u.http/safe-url? url)+ (boolean+ (when-let [host (some-> (u.http/->hostname url) u/lower-case-en)]+ (contains? slack-file-hosts host)))))++(defn- slack-file-request!+ "GET `url` with the bot token attached, leaving any redirect for the caller to vet."+ [client url]+ (http/get url (m/assoc-some+ {:headers {"Authorization" (str "Bearer " (:token client))}+ :as :stream+ :redirect-strategy :none+ :socket-timeout download-timeout-ms+ :connection-timeout download-timeout-ms+ :throw-exceptions false}+ ;; Under a JVM proxy clj-http resolves the proxy, not the target, so the+ ;; policy resolver would judge the wrong host; omit it and rely on the+ ;; Slack host allowlist above.+ :dns-resolver (when-not (u.http/jvm-proxied-url? url)+ (u.http/network-policy-dns-resolver :external-only)))))++(def ^:private redirect-statuses+ "Statuses whose `Location` is worth following."+ #{301 302 303 307 308})++(defn- redirect-target+ "The absolute URL `resp` redirects to, resolved against the `url` that produced it, or nil if it is not a redirect."+ [url resp]+ (when (contains? redirect-statuses (:status resp))+ (when-let [location (or (get-in resp [:headers "location"])+ (get-in resp [:headers "Location"]))]+ (str (.resolve (URI. (str url)) (str location))))))+(defn download-file-stream"Download a file from Slack, returning an InputStream instead of buffering in memory.- Caller is responsible for closing the stream (e.g. via `with-open`)."+ Caller is responsible for closing the stream (e.g. via `with-open`).++ Refuses a URL that is not one of Slack's file hosts: `url` arrives on a `file_share` event, and on a Slack+ remote file it is whatever address the app that registered the file picked. Slack may answer with a redirect+ to another of its file hosts, so one hop is followed, checked the same way before the token goes out again."^InputStream[client url]- (-> (http/get url {:headers {"Authorization" (str "Bearer " (:token client))}- :as :stream})- :body))+ (loop [url url, hops-left 1]+ (when-not (slack-file-url? url)+ (throw (ex-info (tru "Refusing to download a file from a non-Slack host.")+ {:status-code 400, :host (u.http/->hostname url)})))+ (let [resp (slack-file-request! client url)+ status (:status resp)+ ^InputStream body (:body resp)+ target (when (pos? hops-left) (redirect-target url resp))]+ (cond+ (<= 200 status 299) body++ target (do (some-> body .close)+ (recur target (dec hops-left)))++ :else (do (some-> body .close)+ (throw (ex-info (tru "Unexpected response downloading file from Slack (status {0})." status)+ {:status-code 502, :upstream-status status})))))));; -------------------- SLACK STREAMING API --------------------;; These functions implement Slack's chat streaming API for progressive AI responses.diff --git a/src/metabase/slackbot/events.clj b/src/metabase/slackbot/events.cljindex cb773d5d095..1f167c93df7 100644--- a/src/metabase/slackbot/events.clj+++ b/src/metabase/slackbot/events.clj@@ -25,6 +25,7 @@[:name :string][:mimetype {:optional true} [:maybe :string]][:filetype {:optional true} [:maybe :string]]+ [:mode {:optional true} [:maybe :string]][:url_private :string][:size :int]])diff --git a/src/metabase/slackbot/uploads.clj b/src/metabase/slackbot/uploads.cljindex 900ef0e8989..5de6dab641a 100644--- a/src/metabase/slackbot/uploads.clj+++ b/src/metabase/slackbot/uploads.clj@@ -25,11 +25,37 @@[{:keys [filetype]}](contains? allowed-csv-filetypes filetype))+(defn- remote-file?+ "Whether a Slack file is stored outside Slack, so its URL and size come from the app that registered it rather+ than from Slack. Those are never downloaded, whatever filetype they claim."+ [{:keys [mode]}]+ (= "external" mode))++(defn- size-limit-message+ "The user-facing message for a file over [[max-file-size-bytes]]."+ [filename]+ (format "File '%s' exceeds %dMB size limit" filename (quot max-file-size-bytes (* 1024 1024))))+(defn- validate-file-size"Returns nil if valid, error string if too large."[{:keys [name size]}](when (> size max-file-size-bytes)- (format "File '%s' exceeds %dMB size limit" name (quot max-file-size-bytes (* 1024 1024)))))+ (size-limit-message name)))++(defn- copy-to-file!+ "Copy `in` into `file`, refusing more than [[max-file-size-bytes]]. The size on the event is only what the sender+ declared, so the limit has to hold as the bytes arrive."+ [^java.io.InputStream in ^java.io.File file filename]+ (let [buf (byte-array 8192)]+ (with-open [^java.io.OutputStream out (io/output-stream file)]+ (loop [written 0]+ (let [n (.read in buf)]+ (when-not (neg? n)+ (let [total (+ written n)]+ (when (> total max-file-size-bytes)+ (throw (ex-info (size-limit-message filename) {:status-code 400})))+ (.write out buf 0 n)+ (recur total))))))))(defn- upload-settings"Get upload settings map. Returns nil if uploads are not enabled."@@ -50,7 +76,7 @@(let [temp-file (java.io.File/createTempFile "slack-upload-" (str "-" name))](try(with-open [^java.io.InputStream stream (slackbot.client/download-file-stream {:token (channel.settings/unobfuscated-slack-app-token)} url_private)]- (io/copy stream temp-file)+ (copy-to-file! stream temp-file name)(let [result (upload/create-csv-upload!{:filename name:file temp-file@@ -73,18 +99,24 @@(defn- process-file-uploads"Process all files from a Slack event. Returns a map with::results - seq of individual file results- :skipped - seq of non-CSV filenames that were skipped"+ :skipped - seq of non-CSV filenames that were skipped+ :remote - seq of filenames refused for being stored outside Slack"[settings files]- (let [{csv-files true other-files false} (group-by csv-file? files)- skipped (mapv :name other-files)]+ (let [{remote-files true local-files false} (group-by remote-file? files)+ {csv-files true other-files false} (group-by csv-file? local-files)+ skipped (mapv :name other-files)+ remote (mapv :name remote-files)](when (seq skipped)(log/debugf "[slackbot] Skipping %d non-CSV files" (count skipped)))+ (when (seq remote)+ (log/debugf "[slackbot] Refusing %d files stored outside Slack" (count remote))){:results (mapv (partial process-csv-file settings) csv-files)- :skipped skipped}))+ :skipped skipped+ :remote remote}))(defn- build-upload-system-messages"Build system messages to inject into AI request about uploads."- [{:keys [results skipped]}]+ [{:keys [results skipped remote]}](let [successes (filter :model-id results)failures (filter :error results)](cond-> []@@ -107,7 +139,12 @@(seq skipped)(conj {:role :assistant:content (format "The following message included 1 or more non-CSV files which are not supported: %s. Let them know only CSV files can be uploaded."- (str/join ", " skipped))}))))+ (str/join ", " skipped))})++ (seq remote)+ (conj {:role :assistant+ :content (format "The following message included 1 or more files stored outside Slack, which Metabase cannot upload: %s. Let them know the file has to be uploaded to Slack itself."+ (str/join ", " remote))}))))(defn handle-file-uploads"Handle file uploads if present. Returns nil if no files, otherwisediff --git a/test/metabase/slackbot/client_test.clj b/test/metabase/slackbot/client_test.cljnew file mode 100644index 00000000000..5dc1694ec69--- /dev/null+++ b/test/metabase/slackbot/client_test.clj@@ -0,0 +1,91 @@+(ns metabase.slackbot.client-test+ (:require+ [clj-http.client :as http]+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [metabase.slackbot.client :as slackbot.client]+ [metabase.util.http :as u.http]))++(set! *warn-on-reflection* true)++(defn- ok-stream+ "A clj-http 200 response whose body is a small CSV stream."+ []+ {:status 200 :body (io/input-stream (.getBytes "a,b"))})++(defn- redirect-to+ "A clj-http redirect response pointing at `location`."+ [location]+ {:status 302 :headers {"location" location} :body (io/input-stream (.getBytes ""))})++(deftest download-file-stream-test+ (let [client {:token "xoxb-secret"}]+ (testing "an unsafe, non-Slack or non-file-host URL is refused before any request is made"+ (doseq [url ["http://169.254.169.254/latest/meta-data/"+ "https://files.slack.com.evil.test/x.csv"+ "https://evilslack.com/x.csv"+ "http://files.slack.com/files-pri/T1-F1/data.csv"+ ;; the rest of slack.com must not receive the token: the response is saved as a model+ "https://slack.com/api/conversations.list"+ "https://metaboat.slack.com/files/U1/F1/x.csv"]]+ (let [called (atom false)]+ (with-redefs [http/get (fn [& _] (reset! called true) (ok-stream))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"non-Slack host"+ (slackbot.client/download-file-stream client url)))+ (is (false? @called) (str "no request for " url))))))+ (testing "a Slack URL is fetched with the token, without following redirects, under the network policy"+ (let [opts (atom nil)]+ (with-redefs [http/get (fn [_url options]+ (reset! opts options)+ (ok-stream))]+ (with-open [stream (slackbot.client/download-file-stream client "https://FILES.slack.com/files-pri/T1-F1/data.csv")]+ (is (= "a,b" (slurp stream))))+ (is (= "Bearer xoxb-secret" (get-in @opts [:headers "Authorization"])))+ (is (= :none (:redirect-strategy @opts)))+ (is (some? (:dns-resolver @opts))))))+ (testing "the CDN file host is accepted too"+ (with-redefs [http/get (fn [_url _options] (ok-stream))]+ (with-open [stream (slackbot.client/download-file-stream client "https://files-origin.slack.com/files-pri/T1-F1/data.csv")]+ (is (= "a,b" (slurp stream))))))+ (testing "a non-2xx response throws instead of streaming the body"+ (with-redefs [http/get (fn [_url _options] {:status 500 :body (io/input-stream (.getBytes "<html>"))})]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"(?i)unexpected response"+ (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))))+ (testing "a redirect to another Slack file host is followed once, with the token reattached"+ (let [calls (atom [])]+ (with-redefs [http/get (fn [url options]+ (swap! calls conj [url (get-in options [:headers "Authorization"])])+ (if (= url "https://files.slack.com/files-pri/T1-F1/data.csv")+ (redirect-to "https://files-origin.slack.com/files-pri/T1-F1/data.csv")+ (ok-stream)))]+ (with-open [stream (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")]+ (is (= "a,b" (slurp stream))))+ (is (= ["https://files.slack.com/files-pri/T1-F1/data.csv"+ "https://files-origin.slack.com/files-pri/T1-F1/data.csv"]+ (mapv first @calls)))+ (is (every? (fn [[_ auth]] (= "Bearer xoxb-secret" auth)) @calls)))))+ (testing "a redirect off Slack is refused rather than followed"+ (let [calls (atom 0)]+ (with-redefs [http/get (fn [_url _options]+ (swap! calls inc)+ (redirect-to "https://evil.test/x.csv"))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"non-Slack host"+ (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))+ (is (= 1 @calls) "the redirect target is never requested"))))+ (testing "a second redirect is refused instead of followed"+ (let [calls (atom 0)]+ (with-redefs [http/get (fn [_url _options]+ (swap! calls inc)+ (redirect-to "https://files-origin.slack.com/files-pri/T1-F1/data.csv"))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"(?i)unexpected response"+ (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")))+ (is (= 2 @calls) "one hop only"))))+ (testing "behind a JVM proxy the dns-resolver is omitted, leaving the target to the proxy"+ (let [opts (atom nil)]+ (with-redefs [u.http/jvm-proxied-url? (constantly true)+ http/get (fn [_url options]+ (reset! opts options)+ (ok-stream))]+ (with-open [stream (slackbot.client/download-file-stream client "https://files.slack.com/files-pri/T1-F1/data.csv")]+ (is (= "a,b" (slurp stream))))+ (is (not (contains? @opts :dns-resolver))))))))diff --git a/test/metabase/slackbot/uploads_test.clj b/test/metabase/slackbot/uploads_test.cljindex 6c43b7af9f7..abd9873367a 100644--- a/test/metabase/slackbot/uploads_test.clj+++ b/test/metabase/slackbot/uploads_test.clj@@ -306,4 +306,33 @@(is (false? (#'slackbot.uploads/csv-file? {:filetype "pdf"})))(is (false? (#'slackbot.uploads/csv-file? {:filetype "xlsx"})))(is (false? (#'slackbot.uploads/csv-file? {:filetype nil})))- (is (false? (#'slackbot.uploads/csv-file? {})))))+ (is (false? (#'slackbot.uploads/csv-file? {}))))+ (testing "a file stored outside Slack is refused whatever filetype it claims, ordinary upload modes are kept"+ (is (true? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "external"})))+ (is (false? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "snippet"})))+ (is (false? (#'slackbot.uploads/remote-file? {:filetype "csv" :mode "hosted"})))+ (is (false? (#'slackbot.uploads/remote-file? {:filetype "csv"})))))++(deftest ^:parallel remote-files-are-reported-separately-test+ (testing "a remote CSV is refused for being remote, not reported as an unsupported filetype"+ (let [{:keys [skipped remote results]}+ (#'slackbot.uploads/process-file-uploads+ {:db_id 1}+ [{:name "evil.csv" :filetype "csv" :mode "external" :url_private "https://evil.test/x.csv" :size 0}+ {:name "notes.pdf" :filetype "pdf" :mode "hosted" :url_private "https://files.slack.com/notes.pdf" :size 10}])]+ (is (= ["evil.csv"] remote))+ (is (= ["notes.pdf"] skipped))+ (is (empty? results)))))++(deftest copy-to-file!-enforces-the-size-limit-test+ (testing "a stream longer than the limit is refused, since the event only carries the size the sender declared"+ (let [file (java.io.File/createTempFile "slackbot-cap-" ".csv")]+ (try+ (with-redefs-fn {#'slackbot.uploads/max-file-size-bytes 8}+ (fn []+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"exceeds"+ (#'slackbot.uploads/copy-to-file!+ (io/input-stream (.getBytes "0123456789abcdefghij"))+ file+ "big.csv")))))+ (finally (io/delete-file file true))))))