Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Limit unauthenticated request body sizes (#82572)

Nathan Voxland76979e61674f
diff --git a/src/metabase/server/handler.clj b/src/metabase/server/handler.clj
index 9f2ff29a77a..e35d0a2a815 100644
--- a/src/metabase/server/handler.clj
+++ b/src/metabase/server/handler.clj
@@ -6,6 +6,7 @@
[metabase.api.macros :as api.macros]
[metabase.config.core :as config]
[metabase.server.middleware.auth :as mw.auth]
+ [metabase.server.middleware.body-limit :as mw.body-limit]
[metabase.server.middleware.browser-cookie :as mw.browser-cookie]
[metabase.server.middleware.data-app-scope :as mw.data-app-scope]
[metabase.server.middleware.exceptions :as mw.exceptions]
@@ -95,6 +96,7 @@
#'wrap-keyword-params ; converts string keys in :params to keyword keys
#'wrap-params ; parses GET and POST params as :query-params/:form-params and both as :params
#'mw.auth/verify-slack-request ; looks for requests from slack and assocs a :slack/validated? on the request if valid
+ #'mw.body-limit/wrap-limit-request-body ; bounds unauthenticated request body sizes; must be inside wrap-current-user-info and outside everything that reads the body
#'mw.misc/maybe-set-site-url ; set the value of `site-url` if it hasn't been set yet
#'mw.session/reset-session-timeout ; Resets the timeout cookie for user activity to [[metabase.request.cookies/session-timeout]]
#'mw.session/bind-current-user ; Binds *current-user* and *current-user-id* if :metabase-user-id is non-nil
diff --git a/src/metabase/server/middleware/auth.clj b/src/metabase/server/middleware/auth.clj
index 68445f411d7..4af5df6d5e3 100644
--- a/src/metabase/server/middleware/auth.clj
+++ b/src/metabase/server/middleware/auth.clj
@@ -5,6 +5,7 @@
[buddy.core.bytes :as bytes]
[buddy.core.codecs :as codecs]
[buddy.core.mac :as mac]
+ [clojure.string :as str]
[metabase.server.settings :as server.settings]))
(set! *warn-on-reflection* true)
@@ -55,30 +56,44 @@
(defn- verify-slack-signature
"Verify that the request came from Slack using signature verification.
- Returns nil if no signing secret is configured, false if timestamp is too old
- (replay attack prevention) or signature is invalid, true if valid."
- [request-body timestamp slack-signature]
- (when-let [signing-secret (server.settings/unobfuscated-metabot-slack-signing-secret)]
- (and (slack-timestamp-valid? timestamp)
- (some? slack-signature)
- (let [message (str "v0:" timestamp ":" request-body)
- computed-signature (hmac-sha256 signing-secret message)
- expected-signature (str "v0=" computed-signature)]
- ;; Use constant-time comparison to prevent timing attacks
- (bytes/equals? (.getBytes ^String expected-signature "UTF-8")
- (.getBytes ^String slack-signature "UTF-8"))))))
+ Returns false if timestamp is too old (replay attack prevention) or signature is invalid, true if valid."
+ [signing-secret request-body timestamp slack-signature]
+ (and (slack-timestamp-valid? timestamp)
+ (some? slack-signature)
+ (let [message (str "v0:" timestamp ":" request-body)
+ computed-signature (hmac-sha256 signing-secret message)
+ expected-signature (str "v0=" computed-signature)]
+ ;; Use constant-time comparison to prevent timing attacks
+ (bytes/equals? (.getBytes ^String expected-signature "UTF-8")
+ (.getBytes ^String slack-signature "UTF-8")))))
+
+(def ^:private slack-routes-prefix
+ "Where the Slack endpoints that consume `:slack/validated?` are mounted (see `metabase.metabot.api/routes`)."
+ "/api/metabot/slack")
+
+(defn- slack-route? [{:keys [uri]}]
+ (and uri (str/starts-with? uri slack-routes-prefix)))
(defn verify-slack-request
"Middleware that detects if an incoming request is from Slack and sets the `:slack/validated?` keyword on a request
- with a boolean if that request has been correctly signed with our signing secret."
+ with a boolean if that request has been correctly signed with our signing secret.
+
+ This runs before authentication on every route, so it only reads the body for requests to the Slack routes when a
+ signing secret is configured and the caller sent a signature header. The body itself is bounded by
+ [[metabase.server.middleware.body-limit/wrap-limit-request-body]], which must sit outside this middleware."
[handler]
(fn
[req respond raise]
- (let [signature (get-in req [:headers "x-slack-signature"])]
- (if signature
+ (let [signature (when (slack-route? req)
+ (get-in req [:headers "x-slack-signature"]))
+ signing-secret (when signature
+ (server.settings/unobfuscated-metabot-slack-signing-secret))]
+ (if signing-secret
(let [timestamp (get-in req [:headers "x-slack-request-timestamp"])
- str-body (slurp (:body req))
- valid? (verify-slack-signature str-body timestamp signature)]
+ str-body (if-let [body (:body req)]
+ (slurp body)
+ "")
+ valid? (verify-slack-signature signing-secret str-body timestamp signature)]
(handler (-> req
(assoc :body (java.io.ByteArrayInputStream. (.getBytes ^String str-body "UTF-8")))
(assoc :slack/validated? valid?))
diff --git a/src/metabase/server/middleware/body_limit.clj b/src/metabase/server/middleware/body_limit.clj
new file mode 100644
index 00000000000..32b78745c56
--- /dev/null
+++ b/src/metabase/server/middleware/body_limit.clj
@@ -0,0 +1,67 @@
+(ns metabase.server.middleware.body-limit
+ "Middleware that bounds the size of request bodies before anything reads them to protect memory.
+
+ [[wrap-limit-request-body]] sits inside [[metabase.server.middleware.session/wrap-current-user-info]], so it knows
+ whether the caller is authenticated, and outside every middleware that reads the body, so the bound is in place
+ before the first byte is read. Only unauthenticated requests are bounded."
+ (:require
+ [metabase.server.middleware.exceptions :as mw.exceptions]
+ [metabase.server.settings :as server.settings]
+ [metabase.util :as u]
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.log :as log])
+ (:import
+ (java.io InputStream)
+ (org.apache.commons.io.input BoundedInputStream)))
+
+(set! *warn-on-reflection* true)
+
+(defn- body-too-large-exception [max-bytes]
+ (ex-info (tru "Request body exceeds the maximum size of {0} bytes." max-bytes)
+ {:status-code 413, :type ::body-too-large}))
+
+(defn bounded-input-stream
+ "Wrap `in` so that reading more than `max-bytes` bytes from it throws a 413 exception.
+
+ The bound is `max-bytes` + 1 so that a body of exactly `max-bytes` still reads through to EOF; that extra byte is
+ also the most this ever pulls out of `in` past the limit. Once the bound is hit every subsequent read throws,
+ rather than reporting EOF and letting the caller treat a truncated body as the whole thing."
+ ^InputStream [^InputStream in max-bytes]
+ (let [max-bytes (long max-bytes)]
+ (.get (doto (BoundedInputStream/builder)
+ (.setInputStream in)
+ (.setMaxCount (inc max-bytes))
+ (.setOnMaxCount (fn [_max-count _count]
+ (throw (body-too-large-exception max-bytes))))))))
+
+(defn- limit-request-body
+ "For an unauthenticated `request`, reject it outright when its declared `Content-Length` is over the limit;
+ otherwise bound `:body` so a chunked body can't sneak past it. Authenticated requests pass through untouched."
+ [{:keys [body content-length metabase-user-id], :as request}]
+ (if (or (nil? body) metabase-user-id)
+ request
+ (let [max-bytes (server.settings/max-unauthenticated-request-body-bytes)]
+ (when (and content-length (> content-length max-bytes))
+ (throw (body-too-large-exception max-bytes)))
+ (assoc request :body (bounded-input-stream body max-bytes)))))
+
+(defn wrap-limit-request-body
+ "Middleware that bounds unauthenticated request bodies, responding with 413 when a body is larger than
+ [[server.settings/max-unauthenticated-request-body-bytes]]. Must sit inside
+ [[metabase.server.middleware.session/wrap-current-user-info]] and outside every middleware that reads the body."
+ [handler]
+ (fn [request respond raise]
+ (try
+ (handler (limit-request-body request) respond raise)
+ ;; body readers on the global stack sit outside the API exception middleware, so a limit exceeded there
+ ;; surfaces here as a throw rather than through `raise`
+ (catch clojure.lang.ExceptionInfo e
+ (if (= ::body-too-large (:type (ex-data e)))
+ (do
+ ;; this responds before `log-api-call`, so note the rejection here
+ (log/infof "Rejected %s %s: %s"
+ (some-> (:request-method request) name u/upper-case-en)
+ (:uri request)
+ (ex-message e))
+ (respond (mw.exceptions/api-exception-response e request)))
+ (throw e))))))
diff --git a/src/metabase/server/settings.clj b/src/metabase/server/settings.clj
index dedac4e50b9..b1d06971306 100644
--- a/src/metabase/server/settings.clj
+++ b/src/metabase/server/settings.clj
@@ -118,6 +118,15 @@ x.com")
"Timeout in milliseconds to wait after query cancellation before escalating to thread interruption."
(thread-interrupt-escalation-timeout-ms))
+(defsetting max-unauthenticated-request-body-bytes
+ (deferred-tru "Maximum size in bytes of an HTTP request body from a caller who is not authenticated. Larger requests are rejected with HTTP 413 before the body is read.")
+ :type :integer
+ :visibility :internal
+ :setter :none
+ :export? false
+ :default (* 4 1024 1024)
+ :doc "Applies to requests without a valid session, API key or token. The body is bounded before authentication is checked, so this is the most memory an anonymous request can make the server buffer. It covers login, setup, password reset, SSO callbacks, and public or embedded queries; none of those legitimately send more than a few hundred kilobytes. Authenticated requests are not limited.")
+
(defsetting metabot-slack-signing-secret
(deferred-tru "Signing secret for verifying requests from the Metabot Slack app")
:type :string
diff --git a/test/metabase/metabot/middleware/auth_test.clj b/test/metabase/metabot/middleware/auth_test.clj
index 3c42506539a..2bbcf566bc6 100644
--- a/test/metabase/metabot/middleware/auth_test.clj
+++ b/test/metabase/metabot/middleware/auth_test.clj
@@ -5,8 +5,11 @@
[buddy.core.mac :as mac]
[clojure.test :refer :all]
[metabase.server.middleware.auth :as mw.auth]
+ [metabase.server.middleware.body-limit-test :as body-limit-test]
[metabase.test :as mt]
- [ring.mock.request :as ring.mock]))
+ [ring.mock.request :as ring.mock])
+ (:import
+ (java.io ByteArrayInputStream)))
(set! *warn-on-reflection* true)
@@ -32,10 +35,12 @@
identity
(fn [e] (throw e))))
+(def ^:private slack-events-uri "/api/metabot/slack/events")
+
(defn- slack-request
- "Create a mock request with Slack signature headers and a body"
+ "Create a mock request to the Slack events route with Slack signature headers and a body"
^java.util.Map [^String body ^String timestamp ^String signature]
- (-> (ring.mock/request :post "/anyurl")
+ (-> (ring.mock/request :post slack-events-uri)
(ring.mock/header "x-slack-signature" signature)
(ring.mock/header "x-slack-request-timestamp" timestamp)
(assoc :body (java.io.ByteArrayInputStream. (.getBytes body "UTF-8")))))
@@ -70,6 +75,32 @@
result (wrapped-slack-handler (slack-request body timestamp signature))]
(is (nil? (:slack/validated? result))))))))
+(defn- slack-request-with-counted-body
+ "A signed-looking request whose `size`-byte body counts the bytes read from it into `counter`."
+ [size counter]
+ (assoc (slack-request "" (str test-timestamp) "v0=whatever")
+ :body (body-limit-test/counting-stream (ByteArrayInputStream. (byte-array size)) counter)))
+
+(deftest verify-slack-request-body-bound-test
+ (testing "No signing secret configured: the body is not read at all"
+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret nil]
+ (let [counter (atom 0)
+ result (wrapped-slack-handler (slack-request-with-counted-body 4096 counter))]
+ (is (not (contains? result :slack/validated?)))
+ (is (= 0 @counter)))))
+ (testing "Signing secret configured but the request is not for a Slack route: the body is not read at all"
+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]
+ (let [counter (atom 0)
+ result (wrapped-slack-handler (assoc (slack-request-with-counted-body 4096 counter) :uri "/api/card"))]
+ (is (not (contains? result :slack/validated?)))
+ (is (= 0 @counter)))))
+ (testing "Signing secret configured and a Slack route: the body is read, bounded by the global request body limit"
+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]
+ (let [counter (atom 0)
+ result (wrapped-slack-handler (slack-request-with-counted-body 4096 counter))]
+ (is (false? (:slack/validated? result)))
+ (is (= 4096 @counter))))))
+
(defn- validate-request-with-timestamp
"Helper to test timestamp validation. Returns :slack/validated? result."
[timestamp]
diff --git a/test/metabase/server/middleware/body_limit_test.clj b/test/metabase/server/middleware/body_limit_test.clj
new file mode 100644
index 00000000000..07b7cf4014c
--- /dev/null
+++ b/test/metabase/server/middleware/body_limit_test.clj
@@ -0,0 +1,199 @@
+(ns metabase.server.middleware.body-limit-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.server.middleware.body-limit :as mw.body-limit]
+ [metabase.server.test-handler :as server.test-handler]
+ [metabase.test :as mt]
+ [metabase.test.data.users :as test.users]
+ [metabase.test.fixtures :as fixtures])
+ (:import
+ (java.io ByteArrayInputStream InputStream)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db :web-server :test-users))
+
+(defn counting-stream
+ "Wrap `in` so every byte read from it is counted in the `counter` atom."
+ ^InputStream [^InputStream in counter]
+ (proxy [InputStream] []
+ (read
+ ([]
+ (let [b (.read in)]
+ (when (>= b 0) (swap! counter inc))
+ b))
+ ([^bytes buf off len]
+ (let [n (.read in buf (int off) (int len))]
+ (when (pos? n) (swap! counter + n))
+ n)))
+ (close [] (.close in))))
+
+(defn- body-of-size
+ "A request body of `n` bytes of `x`, wrapped so reads are counted in `counter`."
+ [n counter]
+ (counting-stream (ByteArrayInputStream. (.getBytes ^String (apply str (repeat n "x")) "UTF-8")) counter))
+
+(defn- ex-status [^Throwable e]
+ (:status-code (ex-data e)))
+
+(deftest bounded-input-stream-test
+ (testing "a body of exactly the maximum size reads through to EOF without complaint"
+ (let [counter (atom 0)
+ s (mw.body-limit/bounded-input-stream (body-of-size 16 counter) 16)]
+ (is (= (apply str (repeat 16 "x")) (slurp s)))
+ (is (= 16 @counter))))
+ (testing "one byte over the maximum throws a 413, and reading stops there"
+ (let [counter (atom 0)
+ s (mw.body-limit/bounded-input-stream (body-of-size 1000 counter) 16)]
+ (is (= 413 (ex-status (is (thrown? clojure.lang.ExceptionInfo (slurp s))))))
+ (is (<= @counter 17))))
+ (testing "single-byte reads are bounded too, and never pull more than one byte past the limit"
+ (let [counter (atom 0)
+ s (mw.body-limit/bounded-input-stream (body-of-size 1000 counter) 4)]
+ (dotimes [_ 5] (.read s))
+ (is (= 413 (ex-status (is (thrown? clojure.lang.ExceptionInfo (.read s))))))
+ (is (= 5 @counter))))
+ (testing "once the limit is reached, further reads keep throwing instead of returning 0 bytes or EOF"
+ (let [counter (atom 0)
+ s (mw.body-limit/bounded-input-stream (body-of-size 1000 counter) 4)]
+ (.read s (byte-array 100) 0 100)
+ (is (= 413 (ex-status (is (thrown? clojure.lang.ExceptionInfo (.read s (byte-array 100) 0 100))))))
+ (is (= 413 (ex-status (is (thrown? clojure.lang.ExceptionInfo (.read s (byte-array 100) 0 100))))))
+ (is (= 5 @counter)))))
+
+(defn- slurping-handler
+ "A handler that reads the whole body and echoes it back in a 200 response."
+ [request respond _raise]
+ (respond {:status 200, :body (some-> (:body request) slurp)}))
+
+(defn- call-limited
+ "Run `request` through [[mw.body-limit/wrap-limit-request-body]] wrapped around [[slurping-handler]], with the
+ unauthenticated limit set to `max-bytes`, returning the response. `request` is treated as authenticated when it
+ carries a `:metabase-user-id`."
+ [request max-bytes]
+ (mt/with-temp-env-var-value! [mb-max-unauthenticated-request-body-bytes (str max-bytes)]
+ ((mw.body-limit/wrap-limit-request-body slurping-handler)
+ (merge {:request-method :post, :uri "/api/card", :headers {"content-type" "application/json"}} request)
+ identity
+ (fn [e] (throw e)))))
+
+(deftest wrap-limit-request-body-test
+ (testing "a Content-Length over the limit is rejected before a single byte is read"
+ (let [counter (atom 0)
+ response (call-limited {:body (body-of-size 1000 counter), :content-length 1000} 16)]
+ (is (= 413 (:status response)))
+ (is (string? (:body response)))
+ (is (= 0 @counter))))
+ (testing "without a Content-Length (chunked), reading past the limit is rejected"
+ (let [counter (atom 0)
+ response (call-limited {:body (body-of-size 1000 counter)} 16)]
+ (is (= 413 (:status response)))
+ (is (<= @counter 17))))
+ (testing "a body within the limit reaches the handler intact"
+ (let [counter (atom 0)
+ response (call-limited {:body (body-of-size 16 counter), :content-length 16} 16)]
+ (is (= 200 (:status response)))
+ (is (= (apply str (repeat 16 "x")) (:body response)))))
+ (testing "a request with no body passes through"
+ (is (= 200 (:status (call-limited {:body nil} 16)))))
+ (testing "an unauthenticated multipart body is bounded like any other: the content type is caller-supplied, and
+ every endpoint that accepts multipart requires authentication anyway"
+ (let [counter (atom 0)
+ response (call-limited {:body (body-of-size 1000 counter)
+ :content-length 1000
+ :headers {"content-type" "multipart/form-data; boundary=xyz"}}
+ 16)]
+ (is (= 413 (:status response)))
+ (is (= 0 @counter))))
+ (testing "an authenticated multipart body is still left to the endpoint's own multipart limits"
+ (let [counter (atom 0)
+ response (call-limited {:body (body-of-size 1000 counter)
+ :content-length 1000
+ :metabase-user-id 1
+ :headers {"content-type" "multipart/form-data; boundary=xyz"}}
+ 16)]
+ (is (= 200 (:status response)))
+ (is (= 1000 @counter))))
+ (testing "other exceptions from the handler are not swallowed"
+ (mt/with-temp-env-var-value! [mb-max-unauthenticated-request-body-bytes "16"]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ ((mw.body-limit/wrap-limit-request-body (fn [_ _ _] (throw (ex-info "boom" {}))))
+ {:request-method :post, :uri "/api/card", :body (body-of-size 1 (atom 0))}
+ identity
+ (fn [e] (throw e))))))))
+
+(deftest authenticated-requests-are-not-limited-test
+ (testing "authenticated: a body over the unauthenticated limit reaches the handler intact"
+ (let [counter (atom 0)
+ response (call-limited {:body (body-of-size 1000 counter), :content-length 1000, :metabase-user-id 1} 16)]
+ (is (= 200 (:status response)))
+ (is (= (apply str (repeat 1000 "x")) (:body response)))))
+ (testing "authenticated: a chunked body over the unauthenticated limit reaches the handler intact"
+ (let [counter (atom 0)
+ response (call-limited {:body (body-of-size 1000 counter), :metabase-user-id 1} 16)]
+ (is (= 200 (:status response)))
+ (is (= 1000 @counter))))
+ (testing "authenticated: the body is handed through unwrapped"
+ (let [body (body-of-size 1 (atom 0))
+ captured (atom nil)]
+ (mt/with-temp-env-var-value! [mb-max-unauthenticated-request-body-bytes "16"]
+ ((mw.body-limit/wrap-limit-request-body (fn [request respond _] (reset! captured (:body request)) (respond {:status 200})))
+ {:request-method :post, :uri "/api/card", :body body, :metabase-user-id 1}
+ identity
+ (fn [e] (throw e))))
+ (is (identical? body @captured)))))
+
+(defn- request-through-full-stack
+ "POST `body-bytes` bytes to `uri` (`/api/card` by default) through the real middleware stack with `headers`,
+ returning `[response bytes-read]`."
+ [headers body-bytes & {:keys [declare-length? uri], :or {uri "/api/card"}}]
+ (let [counter (atom 0)
+ response (promise)]
+ ((server.test-handler/test-handler)
+ (cond-> {:request-method :post
+ :uri uri
+ :headers headers
+ :body (body-of-size body-bytes counter)}
+ declare-length? (assoc :content-length body-bytes))
+ #(deliver response %)
+ (fn [e] (deliver response e)))
+ [(deref response 10000 ::timeout) @counter]))
+
+(deftest json-body-is-bounded-before-auth-test
+ (mt/with-temp-env-var-value! [mb-max-unauthenticated-request-body-bytes "1024"]
+ (testing "an unauthenticated JSON POST larger than the unauthenticated limit is refused with 413 without buffering it"
+ (let [[response bytes-read] (request-through-full-stack {"content-type" "application/json"} 2048)]
+ (is (= 413 (:status response)))
+ (is (<= bytes-read 1025))))
+ (testing "with a declared Content-Length, nothing on the stack ahead of the limit reads a single byte"
+ (let [[response bytes-read] (request-through-full-stack {"content-type" "application/json"} 2048
+ :declare-length? true)]
+ (is (= 413 (:status response)))
+ (is (= 0 bytes-read))))
+ (testing "a form-encoded body is bounded the same way"
+ (let [[response bytes-read] (request-through-full-stack {"content-type" "application/x-www-form-urlencoded"} 2048)]
+ (is (= 413 (:status response)))
+ (is (<= bytes-read 1025))))
+ (testing "a caller-supplied multipart content type does not lift the bound"
+ (let [[response bytes-read] (request-through-full-stack
+ {"content-type" "multipart/form-data; boundary=xyz"} 2048
+ :declare-length? true)]
+ (is (= 413 (:status response)))
+ (is (= 0 bytes-read))))
+ (testing "Slack signature verification, which slurps the body before it can trust the caller, stays bounded even
+ when the caller claims to be sending multipart"
+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret "test-slack-signing-secret-12345"]
+ (let [[response bytes-read] (request-through-full-stack
+ {"content-type" "multipart/form-data; boundary=xyz"
+ "x-slack-signature" "v0=not-a-real-signature"
+ "x-slack-request-timestamp" "1700000000"}
+ 2048
+ :uri "/api/metabot/slack/events")]
+ (is (= 413 (:status response)))
+ (is (<= bytes-read 1025)))))
+ (testing "the same body with a valid session is not limited (and fails validation instead)"
+ (let [[response bytes-read] (request-through-full-stack {"content-type" "application/json"
+ "x-metabase-session" (test.users/username->token :rasta)}
+ 8192)]
+ (is (= 400 (:status response)))
+ (is (= 8192 bytes-read))))))