Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

harden the custom viz dev server url handling (#80222)

Jakub Chodorowicza4fa8cf2caea
diff --git a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.clj b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.clj
index d20186d83ad..75ef00f2ac9 100644
--- a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.clj
@@ -12,6 +12,7 @@
[metabase.api.routes.common :refer [+auth]]
[metabase.events.core :as events]
[metabase.server.streaming-response :as sr]
+ [metabase.util.http :as u.http]
[metabase.util.log :as log]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2])
@@ -361,20 +362,25 @@
"X-Accel-Buffering" "no"}}
[^OutputStream os canceled-chan]
(try
- (let [resp (http/get sse-url {:as :stream
- :socket-timeout 0
- :connection-timeout 5000
- :headers {"Accept" "text/event-stream"}})]
+ ;; the SSE stream is long-lived and idle between events, so override the 5s read timeout
+ ;; from dev-http-opts (0 = no read timeout)
+ (let [resp (http/get sse-url (merge cache/dev-http-opts
+ {:as :stream
+ :socket-timeout 0
+ :headers {"Accept" "text/event-stream"}}))]
(with-open [^InputStream is (:body resp)
rdr (BufferedReader. (InputStreamReader. is "UTF-8"))]
- (loop []
- (when-not (a/poll! canceled-chan)
- (when-let [line (.readLine rdr)]
- (.write os (.getBytes (str line "\n") "UTF-8"))
- (.flush os)
- (recur))))))
+ (if-not (= "text/event-stream" (u.http/response-content-type resp))
+ (sr/write-error! os {:message "Dev server did not return an event stream"} nil 400)
+ (loop []
+ (when-not (a/poll! canceled-chan)
+ (when-let [line (.readLine rdr)]
+ (.write os (.getBytes (str line "\n") "UTF-8"))
+ (.flush os)
+ (recur)))))))
(catch Exception e
- (log/debugf "SSE proxy for plugin %d ended: %s" id (ex-message e))))))))
+ (log/debugf "SSE proxy for plugin %d ended: %s" id (ex-message e))
+ (sr/write-error! os e nil)))))))
(api.macros/defendpoint :post "/:id/refresh" :- CustomVizPluginResponse
"Re-fetch the manifest from the dev server for a dev-only plugin. For uploaded
diff --git a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.clj b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.clj
index f516778d486..46ee9188cd8 100644
--- a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.clj
+++ b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.clj
@@ -21,6 +21,7 @@
[metabase.util :as u]
[metabase.util.compress :as u.compress]
[metabase.util.files :as u.files]
+ [metabase.util.http :as u.http]
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
@@ -294,13 +295,31 @@
[]
#{"http" "https"})
+(def ^:private dev-network-policy
+ :loopback-and-private)
+
+(defn- disallowed-address-ex
+ [^String label ^String url]
+ (ex-info (str label " must resolve to a loopback or private network address.")
+ {:status-code 400 :url url}))
+
(defn- validate-url!
- "Validate that a URL uses an allowed scheme. Throws on invalid input."
+ "Validate that a URL uses an allowed scheme and does not name an address the dev fetches would refuse to
+ connect to. Throws on invalid input."
[^String url ^String label]
(let [scheme (some-> (java.net.URI. url) .getScheme u/lower-case-en)]
(when-not (contains? (allowed-schemes) scheme)
(throw (ex-info (str label " must use http or https, got: " scheme)
- {:status-code 400 :url url})))))
+ {:status-code 400 :url url})))
+ (when-not (u.http/host-allowed-for-network-policy? dev-network-policy url)
+ (throw (disallowed-address-ex label url)))))
+
+(defn- rethrow-if-refused!
+ "Re-throw `e` as a 400 if it is the network policy refusing to connect, otherwise do nothing. Without this
+ the surrounding catch would report a refused address as a dev server that happens to be down."
+ [^Exception e ^String url]
+ (when (:ssrf (ex-data e))
+ (throw (disallowed-address-ex "Dev bundle URL" url))))
(defn dev-base-url
"Validate and normalize the dev base URL. Ensures http/https scheme and trailing slash."
@@ -313,46 +332,86 @@
^String [^String base-url ^String relative-path]
(str (dev-base-url base-url) relative-path))
-(def ^:private http-opts
- {:socket-timeout 5000 :connection-timeout 5000})
+(def dev-http-opts
+ "clj-http options for every fetch of a caller-supplied dev bundle URL, including the SSE proxy in the api ns."
+ {:socket-timeout 5000
+ :connection-timeout 5000
+ :redirect-strategy :none
+ :dns-resolver (u.http/network-policy-dns-resolver dev-network-policy)})
+
+;; Content-type
+;; The address policy has to keep allowing loopback and private addresses, so a dev URL can still be pointed
+;; at an internal service. Requiring the content-type specific content-type narrows the possibilities
+
+(def ^:private bundle-content-types
+ #{"application/javascript" "text/javascript"})
+
+(def ^:private manifest-content-types
+ #{"application/json" "text/json"})
+
+(defn- check-content-type!
+ [resp allowed ^String url]
+ (let [ctype (u.http/response-content-type resp)]
+ (when-not (contains? allowed ctype)
+ (throw (ex-info (str "Dev bundle URL returned " (or ctype "no content type")
+ ", expected " (str/join " or " (sort allowed)))
+ {:status-code 400 :url url})))))
(defn fetch-dev-bundle
"Fetch a JS bundle from a dev base URL.
- Returns {:content str :hash str}, or nil when the dev server is transiently
- unavailable (e.g. mid-rebuild)."
+ Returns {:content str :hash str}, or nil when the dev server is unreachable or
+ responds with an error status (e.g. mid-rebuild). Throws a 400 when the URL is
+ disallowed or the response has an unexpected content type."
[^String base-url]
- (try
- (let [content (:body (http/get (dev-url base-url bundle-rel-path)
- (assoc http-opts :as :string)))]
- {:content content
- :hash (string-hash content)})
- (catch Exception e
- (log/debugf "Failed to fetch dev bundle from %s: %s" base-url (ex-message e))
- nil)))
+ ;; build (and thereby validate) the URL outside the try -- validation failures must surface as the 400s
+ ;; they are, not be swallowed by the catch as a dev server that happens to be down
+ (let [url (dev-url base-url bundle-rel-path)]
+ (when-let [resp (try
+ (http/get url (assoc dev-http-opts :as :string))
+ (catch Exception e
+ (rethrow-if-refused! e base-url)
+ (log/debugf "Failed to fetch dev bundle from %s: %s" base-url (ex-message e))
+ nil))]
+ (check-content-type! resp bundle-content-types base-url)
+ (let [content (:body resp)]
+ {:content content
+ :hash (string-hash content)}))))
(defn fetch-dev-manifest
"Fetch and parse the manifest from a dev base URL.
- Returns the parsed manifest map or nil on failure. Throws ex-info with
- `:status-code 400` when the manifest is structurally invalid."
+ Returns the parsed manifest map, or nil when the dev server is unreachable or
+ responds with an error status. Throws ex-info with `:status-code 400` when the
+ URL is disallowed, the response has an unexpected content type, or the
+ manifest is structurally invalid."
[^String base-url]
- (when-let [parsed (try
- (let [content (:body (http/get (dev-url base-url (manifest/manifest-path))
- (assoc http-opts :as :string)))]
- (manifest/parse-manifest content))
- (catch Exception e
- (log/debugf "No manifest at %s: %s" base-url (ex-message e))
- nil))]
- (when-let [error (manifest/validation-error parsed)]
- (throw (ex-info (format "%s is invalid: %s" (manifest/manifest-path) (pr-str error))
- {:status-code 400})))
- parsed))
+ ;; as in [[fetch-dev-bundle]], validate the URL outside the try
+ (let [url (dev-url base-url (manifest/manifest-path))]
+ (when-let [parsed (when-let [resp (try
+ (http/get url (assoc dev-http-opts :as :string))
+ (catch Exception e
+ (rethrow-if-refused! e base-url)
+ (log/debugf "No manifest at %s: %s" base-url (ex-message e))
+ nil))]
+ (check-content-type! resp manifest-content-types base-url)
+ (manifest/parse-manifest (:body resp)))]
+ (when-let [error (manifest/validation-error parsed)]
+ (throw (ex-info (format "%s is invalid: %s" (manifest/manifest-path) (pr-str error))
+ {:status-code 400})))
+ parsed)))
(defn fetch-dev-asset
"Fetch a static asset from a dev base URL.
- Returns the bytes or nil on failure."
+ Returns the bytes. Throws on fetch failure; disallowed addresses and
+ unexpected content types surface as 400s."
^bytes [^String base-url ^String asset-name]
- (:body (http/get (dev-url base-url (asset-rel-path asset-name))
- (assoc http-opts :as :byte-array))))
+ (let [url (dev-url base-url (asset-rel-path asset-name))
+ resp (try
+ (http/get url (assoc dev-http-opts :as :byte-array))
+ (catch Exception e
+ (rethrow-if-refused! e base-url)
+ (throw e)))]
+ (check-content-type! resp (some-> (manifest/asset-content-type asset-name) hash-set) base-url)
+ (:body resp)))
(defn set-or-clear-dev-bundle!
"Set or clear the dev base URL for a plugin. Persists to the database."
diff --git a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.clj b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.clj
index 1c47ffab8b2..34ed8eb40f6 100644
--- a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.clj
@@ -789,14 +789,25 @@
(testing "GET /:id/dev-sse forwards Server-Sent Events from the dev server"
(mt/with-dynamic-fn-redefs [cache/resolve-dev-bundle (constantly "http://localhost:5199")
http/get (fn [_url _opts]
- {:body (java.io.ByteArrayInputStream.
- (.getBytes "data: hello\n\n" "UTF-8"))})]
+ {:headers {:content-type "text/event-stream"}
+ :body (java.io.ByteArrayInputStream.
+ (.getBytes "data: hello\n\n" "UTF-8"))})]
(let [resp (mt/user-http-request-full-response
:crowberto :get 200 (str "ee/custom-viz-plugin/" id "/dev-sse"))]
(testing "response is served as an event stream"
(is (str/starts-with? (get-in resp [:headers "Content-Type"]) "text/event-stream")))
(testing "the dev server's event is forwarded to the browser"
- (is (str/includes? (str (:body resp)) "data: hello"))))))))))
+ (is (str/includes? (str (:body resp)) "data: hello"))))))
+ (testing "GET /:id/dev-sse refuses an upstream response that is not an event stream"
+ (mt/with-dynamic-fn-redefs [cache/resolve-dev-bundle (constantly "http://localhost:5199")
+ http/get (fn [_url _opts]
+ {:headers {:content-type "text/html"}
+ :body (java.io.ByteArrayInputStream.
+ (.getBytes "<html>internal secrets</html>" "UTF-8"))})]
+ (let [resp (mt/user-http-request-full-response
+ :crowberto :get 400 (str "ee/custom-viz-plugin/" id "/dev-sse"))]
+ (testing "the internal service's body is never relayed to the browser"
+ (is (not (str/includes? (str (:body resp)) "internal secrets")))))))))))
(deftest list-excludes-dev-plugins-when-dev-mode-disabled-test
(mt/with-premium-features #{:custom-viz}
diff --git a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.clj b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.clj
index 4a64e212abe..948bf9bc3c1 100644
--- a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.clj
@@ -123,8 +123,14 @@
(testing "accepts http URLs"
(is (= "http://localhost:5174/" (cache/dev-base-url "http://localhost:5174")))
(is (= "http://localhost:5174/" (cache/dev-base-url "http://localhost:5174/"))))
- (testing "accepts https URLs"
- (is (= "https://dev.example.com/" (cache/dev-base-url "https://dev.example.com"))))
+ (testing "accepts https URLs to private addresses"
+ (is (= "https://192.168.1.50/" (cache/dev-base-url "https://192.168.1.50"))))
+ (testing "SECURITY: rejects URLs that resolve to public addresses"
+ (is (thrown-with-msg? Exception #"loopback or private"
+ (cache/dev-base-url "https://8.8.8.8"))))
+ (testing "SECURITY: rejects link-local address"
+ (is (thrown-with-msg? Exception #"loopback or private"
+ (cache/dev-base-url "http://169.254.169.254/latest"))))
(testing "SECURITY: rejects ftp:// URLs"
(is (thrown-with-msg? Exception #"http or https"
(cache/dev-base-url "ftp://evil.com/bundle"))))
@@ -140,17 +146,50 @@
(deftest fetch-dev-manifest-test
(testing "returns a well-formed manifest"
- (with-redefs [http/get (constantly {:body (json/encode {:name "dev-viz"})})]
+ (with-redefs [http/get (constantly {:headers {:content-type "application/json"}
+ :body (json/encode {:name "dev-viz"})})]
(is (= {:name "dev-viz"}
(cache/fetch-dev-manifest "http://localhost:5174")))))
(testing "returns nil when the manifest cannot be fetched"
(with-redefs [http/get (fn [& _] (throw (Exception. "connection refused")))]
(is (nil? (cache/fetch-dev-manifest "http://localhost:5174")))))
(testing "rejects a structurally invalid manifest, same as the upload path"
- (with-redefs [http/get (constantly {:body (json/encode {:name 123})})]
+ (with-redefs [http/get (constantly {:headers {:content-type "application/json"}
+ :body (json/encode {:name 123})})]
(is (thrown-with-msg? Exception #"is invalid"
(cache/fetch-dev-manifest "http://localhost:5174"))))))
+(deftest dev-fetch-surfaces-validation-errors-test
+ (testing "a URL that fails validation surfaces as a 400, not as a silently-down dev server --
+ DNS can change between saving the URL and fetching from it"
+ (with-redefs [http/get (fn [& _] (throw (Exception. "should not be reached")))]
+ (doseq [[url pattern] [["https://8.8.8.8" #"loopback or private"]
+ ["file:///etc/passwd" #"http or https"]]]
+ (let [e (is (thrown-with-msg? Exception pattern (cache/fetch-dev-bundle url)))]
+ (is (= 400 (:status-code (ex-data e)))))
+ (let [e (is (thrown-with-msg? Exception pattern (cache/fetch-dev-manifest url)))]
+ (is (= 400 (:status-code (ex-data e)))))
+ (let [e (is (thrown-with-msg? Exception pattern (cache/fetch-dev-asset url "icon.svg")))]
+ (is (= 400 (:status-code (ex-data e)))))))))
+
+(deftest dev-fetch-requires-the-dev-servers-content-type-test
+ (testing "a response that isn't what the dev server would have served is refused, not parsed --
+ a dev URL may still be pointed at an internal service, and this bounds what it can hand back"
+ (doseq [[what ctype body] [["an internal admin page" "text/html" "<html>secrets</html>"]
+ ["a response with no content type" nil "{\"name\":\"x\"}"]]]
+ (testing what
+ (with-redefs [http/get (constantly {:headers (when ctype {:content-type ctype})
+ :body body})]
+ (is (thrown-with-msg? Exception #"Dev bundle URL returned"
+ (cache/fetch-dev-manifest "http://localhost:5174")))
+ (is (thrown-with-msg? Exception #"Dev bundle URL returned"
+ (cache/fetch-dev-bundle "http://localhost:5174")))))))
+ (testing "the content types the CLI's own dev server serves are accepted"
+ (with-redefs [http/get (constantly {:headers {:content-type "application/javascript; charset=utf-8"}
+ :body "export default {}"})]
+ (is (= "export default {}" (:content (cache/fetch-dev-bundle "http://localhost:5174")))
+ "a charset parameter on the header is stripped before comparison"))))
+
(deftest set-or-clear-dev-bundle!-test
(mt/with-premium-features #{:custom-viz}
(mt/with-temp [:model/CustomVizPlugin {id :id} {:identifier "test-viz"
diff --git a/src/metabase/util/http.clj b/src/metabase/util/http.clj
index c66e0abab61..c1721abe0cb 100644
--- a/src/metabase/util/http.clj
+++ b/src/metabase/util/http.clj
@@ -86,14 +86,16 @@
(defn address-allowed-for-network-policy?
"Whether `addr` is allowed by `policy`.
- `:external-only` allows only globally routable public addresses. `:allow-private` additionally allows private,
- unique-local, and carrier-grade NAT addresses, but still rejects loopback, link-local, any-local, multicast, and
- reserved addresses. `:allow-all` imposes no address restriction."
+ `:external-only` allows only globally routable public addresses.
+ `:allow-private` adds private, unique-local and carrier-grade NAT addresses.
+ `:loopback-and-private` allows *only* loopback plus those same private ranges
+ `:allow-all` imposes no address restriction."
[policy ^InetAddress addr]
(case policy
- :external-only (public-address? addr)
- :allow-private (or (public-address? addr) (private-address? addr))
- :allow-all true
+ :external-only (public-address? addr)
+ :allow-private (or (public-address? addr) (private-address? addr))
+ :loopback-and-private (or (private-address? addr) (.isLoopbackAddress addr))
+ :allow-all true
(throw (ex-info (str "Unknown network policy: " (pr-str policy)) {:policy policy}))))
;; one or more scheme segments, so nested schemes (`jdbc:postgresql://...`) are stripped too
@@ -208,6 +210,12 @@
(> (+ total n) max) nil
:else (do (.write out buf 0 n) (recur (+ total n))))))))
+(defn response-content-type
+ "Return the lower-case media type from a clj-http response, without parameters."
+ [resp]
+ (some-> (get-in resp [:headers :content-type])
+ (str/split #";") first str/trim lower-case-en))
+
(defn fetch-bytes
"SSRF-hardened GET of `url`. Returns `{:bytes <byte[]> :content-type <lower-cased string>}` on a
200 response whose (parameter-stripped, lower-cased) content-type is allowed and whose body is
@@ -233,8 +241,7 @@
:throw-exceptions false
:headers {"User-Agent" user-agent}
:dns-resolver ssrf-safe-dns-resolver})
- ctype (some-> (get-in resp [:headers :content-type])
- (str/split #";") first str/trim lower-case-en)
+ ctype (response-content-type resp)
^InputStream body (:body resp)]
(try
(when (and (= 200 (:status resp))
diff --git a/test/metabase/util/http_test.clj b/test/metabase/util/http_test.clj
index af4b2904dd2..9f08f115c4c 100644
--- a/test/metabase/util/http_test.clj
+++ b/test/metabase/util/http_test.clj
@@ -187,6 +187,15 @@
(testing "allow-private still refuses loopback, link-local, any-local and multicast"
(doseq [ip ["127.0.0.1" "::1" "169.254.169.254" "fe80::1" "0.0.0.0" "224.0.0.1" "ff02::1"]]
(is (false? (http/address-allowed-for-network-policy? :allow-private (InetAddress/getByName ip))) ip)))
+ (testing "loopback-and-private admits loopback and the private ranges"
+ (doseq [ip ["127.0.0.1" "127.0.1.5" "::1" "10.1.2.3" "192.168.0.1" "100.64.0.1" "fc00::1"]]
+ (is (true? (http/address-allowed-for-network-policy? :loopback-and-private (InetAddress/getByName ip))) ip)))
+ (testing "loopback-and-private refuses public addresses"
+ (doseq [ip public-ips]
+ (is (false? (http/address-allowed-for-network-policy? :loopback-and-private (InetAddress/getByName ip))) ip)))
+ (testing "loopback-and-private still refuses link-local"
+ (doseq [ip ["169.254.169.254" "fe80::1" "0.0.0.0" "224.0.0.1" "ff02::1"]]
+ (is (false? (http/address-allowed-for-network-policy? :loopback-and-private (InetAddress/getByName ip))) ip)))
(testing "allow-all admits everything"
(doseq [ip (concat public-ips non-public-ips)]
(is (true? (http/address-allowed-for-network-policy? :allow-all (InetAddress/getByName ip))) ip)))