The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
harden the custom viz dev server url handling (#80222)
Jakub Chodorowicza4fa8cf2caea
diff --git a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.clj b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.cljindex d20186d83ad..75ef00f2ac9 100644--- a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.clj+++ b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/api.clj@@ -12,6 +12,7 @@[metabase.api.routes.common :refer [+auth]][metabase.events.core :as events][metabase.server.streaming-response :as sr]+ [metabase.util.http :as u.http][metabase.util.log :as log][metabase.util.malli.schema :as ms][toucan2.core :as t2])@@ -361,20 +362,25 @@"X-Accel-Buffering" "no"}}[^OutputStream os canceled-chan](try- (let [resp (http/get sse-url {:as :stream- :socket-timeout 0- :connection-timeout 5000- :headers {"Accept" "text/event-stream"}})]+ ;; the SSE stream is long-lived and idle between events, so override the 5s read timeout+ ;; from dev-http-opts (0 = no read timeout)+ (let [resp (http/get sse-url (merge cache/dev-http-opts+ {:as :stream+ :socket-timeout 0+ :headers {"Accept" "text/event-stream"}}))](with-open [^InputStream is (:body resp)rdr (BufferedReader. (InputStreamReader. is "UTF-8"))]- (loop []- (when-not (a/poll! canceled-chan)- (when-let [line (.readLine rdr)]- (.write os (.getBytes (str line "\n") "UTF-8"))- (.flush os)- (recur))))))+ (if-not (= "text/event-stream" (u.http/response-content-type resp))+ (sr/write-error! os {:message "Dev server did not return an event stream"} nil 400)+ (loop []+ (when-not (a/poll! canceled-chan)+ (when-let [line (.readLine rdr)]+ (.write os (.getBytes (str line "\n") "UTF-8"))+ (.flush os)+ (recur)))))))(catch Exception e- (log/debugf "SSE proxy for plugin %d ended: %s" id (ex-message e))))))))+ (log/debugf "SSE proxy for plugin %d ended: %s" id (ex-message e))+ (sr/write-error! os e nil)))))))(api.macros/defendpoint :post "/:id/refresh" :- CustomVizPluginResponse"Re-fetch the manifest from the dev server for a dev-only plugin. For uploadeddiff --git a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.clj b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.cljindex f516778d486..46ee9188cd8 100644--- a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.clj+++ b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/cache.clj@@ -21,6 +21,7 @@[metabase.util :as u][metabase.util.compress :as u.compress][metabase.util.files :as u.files]+ [metabase.util.http :as u.http][metabase.util.log :as log][toucan2.core :as t2])(:import@@ -294,13 +295,31 @@[]#{"http" "https"})+(def ^:private dev-network-policy+ :loopback-and-private)++(defn- disallowed-address-ex+ [^String label ^String url]+ (ex-info (str label " must resolve to a loopback or private network address.")+ {:status-code 400 :url url}))+(defn- validate-url!- "Validate that a URL uses an allowed scheme. Throws on invalid input."+ "Validate that a URL uses an allowed scheme and does not name an address the dev fetches would refuse to+ connect to. Throws on invalid input."[^String url ^String label](let [scheme (some-> (java.net.URI. url) .getScheme u/lower-case-en)](when-not (contains? (allowed-schemes) scheme)(throw (ex-info (str label " must use http or https, got: " scheme)- {:status-code 400 :url url})))))+ {:status-code 400 :url url})))+ (when-not (u.http/host-allowed-for-network-policy? dev-network-policy url)+ (throw (disallowed-address-ex label url)))))++(defn- rethrow-if-refused!+ "Re-throw `e` as a 400 if it is the network policy refusing to connect, otherwise do nothing. Without this+ the surrounding catch would report a refused address as a dev server that happens to be down."+ [^Exception e ^String url]+ (when (:ssrf (ex-data e))+ (throw (disallowed-address-ex "Dev bundle URL" url))))(defn dev-base-url"Validate and normalize the dev base URL. Ensures http/https scheme and trailing slash."@@ -313,46 +332,86 @@^String [^String base-url ^String relative-path](str (dev-base-url base-url) relative-path))-(def ^:private http-opts- {:socket-timeout 5000 :connection-timeout 5000})+(def dev-http-opts+ "clj-http options for every fetch of a caller-supplied dev bundle URL, including the SSE proxy in the api ns."+ {:socket-timeout 5000+ :connection-timeout 5000+ :redirect-strategy :none+ :dns-resolver (u.http/network-policy-dns-resolver dev-network-policy)})++;; Content-type+;; The address policy has to keep allowing loopback and private addresses, so a dev URL can still be pointed+;; at an internal service. Requiring the content-type specific content-type narrows the possibilities++(def ^:private bundle-content-types+ #{"application/javascript" "text/javascript"})++(def ^:private manifest-content-types+ #{"application/json" "text/json"})++(defn- check-content-type!+ [resp allowed ^String url]+ (let [ctype (u.http/response-content-type resp)]+ (when-not (contains? allowed ctype)+ (throw (ex-info (str "Dev bundle URL returned " (or ctype "no content type")+ ", expected " (str/join " or " (sort allowed)))+ {:status-code 400 :url url})))))(defn fetch-dev-bundle"Fetch a JS bundle from a dev base URL.- Returns {:content str :hash str}, or nil when the dev server is transiently- unavailable (e.g. mid-rebuild)."+ Returns {:content str :hash str}, or nil when the dev server is unreachable or+ responds with an error status (e.g. mid-rebuild). Throws a 400 when the URL is+ disallowed or the response has an unexpected content type."[^String base-url]- (try- (let [content (:body (http/get (dev-url base-url bundle-rel-path)- (assoc http-opts :as :string)))]- {:content content- :hash (string-hash content)})- (catch Exception e- (log/debugf "Failed to fetch dev bundle from %s: %s" base-url (ex-message e))- nil)))+ ;; build (and thereby validate) the URL outside the try -- validation failures must surface as the 400s+ ;; they are, not be swallowed by the catch as a dev server that happens to be down+ (let [url (dev-url base-url bundle-rel-path)]+ (when-let [resp (try+ (http/get url (assoc dev-http-opts :as :string))+ (catch Exception e+ (rethrow-if-refused! e base-url)+ (log/debugf "Failed to fetch dev bundle from %s: %s" base-url (ex-message e))+ nil))]+ (check-content-type! resp bundle-content-types base-url)+ (let [content (:body resp)]+ {:content content+ :hash (string-hash content)}))))(defn fetch-dev-manifest"Fetch and parse the manifest from a dev base URL.- Returns the parsed manifest map or nil on failure. Throws ex-info with- `:status-code 400` when the manifest is structurally invalid."+ Returns the parsed manifest map, or nil when the dev server is unreachable or+ responds with an error status. Throws ex-info with `:status-code 400` when the+ URL is disallowed, the response has an unexpected content type, or the+ manifest is structurally invalid."[^String base-url]- (when-let [parsed (try- (let [content (:body (http/get (dev-url base-url (manifest/manifest-path))- (assoc http-opts :as :string)))]- (manifest/parse-manifest content))- (catch Exception e- (log/debugf "No manifest at %s: %s" base-url (ex-message e))- nil))]- (when-let [error (manifest/validation-error parsed)]- (throw (ex-info (format "%s is invalid: %s" (manifest/manifest-path) (pr-str error))- {:status-code 400})))- parsed))+ ;; as in [[fetch-dev-bundle]], validate the URL outside the try+ (let [url (dev-url base-url (manifest/manifest-path))]+ (when-let [parsed (when-let [resp (try+ (http/get url (assoc dev-http-opts :as :string))+ (catch Exception e+ (rethrow-if-refused! e base-url)+ (log/debugf "No manifest at %s: %s" base-url (ex-message e))+ nil))]+ (check-content-type! resp manifest-content-types base-url)+ (manifest/parse-manifest (:body resp)))]+ (when-let [error (manifest/validation-error parsed)]+ (throw (ex-info (format "%s is invalid: %s" (manifest/manifest-path) (pr-str error))+ {:status-code 400})))+ parsed)))(defn fetch-dev-asset"Fetch a static asset from a dev base URL.- Returns the bytes or nil on failure."+ Returns the bytes. Throws on fetch failure; disallowed addresses and+ unexpected content types surface as 400s."^bytes [^String base-url ^String asset-name]- (:body (http/get (dev-url base-url (asset-rel-path asset-name))- (assoc http-opts :as :byte-array))))+ (let [url (dev-url base-url (asset-rel-path asset-name))+ resp (try+ (http/get url (assoc dev-http-opts :as :byte-array))+ (catch Exception e+ (rethrow-if-refused! e base-url)+ (throw e)))]+ (check-content-type! resp (some-> (manifest/asset-content-type asset-name) hash-set) base-url)+ (:body resp)))(defn set-or-clear-dev-bundle!"Set or clear the dev base URL for a plugin. Persists to the database."diff --git a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.clj b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.cljindex 1c47ffab8b2..34ed8eb40f6 100644--- a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/api_test.clj@@ -789,14 +789,25 @@(testing "GET /:id/dev-sse forwards Server-Sent Events from the dev server"(mt/with-dynamic-fn-redefs [cache/resolve-dev-bundle (constantly "http://localhost:5199")http/get (fn [_url _opts]- {:body (java.io.ByteArrayInputStream.- (.getBytes "data: hello\n\n" "UTF-8"))})]+ {:headers {:content-type "text/event-stream"}+ :body (java.io.ByteArrayInputStream.+ (.getBytes "data: hello\n\n" "UTF-8"))})](let [resp (mt/user-http-request-full-response:crowberto :get 200 (str "ee/custom-viz-plugin/" id "/dev-sse"))](testing "response is served as an event stream"(is (str/starts-with? (get-in resp [:headers "Content-Type"]) "text/event-stream")))(testing "the dev server's event is forwarded to the browser"- (is (str/includes? (str (:body resp)) "data: hello"))))))))))+ (is (str/includes? (str (:body resp)) "data: hello"))))))+ (testing "GET /:id/dev-sse refuses an upstream response that is not an event stream"+ (mt/with-dynamic-fn-redefs [cache/resolve-dev-bundle (constantly "http://localhost:5199")+ http/get (fn [_url _opts]+ {:headers {:content-type "text/html"}+ :body (java.io.ByteArrayInputStream.+ (.getBytes "<html>internal secrets</html>" "UTF-8"))})]+ (let [resp (mt/user-http-request-full-response+ :crowberto :get 400 (str "ee/custom-viz-plugin/" id "/dev-sse"))]+ (testing "the internal service's body is never relayed to the browser"+ (is (not (str/includes? (str (:body resp)) "internal secrets")))))))))))(deftest list-excludes-dev-plugins-when-dev-mode-disabled-test(mt/with-premium-features #{:custom-viz}diff --git a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.clj b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.cljindex 4a64e212abe..948bf9bc3c1 100644--- a/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.clj+++ b/enterprise/backend/test/metabase_enterprise/custom_viz_plugin/cache_test.clj@@ -123,8 +123,14 @@(testing "accepts http URLs"(is (= "http://localhost:5174/" (cache/dev-base-url "http://localhost:5174")))(is (= "http://localhost:5174/" (cache/dev-base-url "http://localhost:5174/"))))- (testing "accepts https URLs"- (is (= "https://dev.example.com/" (cache/dev-base-url "https://dev.example.com"))))+ (testing "accepts https URLs to private addresses"+ (is (= "https://192.168.1.50/" (cache/dev-base-url "https://192.168.1.50"))))+ (testing "SECURITY: rejects URLs that resolve to public addresses"+ (is (thrown-with-msg? Exception #"loopback or private"+ (cache/dev-base-url "https://8.8.8.8"))))+ (testing "SECURITY: rejects link-local address"+ (is (thrown-with-msg? Exception #"loopback or private"+ (cache/dev-base-url "http://169.254.169.254/latest"))))(testing "SECURITY: rejects ftp:// URLs"(is (thrown-with-msg? Exception #"http or https"(cache/dev-base-url "ftp://evil.com/bundle"))))@@ -140,17 +146,50 @@(deftest fetch-dev-manifest-test(testing "returns a well-formed manifest"- (with-redefs [http/get (constantly {:body (json/encode {:name "dev-viz"})})]+ (with-redefs [http/get (constantly {:headers {:content-type "application/json"}+ :body (json/encode {:name "dev-viz"})})](is (= {:name "dev-viz"}(cache/fetch-dev-manifest "http://localhost:5174")))))(testing "returns nil when the manifest cannot be fetched"(with-redefs [http/get (fn [& _] (throw (Exception. "connection refused")))](is (nil? (cache/fetch-dev-manifest "http://localhost:5174")))))(testing "rejects a structurally invalid manifest, same as the upload path"- (with-redefs [http/get (constantly {:body (json/encode {:name 123})})]+ (with-redefs [http/get (constantly {:headers {:content-type "application/json"}+ :body (json/encode {:name 123})})](is (thrown-with-msg? Exception #"is invalid"(cache/fetch-dev-manifest "http://localhost:5174"))))))+(deftest dev-fetch-surfaces-validation-errors-test+ (testing "a URL that fails validation surfaces as a 400, not as a silently-down dev server --+ DNS can change between saving the URL and fetching from it"+ (with-redefs [http/get (fn [& _] (throw (Exception. "should not be reached")))]+ (doseq [[url pattern] [["https://8.8.8.8" #"loopback or private"]+ ["file:///etc/passwd" #"http or https"]]]+ (let [e (is (thrown-with-msg? Exception pattern (cache/fetch-dev-bundle url)))]+ (is (= 400 (:status-code (ex-data e)))))+ (let [e (is (thrown-with-msg? Exception pattern (cache/fetch-dev-manifest url)))]+ (is (= 400 (:status-code (ex-data e)))))+ (let [e (is (thrown-with-msg? Exception pattern (cache/fetch-dev-asset url "icon.svg")))]+ (is (= 400 (:status-code (ex-data e)))))))))++(deftest dev-fetch-requires-the-dev-servers-content-type-test+ (testing "a response that isn't what the dev server would have served is refused, not parsed --+ a dev URL may still be pointed at an internal service, and this bounds what it can hand back"+ (doseq [[what ctype body] [["an internal admin page" "text/html" "<html>secrets</html>"]+ ["a response with no content type" nil "{\"name\":\"x\"}"]]]+ (testing what+ (with-redefs [http/get (constantly {:headers (when ctype {:content-type ctype})+ :body body})]+ (is (thrown-with-msg? Exception #"Dev bundle URL returned"+ (cache/fetch-dev-manifest "http://localhost:5174")))+ (is (thrown-with-msg? Exception #"Dev bundle URL returned"+ (cache/fetch-dev-bundle "http://localhost:5174")))))))+ (testing "the content types the CLI's own dev server serves are accepted"+ (with-redefs [http/get (constantly {:headers {:content-type "application/javascript; charset=utf-8"}+ :body "export default {}"})]+ (is (= "export default {}" (:content (cache/fetch-dev-bundle "http://localhost:5174")))+ "a charset parameter on the header is stripped before comparison"))))+(deftest set-or-clear-dev-bundle!-test(mt/with-premium-features #{:custom-viz}(mt/with-temp [:model/CustomVizPlugin {id :id} {:identifier "test-viz"diff --git a/src/metabase/util/http.clj b/src/metabase/util/http.cljindex c66e0abab61..c1721abe0cb 100644--- a/src/metabase/util/http.clj+++ b/src/metabase/util/http.clj@@ -86,14 +86,16 @@(defn address-allowed-for-network-policy?"Whether `addr` is allowed by `policy`.- `:external-only` allows only globally routable public addresses. `:allow-private` additionally allows private,- unique-local, and carrier-grade NAT addresses, but still rejects loopback, link-local, any-local, multicast, and- reserved addresses. `:allow-all` imposes no address restriction."+ `:external-only` allows only globally routable public addresses.+ `:allow-private` adds private, unique-local and carrier-grade NAT addresses.+ `:loopback-and-private` allows *only* loopback plus those same private ranges+ `:allow-all` imposes no address restriction."[policy ^InetAddress addr](case policy- :external-only (public-address? addr)- :allow-private (or (public-address? addr) (private-address? addr))- :allow-all true+ :external-only (public-address? addr)+ :allow-private (or (public-address? addr) (private-address? addr))+ :loopback-and-private (or (private-address? addr) (.isLoopbackAddress addr))+ :allow-all true(throw (ex-info (str "Unknown network policy: " (pr-str policy)) {:policy policy}))));; one or more scheme segments, so nested schemes (`jdbc:postgresql://...`) are stripped too@@ -208,6 +210,12 @@(> (+ total n) max) nil:else (do (.write out buf 0 n) (recur (+ total n))))))))+(defn response-content-type+ "Return the lower-case media type from a clj-http response, without parameters."+ [resp]+ (some-> (get-in resp [:headers :content-type])+ (str/split #";") first str/trim lower-case-en))+(defn fetch-bytes"SSRF-hardened GET of `url`. Returns `{:bytes <byte[]> :content-type <lower-cased string>}` on a200 response whose (parameter-stripped, lower-cased) content-type is allowed and whose body is@@ -233,8 +241,7 @@:throw-exceptions false:headers {"User-Agent" user-agent}:dns-resolver ssrf-safe-dns-resolver})- ctype (some-> (get-in resp [:headers :content-type])- (str/split #";") first str/trim lower-case-en)+ ctype (response-content-type resp)^InputStream body (:body resp)](try(when (and (= 200 (:status resp))diff --git a/test/metabase/util/http_test.clj b/test/metabase/util/http_test.cljindex af4b2904dd2..9f08f115c4c 100644--- a/test/metabase/util/http_test.clj+++ b/test/metabase/util/http_test.clj@@ -187,6 +187,15 @@(testing "allow-private still refuses loopback, link-local, any-local and multicast"(doseq [ip ["127.0.0.1" "::1" "169.254.169.254" "fe80::1" "0.0.0.0" "224.0.0.1" "ff02::1"]](is (false? (http/address-allowed-for-network-policy? :allow-private (InetAddress/getByName ip))) ip)))+ (testing "loopback-and-private admits loopback and the private ranges"+ (doseq [ip ["127.0.0.1" "127.0.1.5" "::1" "10.1.2.3" "192.168.0.1" "100.64.0.1" "fc00::1"]]+ (is (true? (http/address-allowed-for-network-policy? :loopback-and-private (InetAddress/getByName ip))) ip)))+ (testing "loopback-and-private refuses public addresses"+ (doseq [ip public-ips]+ (is (false? (http/address-allowed-for-network-policy? :loopback-and-private (InetAddress/getByName ip))) ip)))+ (testing "loopback-and-private still refuses link-local"+ (doseq [ip ["169.254.169.254" "fe80::1" "0.0.0.0" "224.0.0.1" "ff02::1"]]+ (is (false? (http/address-allowed-for-network-policy? :loopback-and-private (InetAddress/getByName ip))) ip)))(testing "allow-all admits everything"(doseq [ip (concat public-ips non-public-ips)](is (true? (http/address-allowed-for-network-policy? :allow-all (InetAddress/getByName ip))) ip)))