The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Disallow adding new classpath resources as geojson paths (#70745)
Ben Grabow8d41c7d06864
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex ac7804c5338..d5be77254a5 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -975,6 +975,7 @@:api #{metabase.geojson.apimetabase.geojson.init}:uses #{api+ configpermissionssettingsutil}}diff --git a/e2e/test/scenarios/admin-2/settings.cy.spec.js b/e2e/test/scenarios/admin-2/settings.cy.spec.jsindex d18e07c63b3..9561668f2d5 100644--- a/e2e/test/scenarios/admin-2/settings.cy.spec.js+++ b/e2e/test/scenarios/admin-2/settings.cy.spec.js@@ -959,7 +959,7 @@ describe("scenarios > admin > settings > map settings", () => {cy.findByText("Load").click();// eslint-disable-next-line metabase/no-unscoped-text-selectors -- deprecated usagecy.findByText(- "Invalid GeoJSON file location: must either start with http:// or https:// or be a relative path to a file on the classpath. " ++ "Invalid GeoJSON file location: must start with http:// or https://. " +"URLs referring to hosts that supply internal hosting metadata are prohibited.",);});diff --git a/src/metabase/geojson/api.clj b/src/metabase/geojson/api.cljindex 898ffec6216..cbbd6012e9a 100644--- a/src/metabase/geojson/api.clj+++ b/src/metabase/geojson/api.clj@@ -64,7 +64,8 @@:else (:body resp))))(defn- url->reader [url]- (if-let [resource (io/resource url)]+ (if-let [resource (and (geojson.settings/valid-geojson-resource-path? url)+ (io/resource url))](io/reader resource)(url->geojson url)))@@ -90,9 +91,9 @@_requestrespondraise]- (when-not (or (geojson.settings/custom-geojson-enabled) ((geojson.settings/builtin-geojson) (keyword k)))+ (when-not (geojson.settings/custom-geojson-enabled)(raise (ex-info (tru "Custom GeoJSON is not enabled") {:status-code 400})))- (if-let [url (get-in (geojson.settings/custom-geojson) [(keyword k) :url])]+ (if-let [url (get-in (geojson.settings/user-defined-custom-geojson) [(keyword k) :url])](try(read-url-and-respond url respond)(catch Throwable ediff --git a/src/metabase/geojson/settings.clj b/src/metabase/geojson/settings.cljindex 82d62aa45b0..abe072a658e 100644--- a/src/metabase/geojson/settings.clj+++ b/src/metabase/geojson/settings.clj@@ -1,12 +1,14 @@(ns metabase.geojson.settings(:require[clojure.java.io :as io]+ [metabase.config.core :as config][metabase.settings.core :as setting :refer [defsetting]]+ [metabase.util.http :as http][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms])(:import- (java.net InetAddress URL)))+ (java.net URI URL)))(set! *warn-on-reflection* true)@@ -52,25 +54,20 @@(def ^:private CustomGeoJSONValidator (mr/validator CustomGeoJSON))+(defn allow-classpath-geojson?+ "Whether classpath GeoJSON resources are allowed, controlled by MB_ALLOW_CLASSPATH_GEOJSON."+ []+ (config/config-bool :mb-allow-classpath-geojson))+(defn invalid-location-msg"Error message when a GeoJSON URL is invalid."[]- (str (tru "Invalid GeoJSON file location: must either start with http:// or https:// or be a relative path to a file on the classpath.")+ (str (if (allow-classpath-geojson?)+ (tru "Invalid GeoJSON file location: must either start with http:// or https:// or be a relative path to a file on the classpath.")+ (tru "Invalid GeoJSON file location: must start with http:// or https://."))" "(tru "URLs referring to hosts that supply internal hosting metadata are prohibited.")))-(def ^:private invalid-hosts- #{"metadata.google.internal"}) ; internal metadata for GCP--(defn- valid-host?- [^URL url]- (let [host (.getHost url)- host->url (fn [host] (URL. (str "http://" host)))- base-url (host->url (.getHost url))]- (and (not-any? (fn [invalid-url] (.equals ^URL base-url invalid-url))- (map host->url invalid-hosts))- (not (.isLinkLocalAddress (InetAddress/getByName host))))))-(defn- valid-protocol?[^URL url](#{"http" "https"} (.getProtocol url)))@@ -78,16 +75,22 @@(defn- valid-url?[url-string](try- (let [url (URL. url-string)]+ (let [url (.toURL (URI. url-string))](and (valid-protocol? url)- (valid-host? url)))- (catch Throwable e- (throw (ex-info (invalid-location-msg) {:status-code 400, :url url-string} e)))))+ (http/valid-host? :external-only url)))+ (catch Throwable _ false)))++(defn valid-geojson-resource-path?+ "Whether GeoJSON `url` points to a valid resource. Does not check whether the contents are valid GeoJSON or not.+ User-defined classpath resources are only allowed when MB_ALLOW_CLASSPATH_GEOJSON is true."+ [url]+ (and (allow-classpath-geojson?)+ (boolean (io/resource url))))(defn valid-geojson-url?- "Whether GeoJSON `url` points to a valid resource. Does not check whether the contents are valid GeoJSON or not."+ "Whether GeoJSON `url` points to a valid resource or "[url]- (or (io/resource url)+ (or (valid-geojson-resource-path? url)(valid-url? url)))(defn- valid-geojson-urls?@@ -134,3 +137,8 @@:visibility :public:export? true:audit :raw-value)++(defn user-defined-custom-geojson+ "Returns the subset of custom-geojson that users defined, without the built-in geojson entries."+ []+ (reduce dissoc (custom-geojson) (keys (builtin-geojson))))diff --git a/test/metabase/geojson/api_test.clj b/test/metabase/geojson/api_test.cljindex 187a6edd2cf..5994cd15a05 100644--- a/test/metabase/geojson/api_test.clj+++ b/test/metabase/geojson/api_test.clj@@ -79,21 +79,31 @@(mt/user-http-request :crowberto :put 204 "setting/custom-geojson" {:value test-custom-geojson})(mt/user-http-request :crowberto :get 200 "setting/custom-geojson"))))))(testing "passing in an invalid URL" ; see above validation test- (is (= (str "Invalid GeoJSON file location: must either start with http:// or https:// or be a relative path to a file on the classpath. "+ (is (= (str "Invalid GeoJSON file location: must start with http:// or https://. ""URLs referring to hosts that supply internal hosting metadata are prohibited.")(mt/user-http-request :crowberto :put 400 "setting/custom-geojson"{:value {:mordor (assoc (first (vals test-custom-geojson)):url "ftp://example.com")}}))))- (testing "it accepts resources"+ (testing "it rejects classpath resources when MB_ALLOW_CLASSPATH_GEOJSON is not set"(let [resource-geojson {(first (keys test-custom-geojson))(assoc (first (vals test-custom-geojson))- :url "c3p0.properties")}]- (is (= (merge (@#'geojson.settings/builtin-geojson) resource-geojson)- (u/auto-retry 3- (mt/with-temporary-setting-values [custom-geojson nil]- (mt/user-http-request :crowberto :put 204 "setting/custom-geojson"- {:value resource-geojson})- (mt/user-http-request :crowberto :get 200 "setting/custom-geojson")))))))))+ :url "geojson/custom/my-map.json")}]+ (mt/with-temporary-setting-values [custom-geojson nil]+ (is (= (str "Invalid GeoJSON file location: must start with http:// or https://. "+ "URLs referring to hosts that supply internal hosting metadata are prohibited.")+ (mt/user-http-request :crowberto :put 400 "setting/custom-geojson"+ {:value resource-geojson}))))))+ (testing "it accepts classpath resources when MB_ALLOW_CLASSPATH_GEOJSON is true"+ (mt/with-temp-env-var-value! [mb-allow-classpath-geojson "true"]+ (let [resource-geojson {(first (keys test-custom-geojson))+ (assoc (first (vals test-custom-geojson))+ :url "test.geojson")}]+ (is (= (merge (@#'geojson.settings/builtin-geojson) resource-geojson)+ (u/auto-retry 3+ (mt/with-temporary-setting-values [custom-geojson nil]+ (mt/user-http-request :crowberto :put 204 "setting/custom-geojson"+ {:value resource-geojson})+ (mt/user-http-request :crowberto :get 200 "setting/custom-geojson"))))))))))(deftest ^:parallel url-proxy-endpoint-test(with-geojson-mocks@@ -107,8 +117,8 @@(mt/user-http-request :crowberto :get 400 "geojson":url test-broken-geojson-url))))(testing "error is returned if URL is invalid"- (is (= (str "Invalid GeoJSON file location: must either start with http:// or https:// or be a relative path to "- "a file on the classpath. URLs referring to hosts that supply internal hosting metadata are "+ (is (= (str "Invalid GeoJSON file location: must start with http:// or https://. "+ "URLs referring to hosts that supply internal hosting metadata are ""prohibited.")(mt/user-http-request :crowberto :get 400 "geojson" :url "file://tmp"))))(testing "error is returned if response is not JSON"@@ -150,11 +160,12 @@;; test flakes after 45 seconds with `mt/user-http-request` times out. And presumably other clients have similar;; issues. This ensures we give a good error message in this case.(with-open [server (non-responding-server)]- (let [never-responds-url (str "http://localhost:" (-port server))]- (testing "error is returned if URL connection fails"- (is (= "GeoJSON URL failed to load"- (mt/user-http-request :crowberto :get 400 "geojson"- :url never-responds-url)))))))))+ (with-redefs [geojson.settings/valid-geojson-url? (constantly true)]+ (let [never-responds-url (str "http://localhost:" (-port server))]+ (testing "error is returned if URL connection fails"+ (is (= "GeoJSON URL failed to load"+ (mt/user-http-request :crowberto :get 400 "geojson"+ :url never-responds-url))))))))))(deftest key-proxy-endpoint-test(with-geojson-mocks@@ -201,7 +212,7 @@(.add "metabase.com"(into-array [(InetAddress/getByAddress (byte-array [1 1 1 1]))(InetAddress/getByAddress (byte-array [169 254 169 254]))])))]- (is (= (str "Invalid GeoJSON file location: must either start with http:// or https:// or be a relative path to "- "a file on the classpath. URLs referring to hosts that supply internal hosting metadata are "+ (is (= (str "Invalid GeoJSON file location: must start with http:// or https://. "+ "URLs referring to hosts that supply internal hosting metadata are ""prohibited.")(mt/user-http-request :crowberto :get 400 "geojson" :url test-geojson-url))))))diff --git a/test/metabase/geojson/settings_test.clj b/test/metabase/geojson/settings_test.cljindex c5de1aabcbb..6dd8a6681b9 100644--- a/test/metabase/geojson/settings_test.clj+++ b/test/metabase/geojson/settings_test.clj@@ -23,7 +23,7 @@(is (@#'geojson.settings/CustomGeoJSONValidator test-custom-geojson)))(deftest ^:parallel validate-geojson-test- (testing "It validates URLs and files appropriately"+ (testing "It validates URLs and files appropriately (classpath resources disabled by default)"(let [examples {;; Internal metadata for GCP"metadata.google.internal" false"https://metadata.google.internal" false@@ -54,8 +54,8 @@"http://192.0.2.0" true;; this following test flakes in CI for unknown reasons;;"http://0xc0000200" true- ;; Resources (files on classpath) are valid- "c3p0.properties" true+ ;; Classpath resources are NOT valid when env var is not set+ "test.geojson" false;; Other files are not"./README.md" false"file:///tmp" false@@ -73,6 +73,16 @@(is (valid? geojson geojson) (str url))(is (thrown? clojure.lang.ExceptionInfo (valid? geojson geojson)) (str url))))))))+(deftest classpath-geojson-env-var-test+ (testing "When MB_ALLOW_CLASSPATH_GEOJSON is true"+ (mt/with-temp-env-var-value! [mb-allow-classpath-geojson "true"]+ (testing "classpath resources are accepted"+ (let [geojson {:deadb33f {:name "Test" :url "test.geojson" :region_key nil :region_name nil}}]+ (is (#'geojson.settings/validate-geojson geojson geojson))))))+ (testing "When MB_ALLOW_CLASSPATH_GEOJSON is not set, classpath resources are rejected"+ (let [geojson {:deadb33f {:name "Test" :url "test.geojson" :region_key nil :region_name nil}}]+ (is (thrown? clojure.lang.ExceptionInfo (#'geojson.settings/validate-geojson geojson geojson))))))+(deftest custom-geojson-disallow-overriding-builtins-test(testing "We shouldn't let people override the builtin GeoJSON and put weird stuff in there; ignore changes to them"(mt/with-temporary-setting-values [custom-geojson nil]