The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Validate MB_PGVECTOR_DB_URL more strictly (#85900)
Chris Trutere715f63f070d
diff --git a/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.cljindex 478cb0a41ba..825e26e3ab7 100644--- a/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.clj+++ b/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.clj@@ -129,21 +129,46 @@[(str scheme more) credentials])[base {}]))+(def ^:private credential-params+ "The URL params that carry credentials. They go to pgjdbc as connection properties, not on the URL."+ #{"user" "password" "sslpassword"})+(defn- check-no-stray-at!"Throw if an `@` is left in `url` or `pairs` once the credentials are off, without quoting either."[^String url pairs];; An `@` left on `url` means a password with an unencoded `/`, or an `@` that pgjdbc requires encoded anyway.- ;; An unencoded `?` in a password ends the host part early, before the `/` pgjdbc requires after a host, and- ;; moves the `@` into `pairs`.- ;; Any other `@` in `pairs` is a plain value that pgjdbc accepts, such as Azure's `user=name@server`.+ ;; An unencoded `/` or `?` in a password ends the host part early and moves the `@` into `pairs`.+ ;; A credential value may hold a plain `@`, such as Azure's `user=name@server`; any other `@` must be encoded.+ ;; A URL with hosts but no path can't have its query `@` checked that way: the `?` that ended its host part+ ;; may belong to the credentials, so every `@` there must be encoded.;; Refuse a stray `@` before an error or the URL can quote the part of the password it may hold.(when (or (str/includes? url "@")- (and (re-matches #"[^/]*//[^/]+" url)- (some (fn [[k v]] (str/includes? (str k v) "@")) pairs)))+ (some (fn [[k v]]+ (or (str/includes? k "@")+ (and (str/includes? v "@")+ (or (not (credential-params k))+ (re-matches #"[^/]*//[^/]+" url)))))+ pairs))(throw (ex-info (str "MB_PGVECTOR_DB_URL has an unencoded @ outside its credentials. ""Percent-encode reserved characters, e.g. @ as %40, / as %2F and ? as %3F."){}))))+(def ^:private host-list-re+ "An empty host list, or comma-separated hosts, each a name or a bracketed IPv6 address with an optional port."+ (let [host "(?:[A-Za-z0-9._-]+|\\[[0-9A-Za-z:.%_~-]+\\])(?::[0-9]+)?"]+ (re-pattern (format "(?:%s(?:,%s)*)?" host host))))++(defn- check-host-list!+ "Throw if the host part of `url` isn't a host list, without quoting it."+ [^String url]+ ;; An unencoded `/` or `?` in the credentials ends the host part early, before their `@`.+ ;; Read only the host part, so nothing after it can make the check skip.+ (when-let [[_ host-list] (re-find #"^[^/]*//([^/]*)" url)]+ (when-not (re-matches host-list-re host-list)+ (throw (ex-info (str "MB_PGVECTOR_DB_URL has a malformed host list. Percent-encode reserved characters "+ "in its credentials, e.g. @ as %40, / as %2F and ? as %3F.")+ {})))))+(defn- parse-db-url"Parse a pgvector JDBC URL into {:jdbc-url ... :credentials ... :pool-props ...}, or throw if it ismalformed or carries an unrecognized parameter.@@ -152,6 +177,7 @@(let [[base pairs] (split-query url)[base-url host-credentials] (split-userinfo base)_ (check-no-stray-at! base-url pairs)+ _ (check-host-list! base-url)default-pool (merge fixed-pool-props(update-vals tunable-pool-props first)){:keys [pool conn credentials]}@@ -166,16 +192,17 @@;; credentials: passed to pgjdbc as connection properties, off the URL, because the URL gets;; printed -- pgjdbc logs it at DEBUG on every connection, and DriverManager quotes it in its;; "No suitable driver" error- (contains? #{"user" "password" "sslpassword"} k)+ (contains? credential-params k)(assoc-in acc [:credentials (keyword k)] (url-decode raw-v));; a c3p0 knob: coerce and apply to the pool (pgjdbc can't read these off the URL)+ ;; Errors name only known params and never quote a value.(contains? tunable-pool-props k)(let [parse (second (tunable-pool-props k))parsed (parse (url-decode raw-v))](when (nil? parsed)- (throw (ex-info (format "Invalid value for pgvector pool parameter %s: %s" k raw-v)- {:param k, :value raw-v})))+ (throw (ex-info (format "Invalid value for pgvector pool parameter %s" k)+ {:param k})))(assoc-in acc [:pool k] parsed));; a recognized pgjdbc connection property: leave it on the URL for the driver@@ -184,10 +211,10 @@;; pgjdbc would silently ignore anything else, so treat it as a typo and fail loudly:else- (throw (ex-info (format (str "Unknown pgvector URL parameter %s. Expected a c3p0 pool knob "+ (throw (ex-info (format (str "Unknown pgvector URL parameter. Expected a c3p0 pool knob ""(%s) or a Postgres connection property.")- k (str/join ", " (sort (keys tunable-pool-props))))- {:param k}))))+ (str/join ", " (sort (keys tunable-pool-props))))+ {})))){:pool default-pool, :conn [], :credentials host-credentials}pairs)]{:jdbc-url (cond-> base-url (seq conn) (str "?" (str/join "&" conn)))diff --git a/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.clj b/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.cljindex 725b9e57c9c..efa2dda37eb 100644--- a/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.clj+++ b/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.search.semantic.db-test(:require+ [clojure.math.combinatorics :as math.combo][clojure.string :as str][clojure.test :refer :all][metabase-enterprise.search.semantic.db.datasource :as semantic.db.datasource])@@ -195,40 +196,122 @@#"The pgvector URL sets password both before the host and as a parameter"(parse-db-url (str (userinfo-url "postgres:secret") "?password=other"))))](is (not (re-find #"secret|other" (str (ex-message e) (ex-data e)))))))- (testing "an unencoded `@` in a query value passes through, as pgjdbc accepts it"+ (testing "an unencoded `@` in a credential value passes through, as pgjdbc accepts it"(doseq [url [(str base-url "?user=mylogin@srv&password=p@ss")"jdbc:postgresql://?service=pgvector&user=mylogin@srv&password=p@ss"]](is (=? {:credentials {:user "mylogin@srv", :password "p@ss"}}- (parse-db-url url))))))+ (parse-db-url url)))))+ (testing "the host list may name several hosts, IPv6 addresses, or none"+ (doseq [url ["jdbc:postgresql://db1:5432,db_2,[::1]:5433,[fe80::1%25eth0],[fe80::1%25br-test]:5432/mb"+ "jdbc:postgresql:///mb_semantic_search"+ "jdbc:postgresql://?service=pgvector"+ "jdbc:postgresql:mb_semantic_search"]]+ (is (=? {:jdbc-url url} (parse-db-url url)))))+ (testing "an encoded `@` in another param's value stays on the URL as written"+ (is (=? {:jdbc-url (str base-url "?sslrootcert=/certs/a%40b/ca.pem")}+ (parse-db-url (str base-url "?sslrootcert=/certs/a%40b/ca.pem"))))))(deftest parse-db-url-validation-test- (testing "an unrecognized param throws rather than being silently ignored by pgjdbc"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Unknown pgvector URL parameter maxPoolSizee"- (parse-db-url (str base-url "?maxPoolSizee=5")))))- (testing "a malformed pool-knob value throws"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Invalid value for pgvector pool parameter maxPoolSize"- (parse-db-url (str base-url "?maxPoolSize=lots")))))- (testing "an `@` outside the host part throws without quoting the URL"- (doseq [url ["jdbc:postgresql://alice:se/cret@db/mb" ; `/` in the password- "jdbc:postgresql://alice:se?cret@db/mb" ; `?` in the password- "jdbc:postgresql://alice:se?service=cret@db/mb" ; `?`, putting the `@` in a param- "jdbc:postgresql://alice:s@e?cret@db/mb" ; `@` and then `?` in the password- "jdbc:postgresql://alice:secret@db/mb@elsewhere"]] ; `@` in the database name- (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url url)))]- (is (=? {:message (str "MB_PGVECTOR_DB_URL has an unencoded @ outside its credentials. "- "Percent-encode reserved characters, e.g. @ as %40, / as %2F and ? as %3F.")- :data {}}- {:message (ex-message e), :data (ex-data e)}))- (is (not (re-find #"cret|elsewhere" (str (ex-message e) (ex-data e))))))))+ (testing "an unrecognized param throws rather than being silently ignored by pgjdbc, without quoting it"+ (let [e (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Unknown pgvector URL parameter\. Expected a c3p0 pool knob"+ (parse-db-url (str base-url "?maxPoolSizee=5"))))]+ (is (= {} (ex-data e)))+ (is (not (str/includes? (ex-message e) "maxPoolSizee")))))+ (testing "a malformed pool-knob value throws, naming the knob but not the value"+ (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url (str base-url "?maxPoolSize=lots"))))]+ (is (= {:message "Invalid value for pgvector pool parameter maxPoolSize"+ :data {:param "maxPoolSize"}}+ {:message (ex-message e), :data (ex-data e)}))))+ (testing "an `@` outside the host part and credential values throws without quoting the URL"+ (doseq [url ["jdbc:postgresql://alice:se/cret@db/mb" ; `/` in the password+ "jdbc:postgresql://alice:se?cret@db/mb" ; `?` in the password+ "jdbc:postgresql://alice:se?service=cret@db/mb" ; `?`, putting the `@` in a param+ "jdbc:postgresql://alice:s@e?cret@db/mb" ; `@` and then `?` in the password+ "jdbc:postgresql://alice:secret@db/mb@elsewhere" ; `@` in the database name+ "jdbc:postgresql://alice:se/c?cret@db/mb" ; `/` and then `?` in the password+ "jdbc:postgresql://alice:se/c?sslmode=cret@db/mb"+ "jdbc:postgresql://alice:se/cret?service=word@db/mb"+ "jdbc:postgresql://alice:se/c?x=cret@db"+ "jdbc:postgresql://alice:se/c?x=cret@db:5432"+ "jdbc:postgresql://alice:5432?sslmode=cret&password=word@db/mb" ; digits and then `?` in the password+ "jdbc:postgresql://alice:5432,cret?password=word@db/mb"+ "jdbc:postgresql://alice:se?user=cret@db/mb"+ "jdbc:postgresql://db?user=mylogin@srv" ; hosts but no path+ (str base-url "?sslrootcert=/certs/a@b/ca.pem")]] ; `@` in a non-credential value+ (testing url+ (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url url)))]+ (is (= {:message (str "MB_PGVECTOR_DB_URL has an unencoded @ outside its credentials. "+ "Percent-encode reserved characters, e.g. @ as %40, / as %2F and ? as %3F.")+ :data {}}+ {:message (ex-message e), :data (ex-data e)}))))))+ (testing "a host list that isn't hosts and ports throws without quoting it"+ (doseq [url ["jdbc:postgresql://alice:se/c?password=cret@db/mb" ; `/` and then `?` in the password+ "jdbc:postgresql://alice:se/c?user=cret@db/mb"+ "jdbc:postgresql://alice:se/c\nx?password=cret@db/mb" ; a line break after the `/`+ "jdbc:postgresql://alice:se/c\r\nx?password=cret@db/mb"+ "jdbc:postgresql://alice:se/c\u2028x?password=cret@db/mb"+ "jdbc:postgresql://db:port/mb"+ "jdbc:postgresql://db:5432,/mb"]]+ (testing url+ (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url url)))]+ (is (= {:message (str "MB_PGVECTOR_DB_URL has a malformed host list. Percent-encode reserved "+ "characters in its credentials, e.g. @ as %40, / as %2F and ? as %3F.")+ :data {}}+ {:message (ex-message e), :data (ex-data e)}))))))(testing "a malformed %-escape in a credential throws without quoting any of the value"(let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url (str base-url "?password=hunter%zz"))))](is (=? {:message "Malformed %-escape in MB_PGVECTOR_DB_URL", :data {}, :cause nil}{:message (ex-message e), :data (ex-data e), :cause (ex-cause e)})))))+(def ^:private password-fragments+ "Distinct words to join into generated passwords. None appears anywhere else in a generated URL, so finding+ one outside the parsed credentials means part of the password went there."+ ["kiwi" "plum" "pear" "lime" "date" "fig" "yuzu"])++(defn- generated-passwords+ "Passwords made of [[password-fragments]] joined by every ordering of every non-empty subset of the URL's+ reserved characters, and the same with each fragment after a `?` or `&` named as a credential param."+ []+ (for [chars (mapcat math.combo/permutations (rest (math.combo/subsets [\/ \? \@ \: \& \=])))+ param? [false true]+ :let [words (map (fn [prev-char fragment]+ (if (and param? (#{\? \&} prev-char)) "password" fragment))+ (cons nil chars)+ password-fragments)]]+ (apply str (interleave words (concat chars [""])))))++(defn- percent-encode [^String s]+ (str/replace s #"[%/?@:&=]" #(format "%%%02X" (int (first %)))))++(deftest parse-db-url-generated-password-test+ (let [hosts ["db.example:5432/mb" "db.example/mb?sslmode=require" "db.example" "db1:5432,db2:5433/mb"]]+ (testing "a host-part password with unencoded reserved characters never reaches an error or the URL"+ (let [leaks (for [password (generated-passwords)+ host hosts+ :let [url (str "jdbc:postgresql://alice:" password "@" host)+ result (try+ (dissoc (parse-db-url url) :credentials)+ (catch Throwable e+ [(ex-message e) (ex-data e) (some-> (ex-cause e) ex-message)]))+ leaked (filter #(str/includes? (str result) %) password-fragments)]+ :when (seq leaked)]+ {:url url, :leaked leaked})]+ (is (empty? (take 5 leaks)))))+ (testing "the same passwords percent-encoded parse, with the password off the URL"+ (let [failures (for [password (generated-passwords)+ host hosts+ :let [url (str "jdbc:postgresql://alice:" (percent-encode password) "@" host)+ expected {:jdbc-url (str "jdbc:postgresql://" host)+ :credentials {:user "alice", :password password}}+ actual (try+ (select-keys (parse-db-url url) [:jdbc-url :credentials])+ (catch Throwable e (ex-message e)))]+ :when (not= expected actual)]+ {:url url, :actual actual})]+ (is (empty? (take 5 failures)))))))+(deftest pool-props-applied-by-c3p0-test(testing "c3p0 actually understands and applies every pool property name we set";; We hand c3p0 a Properties map; like pgjdbc, c3p0 silently ignores names it doesn't recognize, so a