Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Validate MB_PGVECTOR_DB_URL more strictly (#85900)

Chris Trutere715f63f070d
diff --git a/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.clj
index 478cb0a41ba..825e26e3ab7 100644
--- a/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.clj
+++ b/enterprise/backend/src/metabase_enterprise/search/semantic/db/datasource.clj
@@ -129,21 +129,46 @@
[(str scheme more) credentials])
[base {}]))
+(def ^:private credential-params
+ "The URL params that carry credentials. They go to pgjdbc as connection properties, not on the URL."
+ #{"user" "password" "sslpassword"})
+
(defn- check-no-stray-at!
"Throw if an `@` is left in `url` or `pairs` once the credentials are off, without quoting either."
[^String url pairs]
;; An `@` left on `url` means a password with an unencoded `/`, or an `@` that pgjdbc requires encoded anyway.
- ;; An unencoded `?` in a password ends the host part early, before the `/` pgjdbc requires after a host, and
- ;; moves the `@` into `pairs`.
- ;; Any other `@` in `pairs` is a plain value that pgjdbc accepts, such as Azure's `user=name@server`.
+ ;; An unencoded `/` or `?` in a password ends the host part early and moves the `@` into `pairs`.
+ ;; A credential value may hold a plain `@`, such as Azure's `user=name@server`; any other `@` must be encoded.
+ ;; A URL with hosts but no path can't have its query `@` checked that way: the `?` that ended its host part
+ ;; may belong to the credentials, so every `@` there must be encoded.
;; Refuse a stray `@` before an error or the URL can quote the part of the password it may hold.
(when (or (str/includes? url "@")
- (and (re-matches #"[^/]*//[^/]+" url)
- (some (fn [[k v]] (str/includes? (str k v) "@")) pairs)))
+ (some (fn [[k v]]
+ (or (str/includes? k "@")
+ (and (str/includes? v "@")
+ (or (not (credential-params k))
+ (re-matches #"[^/]*//[^/]+" url)))))
+ pairs))
(throw (ex-info (str "MB_PGVECTOR_DB_URL has an unencoded @ outside its credentials. "
"Percent-encode reserved characters, e.g. @ as %40, / as %2F and ? as %3F.")
{}))))
+(def ^:private host-list-re
+ "An empty host list, or comma-separated hosts, each a name or a bracketed IPv6 address with an optional port."
+ (let [host "(?:[A-Za-z0-9._-]+|\\[[0-9A-Za-z:.%_~-]+\\])(?::[0-9]+)?"]
+ (re-pattern (format "(?:%s(?:,%s)*)?" host host))))
+
+(defn- check-host-list!
+ "Throw if the host part of `url` isn't a host list, without quoting it."
+ [^String url]
+ ;; An unencoded `/` or `?` in the credentials ends the host part early, before their `@`.
+ ;; Read only the host part, so nothing after it can make the check skip.
+ (when-let [[_ host-list] (re-find #"^[^/]*//([^/]*)" url)]
+ (when-not (re-matches host-list-re host-list)
+ (throw (ex-info (str "MB_PGVECTOR_DB_URL has a malformed host list. Percent-encode reserved characters "
+ "in its credentials, e.g. @ as %40, / as %2F and ? as %3F.")
+ {})))))
+
(defn- parse-db-url
"Parse a pgvector JDBC URL into {:jdbc-url ... :credentials ... :pool-props ...}, or throw if it is
malformed or carries an unrecognized parameter.
@@ -152,6 +177,7 @@
(let [[base pairs] (split-query url)
[base-url host-credentials] (split-userinfo base)
_ (check-no-stray-at! base-url pairs)
+ _ (check-host-list! base-url)
default-pool (merge fixed-pool-props
(update-vals tunable-pool-props first))
{:keys [pool conn credentials]}
@@ -166,16 +192,17 @@
;; credentials: passed to pgjdbc as connection properties, off the URL, because the URL gets
;; printed -- pgjdbc logs it at DEBUG on every connection, and DriverManager quotes it in its
;; "No suitable driver" error
- (contains? #{"user" "password" "sslpassword"} k)
+ (contains? credential-params k)
(assoc-in acc [:credentials (keyword k)] (url-decode raw-v))
;; a c3p0 knob: coerce and apply to the pool (pgjdbc can't read these off the URL)
+ ;; Errors name only known params and never quote a value.
(contains? tunable-pool-props k)
(let [parse (second (tunable-pool-props k))
parsed (parse (url-decode raw-v))]
(when (nil? parsed)
- (throw (ex-info (format "Invalid value for pgvector pool parameter %s: %s" k raw-v)
- {:param k, :value raw-v})))
+ (throw (ex-info (format "Invalid value for pgvector pool parameter %s" k)
+ {:param k})))
(assoc-in acc [:pool k] parsed))
;; a recognized pgjdbc connection property: leave it on the URL for the driver
@@ -184,10 +211,10 @@
;; pgjdbc would silently ignore anything else, so treat it as a typo and fail loudly
:else
- (throw (ex-info (format (str "Unknown pgvector URL parameter %s. Expected a c3p0 pool knob "
+ (throw (ex-info (format (str "Unknown pgvector URL parameter. Expected a c3p0 pool knob "
"(%s) or a Postgres connection property.")
- k (str/join ", " (sort (keys tunable-pool-props))))
- {:param k}))))
+ (str/join ", " (sort (keys tunable-pool-props))))
+ {}))))
{:pool default-pool, :conn [], :credentials host-credentials}
pairs)]
{:jdbc-url (cond-> base-url (seq conn) (str "?" (str/join "&" conn)))
diff --git a/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.clj b/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.clj
index 725b9e57c9c..efa2dda37eb 100644
--- a/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/semantic/db_test.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.search.semantic.db-test
(:require
+ [clojure.math.combinatorics :as math.combo]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.search.semantic.db.datasource :as semantic.db.datasource])
@@ -195,40 +196,122 @@
#"The pgvector URL sets password both before the host and as a parameter"
(parse-db-url (str (userinfo-url "postgres:secret") "?password=other"))))]
(is (not (re-find #"secret|other" (str (ex-message e) (ex-data e)))))))
- (testing "an unencoded `@` in a query value passes through, as pgjdbc accepts it"
+ (testing "an unencoded `@` in a credential value passes through, as pgjdbc accepts it"
(doseq [url [(str base-url "?user=mylogin@srv&password=p@ss")
"jdbc:postgresql://?service=pgvector&user=mylogin@srv&password=p@ss"]]
(is (=? {:credentials {:user "mylogin@srv", :password "p@ss"}}
- (parse-db-url url))))))
+ (parse-db-url url)))))
+ (testing "the host list may name several hosts, IPv6 addresses, or none"
+ (doseq [url ["jdbc:postgresql://db1:5432,db_2,[::1]:5433,[fe80::1%25eth0],[fe80::1%25br-test]:5432/mb"
+ "jdbc:postgresql:///mb_semantic_search"
+ "jdbc:postgresql://?service=pgvector"
+ "jdbc:postgresql:mb_semantic_search"]]
+ (is (=? {:jdbc-url url} (parse-db-url url)))))
+ (testing "an encoded `@` in another param's value stays on the URL as written"
+ (is (=? {:jdbc-url (str base-url "?sslrootcert=/certs/a%40b/ca.pem")}
+ (parse-db-url (str base-url "?sslrootcert=/certs/a%40b/ca.pem"))))))
(deftest parse-db-url-validation-test
- (testing "an unrecognized param throws rather than being silently ignored by pgjdbc"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Unknown pgvector URL parameter maxPoolSizee"
- (parse-db-url (str base-url "?maxPoolSizee=5")))))
- (testing "a malformed pool-knob value throws"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Invalid value for pgvector pool parameter maxPoolSize"
- (parse-db-url (str base-url "?maxPoolSize=lots")))))
- (testing "an `@` outside the host part throws without quoting the URL"
- (doseq [url ["jdbc:postgresql://alice:se/cret@db/mb" ; `/` in the password
- "jdbc:postgresql://alice:se?cret@db/mb" ; `?` in the password
- "jdbc:postgresql://alice:se?service=cret@db/mb" ; `?`, putting the `@` in a param
- "jdbc:postgresql://alice:s@e?cret@db/mb" ; `@` and then `?` in the password
- "jdbc:postgresql://alice:secret@db/mb@elsewhere"]] ; `@` in the database name
- (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url url)))]
- (is (=? {:message (str "MB_PGVECTOR_DB_URL has an unencoded @ outside its credentials. "
- "Percent-encode reserved characters, e.g. @ as %40, / as %2F and ? as %3F.")
- :data {}}
- {:message (ex-message e), :data (ex-data e)}))
- (is (not (re-find #"cret|elsewhere" (str (ex-message e) (ex-data e))))))))
+ (testing "an unrecognized param throws rather than being silently ignored by pgjdbc, without quoting it"
+ (let [e (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Unknown pgvector URL parameter\. Expected a c3p0 pool knob"
+ (parse-db-url (str base-url "?maxPoolSizee=5"))))]
+ (is (= {} (ex-data e)))
+ (is (not (str/includes? (ex-message e) "maxPoolSizee")))))
+ (testing "a malformed pool-knob value throws, naming the knob but not the value"
+ (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url (str base-url "?maxPoolSize=lots"))))]
+ (is (= {:message "Invalid value for pgvector pool parameter maxPoolSize"
+ :data {:param "maxPoolSize"}}
+ {:message (ex-message e), :data (ex-data e)}))))
+ (testing "an `@` outside the host part and credential values throws without quoting the URL"
+ (doseq [url ["jdbc:postgresql://alice:se/cret@db/mb" ; `/` in the password
+ "jdbc:postgresql://alice:se?cret@db/mb" ; `?` in the password
+ "jdbc:postgresql://alice:se?service=cret@db/mb" ; `?`, putting the `@` in a param
+ "jdbc:postgresql://alice:s@e?cret@db/mb" ; `@` and then `?` in the password
+ "jdbc:postgresql://alice:secret@db/mb@elsewhere" ; `@` in the database name
+ "jdbc:postgresql://alice:se/c?cret@db/mb" ; `/` and then `?` in the password
+ "jdbc:postgresql://alice:se/c?sslmode=cret@db/mb"
+ "jdbc:postgresql://alice:se/cret?service=word@db/mb"
+ "jdbc:postgresql://alice:se/c?x=cret@db"
+ "jdbc:postgresql://alice:se/c?x=cret@db:5432"
+ "jdbc:postgresql://alice:5432?sslmode=cret&password=word@db/mb" ; digits and then `?` in the password
+ "jdbc:postgresql://alice:5432,cret?password=word@db/mb"
+ "jdbc:postgresql://alice:se?user=cret@db/mb"
+ "jdbc:postgresql://db?user=mylogin@srv" ; hosts but no path
+ (str base-url "?sslrootcert=/certs/a@b/ca.pem")]] ; `@` in a non-credential value
+ (testing url
+ (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url url)))]
+ (is (= {:message (str "MB_PGVECTOR_DB_URL has an unencoded @ outside its credentials. "
+ "Percent-encode reserved characters, e.g. @ as %40, / as %2F and ? as %3F.")
+ :data {}}
+ {:message (ex-message e), :data (ex-data e)}))))))
+ (testing "a host list that isn't hosts and ports throws without quoting it"
+ (doseq [url ["jdbc:postgresql://alice:se/c?password=cret@db/mb" ; `/` and then `?` in the password
+ "jdbc:postgresql://alice:se/c?user=cret@db/mb"
+ "jdbc:postgresql://alice:se/c\nx?password=cret@db/mb" ; a line break after the `/`
+ "jdbc:postgresql://alice:se/c\r\nx?password=cret@db/mb"
+ "jdbc:postgresql://alice:se/c\u2028x?password=cret@db/mb"
+ "jdbc:postgresql://db:port/mb"
+ "jdbc:postgresql://db:5432,/mb"]]
+ (testing url
+ (let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url url)))]
+ (is (= {:message (str "MB_PGVECTOR_DB_URL has a malformed host list. Percent-encode reserved "
+ "characters in its credentials, e.g. @ as %40, / as %2F and ? as %3F.")
+ :data {}}
+ {:message (ex-message e), :data (ex-data e)}))))))
(testing "a malformed %-escape in a credential throws without quoting any of the value"
(let [e (is (thrown? clojure.lang.ExceptionInfo (parse-db-url (str base-url "?password=hunter%zz"))))]
(is (=? {:message "Malformed %-escape in MB_PGVECTOR_DB_URL", :data {}, :cause nil}
{:message (ex-message e), :data (ex-data e), :cause (ex-cause e)})))))
+(def ^:private password-fragments
+ "Distinct words to join into generated passwords. None appears anywhere else in a generated URL, so finding
+ one outside the parsed credentials means part of the password went there."
+ ["kiwi" "plum" "pear" "lime" "date" "fig" "yuzu"])
+
+(defn- generated-passwords
+ "Passwords made of [[password-fragments]] joined by every ordering of every non-empty subset of the URL's
+ reserved characters, and the same with each fragment after a `?` or `&` named as a credential param."
+ []
+ (for [chars (mapcat math.combo/permutations (rest (math.combo/subsets [\/ \? \@ \: \& \=])))
+ param? [false true]
+ :let [words (map (fn [prev-char fragment]
+ (if (and param? (#{\? \&} prev-char)) "password" fragment))
+ (cons nil chars)
+ password-fragments)]]
+ (apply str (interleave words (concat chars [""])))))
+
+(defn- percent-encode [^String s]
+ (str/replace s #"[%/?@:&=]" #(format "%%%02X" (int (first %)))))
+
+(deftest parse-db-url-generated-password-test
+ (let [hosts ["db.example:5432/mb" "db.example/mb?sslmode=require" "db.example" "db1:5432,db2:5433/mb"]]
+ (testing "a host-part password with unencoded reserved characters never reaches an error or the URL"
+ (let [leaks (for [password (generated-passwords)
+ host hosts
+ :let [url (str "jdbc:postgresql://alice:" password "@" host)
+ result (try
+ (dissoc (parse-db-url url) :credentials)
+ (catch Throwable e
+ [(ex-message e) (ex-data e) (some-> (ex-cause e) ex-message)]))
+ leaked (filter #(str/includes? (str result) %) password-fragments)]
+ :when (seq leaked)]
+ {:url url, :leaked leaked})]
+ (is (empty? (take 5 leaks)))))
+ (testing "the same passwords percent-encoded parse, with the password off the URL"
+ (let [failures (for [password (generated-passwords)
+ host hosts
+ :let [url (str "jdbc:postgresql://alice:" (percent-encode password) "@" host)
+ expected {:jdbc-url (str "jdbc:postgresql://" host)
+ :credentials {:user "alice", :password password}}
+ actual (try
+ (select-keys (parse-db-url url) [:jdbc-url :credentials])
+ (catch Throwable e (ex-message e)))]
+ :when (not= expected actual)]
+ {:url url, :actual actual})]
+ (is (empty? (take 5 failures)))))))
+
(deftest pool-props-applied-by-c3p0-test
(testing "c3p0 actually understands and applies every pool property name we set"
;; We hand c3p0 a Properties map; like pgjdbc, c3p0 silently ignores names it doesn't recognize, so a