Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add MB_DISABLE_LEGACY_STARTUP_ENCRYPTION to refuse startup instead of encrypting legacy plaintext (#82097)

Alexander Polyankin4b2cfdc62327
diff --git a/src/metabase/app_db/encryption.clj b/src/metabase/app_db/encryption.clj
index ca4d9f749df..15d5e7e620c 100644
--- a/src/metabase/app_db/encryption.clj
+++ b/src/metabase/app_db/encryption.clj
@@ -7,6 +7,7 @@
[metabase.app-db.db :as mdb.db]
[metabase.app-db.query :as mdb.query]
[metabase.app-db.setting :as mdb.setting]
+ [metabase.config.core :as config]
[metabase.util :as u]
[metabase.util.encryption :as encryption]
[metabase.util.i18n :refer [trs]]
@@ -315,11 +316,24 @@
(string? value)
(not (encryption/decryptable-string? value opts)))))
+(defn legacy-startup-encryption-disabled?
+ "Whether `MB_DISABLE_LEGACY_STARTUP_ENCRYPTION` is set to true. By default startup encrypts, with a warning, any
+ value that a previous version of Metabase stored unencrypted in an encrypted-at-rest column; when disabled, finding
+ such a value is an error and startup refuses to run instead."
+ []
+ (boolean (config/config-bool :mb-disable-legacy-startup-encryption)))
+
(defn- handle-legacy-unencrypted-values!
"What happens when legacy values that a previous version of Metabase stored unencrypted are found in `location` (a
- `table.column`), before they are encrypted: for now a warning."
+ `table.column`), before they are encrypted: a warning, or an error when [[legacy-startup-encryption-disabled?]]."
[location]
- (log/warnf "Encrypting legacy values in %s that a previous version of Metabase stored unencrypted." location))
+ (if (legacy-startup-encryption-disabled?)
+ (throw (ex-info (format (str "Found legacy values in %s that a previous version of Metabase stored unencrypted, "
+ "and MB_DISABLE_LEGACY_STARTUP_ENCRYPTION is set. Unset it to let Metabase encrypt "
+ "them on startup.")
+ location)
+ {:location location}))
+ (log/warnf "Encrypting legacy values in %s that a previous version of Metabase stored unencrypted." location)))
(defn encrypt-plaintext-columns!
"Encrypt at rest any plaintext value in the encrypted-at-rest string columns. Runs on every startup, and is the only
@@ -329,7 +343,8 @@
transforms (e.g. notification seeding re-creates `notification_recipient.details` rows every boot), and
`load-from-h2` copies a decrypted dump's values verbatim. A value that decrypts with the current key is left
byte-identical; whether a value is encrypted is decided by [[encryption/decryptable-string?]] (actually decrypting),
- never by shape. Streams each column, warning once per column before its first row is encrypted. The `^bytes`
+ never by shape. Streams each column, warning once per column before its first row is encrypted -- or refusing to
+ encrypt at all when [[legacy-startup-encryption-disabled?]]. The `^bytes`
columns are not scanned: every shipped version writes those encrypted, so they cannot regress this way. No-op when
MB_ENCRYPTION_SECRET_KEY is not set."
[]
diff --git a/src/metabase/app_db/setup.clj b/src/metabase/app_db/setup.clj
index f16c20b4de6..230622846a4 100644
--- a/src/metabase/app_db/setup.clj
+++ b/src/metabase/app_db/setup.clj
@@ -254,9 +254,16 @@
[db-state]
(when (#{:encrypted :unencrypted :fresh :pre-sentinel} db-state)
(when (mdb.db/unmigrated-settings?)
- (log/warn (str "Some settings were saved by an older version of Metabase and are being converted to the current "
- "storage format" (when (encryption/default-encryption-enabled?) ", encrypted with MB_ENCRYPTION_SECRET_KEY")
- ". This is expected once after an upgrade.")))
+ (if (and (encryption/default-encryption-enabled?)
+ (mdb.encryption/legacy-startup-encryption-disabled?))
+ (throw (ex-info (str "Some settings were saved by an older version of Metabase and would be converted to the "
+ "current storage format, encrypted with MB_ENCRYPTION_SECRET_KEY, but "
+ "MB_DISABLE_LEGACY_STARTUP_ENCRYPTION is set. Unset it to let Metabase convert them on "
+ "startup.")
+ {}))
+ (log/warn (str "Some settings were saved by an older version of Metabase and are being converted to the current "
+ "storage format" (when (encryption/default-encryption-enabled?) ", encrypted with MB_ENCRYPTION_SECRET_KEY")
+ ". This is expected once after an upgrade."))))
(mdb.setting/migrate-settings!)))
;; TODO -- consider renaming to something like `verify-connection-and-migrate!`
diff --git a/src/metabase/cmd/resources/other-env-vars.md b/src/metabase/cmd/resources/other-env-vars.md
index ebbebd6f777..6d0566491e5 100644
--- a/src/metabase/cmd/resources/other-env-vars.md
+++ b/src/metabase/cmd/resources/other-env-vars.md
@@ -219,6 +219,13 @@ Default: `null`
Used during development of third-party drivers. Set the value to have that plugin manifest get loaded during startup. Specify multiple plugin manifests by comma-separating them.
+### `MB_DISABLE_LEGACY_STARTUP_ENCRYPTION`
+
+Type: boolean<br>
+Default: `false`
+
+By default, when [MB_ENCRYPTION_SECRET_KEY](#mb_encryption_secret_key) is set, Metabase encrypts on startup any values that an older version of Metabase stored unencrypted, and logs a warning for each column it had to encrypt. When `true`, Metabase will refuse to start instead of encrypting such values, including settings saved by an older version.
+
### `MB_DISABLE_SCHEDULER`
Type: boolean<br>
diff --git a/test/metabase/app_db/encryption_test.clj b/test/metabase/app_db/encryption_test.clj
index ea63a118330..e4229816cec 100644
--- a/test/metabase/app_db/encryption_test.clj
+++ b/test/metabase/app_db/encryption_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase.app-db.core :as mdb]
+ [metabase.app-db.encryption :as mdb.encryption]
[metabase.notification.core :as notification]
[metabase.test :as mt]
[metabase.util.encryption :as encryption]
@@ -43,7 +44,21 @@
(testing "idempotent: a second run leaves every value byte-identical"
(let [snapshot (recipient-details)]
(mdb/encrypt-plaintext-columns!)
- (is (= snapshot (recipient-details))))))))
+ (is (= snapshot (recipient-details)))))
+ (testing "with MB_DISABLE_LEGACY_STARTUP_ENCRYPTION the heal refuses instead, leaving the rows as they are"
+ (t2/query {:update :notification_recipient
+ :set {:details "{\"pattern\":\"plain\"}"}
+ :where [:!= :details nil]})
+ (mt/with-temp-env-var-value! [mb-disable-legacy-startup-encryption "true"]
+ (is (mdb.encryption/legacy-startup-encryption-disabled?))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Found legacy values in notification_recipient\.details .* MB_DISABLE_LEGACY_STARTUP_ENCRYPTION is set"
+ (mdb/encrypt-plaintext-columns!)))
+ (is (not-any? encryption/decryptable-string? (recipient-details)) "nothing was encrypted"))
+ (testing "unset again, the heal runs"
+ (is (not (mdb.encryption/legacy-startup-encryption-disabled?)))
+ (mdb/encrypt-plaintext-columns!)
+ (is (every? encryption/decryptable-string? (recipient-details))))))))
(testing "without an encryption key nothing happens"
(mt/with-temp-empty-app-db [_conn :h2]
(mdb/setup-db! :create-sample-content? false)
diff --git a/test/metabase/app_db/setup_test.clj b/test/metabase/app_db/setup_test.clj
index 50726f9289a..a57a475fb7c 100644
--- a/test/metabase/app_db/setup_test.clj
+++ b/test/metabase/app_db/setup_test.clj
@@ -7,6 +7,7 @@
[metabase.app-db.core :as mdb]
[metabase.app-db.data-source :as mdb.data-source]
[metabase.app-db.db :as mdb.db]
+ [metabase.app-db.encryption :as mdb.encryption]
[metabase.app-db.liquibase :as liquibase]
[metabase.app-db.setting :as mdb.setting]
[metabase.app-db.setup :as mdb.setup]
@@ -261,6 +262,49 @@
(is (not (mdb.db/unmigrated-settings?)))
(is (= "Sad Can" (t2/select-one-fn :value_with_aad :setting :key "site-name"))))))
+(deftest setup-fresh-db-with-legacy-encryption-disabled-test
+ (testing "with MB_DISABLE_LEGACY_STARTUP_ENCRYPTION and a key, a fresh database starts: nothing is legacy, so nothing is refused or warned about"
+ (mt/with-temp-env-var-value! [mb-disable-legacy-startup-encryption "true"]
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (encryption-test/with-secret-key "ABCDEFGH12345678"
+ (is (mdb.encryption/legacy-startup-encryption-disabled?))
+ (mt/with-log-messages-for-level [messages :warn]
+ (is (= :done (mdb/setup-db! :create-sample-content? true)))
+ (mdb/encrypt-plaintext-columns!)
+ (is (empty? (filter #(re-find #"older version|legacy values" (:message %)) (messages)))))
+ (is (= :valid (mdb/encryption-check-status)))
+ (is (not (mdb.db/unmigrated-settings?)))
+ (is (= ["E-commerce Insights"] (t2/select-fn-vec :name :report_dashboard)) "sample content was created"))))))
+
+(deftest setup-db-refuses-settings-from-an-older-version-when-legacy-encryption-disabled-test
+ (testing "with MB_DISABLE_LEGACY_STARTUP_ENCRYPTION and a key, a setting row without value_with_aad refuses startup"
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
+ (encryption-test/with-secret-key "ABCDEFGH12345678"
+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)
+ (t2/query {:insert-into :setting, :values [{:key "site-name", :value "Sad Can"}]})
+ (is (mdb.db/unmigrated-settings?))
+ (mt/with-temp-env-var-value! [mb-disable-legacy-startup-encryption "true"]
+ (reset! (:status mdb.connection/*application-db*) ::not-set-up)
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Some settings were saved by an older version of Metabase .* MB_DISABLE_LEGACY_STARTUP_ENCRYPTION is set"
+ (mdb/setup-db! :create-sample-content? false)))
+ (is (mdb.db/unmigrated-settings?) "nothing was converted"))
+ (testing "unset again, setup converts them"
+ (reset! (:status mdb.connection/*application-db*) ::not-set-up)
+ (is (= :done (mdb/setup-db! :create-sample-content? false)))
+ (is (not (mdb.db/unmigrated-settings?))))))
+ (testing "without a key there is nothing to encrypt, so the setting is converted with a warning as usual"
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
+ (t2/query {:insert-into :setting, :values [{:key "site-name", :value "Sad Can"}]})
+ (is (mdb.db/unmigrated-settings?))
+ (mt/with-temp-env-var-value! [mb-disable-legacy-startup-encryption "true"]
+ (reset! (:status mdb.connection/*application-db*) ::not-set-up)
+ (is (= :done (mdb/setup-db! :create-sample-content? false)))
+ (is (not (mdb.db/unmigrated-settings?)))
+ (is (= "Sad Can" (t2/select-one-fn :value_with_aad :setting :key "site-name"))))))))
+
(deftest migrate-settings-decides-by-decrypting-test
(testing "the value_with_aad backfill treats a value as encrypted only if it decrypts, and copes with one that encrypts to nothing"
(mt/with-temp-empty-app-db [_conn :h2]
diff --git a/test/metabase/test/initialize/db.clj b/test/metabase/test/initialize/db.clj
index 5cafbd23a5e..4cd74fdae10 100644
--- a/test/metabase/test/initialize/db.clj
+++ b/test/metabase/test/initialize/db.clj
@@ -1,5 +1,6 @@
(ns metabase.test.initialize.db
(:require
+ [metabase.app-db.connection :as mdb.connection]
[metabase.app-db.core :as mdb]
[metabase.task.bootstrap :as task.bootstrap]
[metabase.util :as u]
@@ -8,12 +9,17 @@
(set! *warn-on-reflection* true)
-(defn init! []
- (log/info (u/format-color 'blue "Setting up %s test DB and running migrations..." (mdb/db-type)))
- (task.bootstrap/set-jdbc-backend-properties! (mdb/db-type))
- (mdb/setup-db! :create-sample-content? false) ; skip sample content for speedy tests. this doesn't reflect production
- (log/info (t2/with-connection [^java.sql.Connection conn]
- (let [metadata (.getMetaData conn)]
- (u/format-color 'blue "Application DB is %s %s"
- (.getDatabaseProductName metadata)
- (.getDatabaseProductVersion metadata))))))
+(defn init!
+ "Set up the shared test app DB. Always the root [[mdb.connection/*application-db*]], whatever is bound when this is
+ triggered: the initialized marker is global, so setting up a temporarily bound DB (e.g. inside `with-temp-empty-app-db`)
+ would leave the shared one empty for every later test."
+ []
+ (binding [mdb.connection/*application-db* (.getRawRoot ^clojure.lang.Var #'mdb.connection/*application-db*)]
+ (log/info (u/format-color 'blue "Setting up %s test DB and running migrations..." (mdb/db-type)))
+ (task.bootstrap/set-jdbc-backend-properties! (mdb/db-type))
+ (mdb/setup-db! :create-sample-content? false) ; skip sample content for speedy tests. this doesn't reflect production
+ (log/info (t2/with-connection [^java.sql.Connection conn]
+ (let [metadata (.getMetaData conn)]
+ (u/format-color 'blue "Application DB is %s %s"
+ (.getDatabaseProductName metadata)
+ (.getDatabaseProductVersion metadata)))))))
diff --git a/test/metabase/util/encryption_test.clj b/test/metabase/util/encryption_test.clj
index 6b90d5110a3..7e9343b549a 100644
--- a/test/metabase/util/encryption_test.clj
+++ b/test/metabase/util/encryption_test.clj
@@ -5,7 +5,6 @@
[clojure.test :refer :all]
[metabase.settings.models.setting.cache :as setting.cache]
[metabase.test :as mt]
- [metabase.test.initialize :as initialize]
[metabase.util.encryption :as encryption]
[metabase.util.string :as string])
(:import (java.io ByteArrayInputStream)
@@ -23,7 +22,6 @@
(.set ^java.util.concurrent.atomic.AtomicLong @#'setting.cache/last-update-check 0))
(defn do-with-secret-key! [^String secret-key thunk]
- (initialize/initialize-if-needed! :db)
(clear-setting-cache!)
(try
(with-redefs [encryption/default-secret-key (when (seq secret-key)