Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Encrypt plaintext values of encrypted-at-rest columns on startup (#81498)

Alexander Polyankind568f38f1a63
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index a56ebff9e01..a31a2d713d8 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -59,7 +59,8 @@
[mdb.encryption
decrypt-db
encrypt-db
- encryption-check-status]
+ encryption-check-status
+ encrypt-plaintext-columns!]
[metabase.app-db.format
format-sql]
[mdb.setup
diff --git a/src/metabase/app_db/encryption.clj b/src/metabase/app_db/encryption.clj
index 0dfb15a159f..46daa382f11 100644
--- a/src/metabase/app_db/encryption.clj
+++ b/src/metabase/app_db/encryption.clj
@@ -284,6 +284,26 @@
(t2/update! :conn conn table {:id id} {column (encrypt-bytes-fn decrypted)}))))
(t2/reducible-select [table :id [column :value]])))
+(defn encrypt-plaintext-columns!
+ "Encrypt at rest any plaintext value in the encrypted-at-rest string columns. Runs on every startup: the one-shot
+ encryption backfill migrations cannot be relied on to have done this -- run without MB_ENCRYPTION_SECRET_KEY (the
+ `migrate` command does not check the key) they are recorded as executed while doing nothing, a boot of an older
+ version re-writes these columns through its own plaintext-era transforms (e.g. notification seeding re-creates
+ `notification_recipient.details` rows every boot), and `load-from-h2` copies a decrypted dump's values verbatim.
+ A value that decrypts with the current key is left byte-identical; whether a value is encrypted is
+ decided by [[encryption/decryptable-string?]] (actually decrypting), never by shape. The `^bytes` columns are not
+ scanned: every shipped version writes those encrypted, so they cannot regress this way. No-op when
+ MB_ENCRYPTION_SECRET_KEY is not set."
+ []
+ (when (encryption/default-encryption-enabled?)
+ (t2/with-transaction [conn]
+ (doseq [[table column] encrypted-string-columns]
+ (run! (fn [{:keys [id value]}]
+ (when (and (string? value)
+ (not (encryption/decryptable-string? value)))
+ (t2/update! :conn conn table {:id id} {column (encryption/encrypt value)})))
+ (t2/reducible-select [table :id [column :value]] {:where [:!= column nil]}))))))
+
(defn- do-encryption
"Encrypt or decrypts the db using the current `MB_ENCRYPTION_SECRET_KEY` to read data.
diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.clj
index 13d53b6912a..1641b0f8bc9 100644
--- a/src/metabase/core/core.clj
+++ b/src/metabase/core/core.clj
@@ -191,6 +191,7 @@
(mdb/setup-db! :create-sample-content? (not config/is-test?))
;; runs before anything reads settings -- see its docstring
(setting/migrate-encrypted-settings!)
+ (mdb/encrypt-plaintext-columns!)
;; In OSS, convert any Data Analysts group with members to a normal visible group
(perms/sync-data-analyst-group-for-oss!)
;; Disable read-only mode if its on during startup.
diff --git a/test/metabase/app_db/encryption_test.clj b/test/metabase/app_db/encryption_test.clj
new file mode 100644
index 00000000000..ee2c64f8cd7
--- /dev/null
+++ b/test/metabase/app_db/encryption_test.clj
@@ -0,0 +1,50 @@
+(ns metabase.app-db.encryption-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.app-db.core :as mdb]
+ [metabase.notification.core :as notification]
+ [metabase.test :as mt]
+ [metabase.util.encryption :as encryption]
+ [metabase.util.encryption-test :as encryption-test]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- recipient-details []
+ (t2/select-fn-vec :details :notification_recipient :details [:!= nil]))
+
+(deftest encrypt-plaintext-columns!-test
+ ;; Re-enacts how a boot of a pre-encryption build undoes the one-shot encryption backfills: its seeding reads the
+ ;; encrypted `notification_recipient.details` through plain `transform-json`, gets a ciphertext string instead of a
+ ;; map, decides the row changed, and re-creates it through its plaintext-era transforms. Isolated app DB: runs with
+ ;; an encryption key active, so nothing here may touch the shared test DB.
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
+ (encryption-test/with-secret-key "ABCDEFGH12345678"
+ (notification/seed-notification!)
+ (let [seeded (recipient-details)]
+ (is (seq seeded) "seeding created recipients with details")
+ (is (every? encryption/decryptable-string? seeded) "seeded through the current build: encrypted at rest")
+ (testing "an old build's seed re-writes the rows plaintext; the heal re-encrypts them"
+ (t2/query {:update :notification_recipient
+ :set {:details "{\"pattern\":\"plain\"}"}
+ :where [:!= :details nil]})
+ (is (not-any? encryption/decryptable-string? (recipient-details)) "now plaintext, as an old build leaves them")
+ (mdb/encrypt-plaintext-columns!)
+ (let [healed (recipient-details)]
+ (is (every? encryption/decryptable-string? healed))
+ (is (= "{\"pattern\":\"plain\"}" (encryption/decrypt (first healed))))))
+ (testing "the strict reader that crashed startup now works: seeding runs cleanly again"
+ (notification/seed-notification!))
+ (testing "idempotent: a second run leaves every value byte-identical"
+ (let [snapshot (recipient-details)]
+ (mdb/encrypt-plaintext-columns!)
+ (is (= snapshot (recipient-details))))))))
+ (testing "without an encryption key nothing happens"
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
+ (encryption-test/with-secret-key nil
+ (notification/seed-notification!)
+ (let [before (recipient-details)]
+ (mdb/encrypt-plaintext-columns!)
+ (is (= before (recipient-details))))))))