The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Reset undecryptable rows to an empty value instead of aborting the entire decryption process (#78383)
Ben Grabow93f9a2fa2cda
diff --git a/src/metabase/app_db/encryption.clj b/src/metabase/app_db/encryption.cljindex 8d530810f5b..70efe53da28 100644--- a/src/metabase/app_db/encryption.clj+++ b/src/metabase/app_db/encryption.clj@@ -3,6 +3,8 @@[metabase.util.encryption :as encryption][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [trs]]+ [metabase.util.log :as log]+ [metabase.util.string :as string][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -21,16 +23,50 @@[:workspace_database :database_details][:auth_identity :credentials]])+;; Older versions of dump-to-h2 and key rotation only processed `metabase_database.details` (plus settings and+;; secrets), skipping every other encrypted JSON column. A dump or rotation from such a version left the skipped+;; columns encrypted with the source instance's key, so on databases that have been through one they can hold values+;; the current (otherwise correct) key cannot decrypt. Only the columns listed here — the ones confirmed affected in+;; production — may be cleared when undecryptable; everything else still aborts, so that legitimately decryptable+;; data can never be cleared by mistake.+(def ^:private clearable-when-undecryptable+ #{[:core_user :settings]})++(defn- encryption-check-status+ "Whether the current MB_ENCRYPTION_SECRET_KEY is the right key for this database, according to the+ `encryption-check` sentinel setting (a random UUID stored encrypted whenever the database is encrypted):++ :valid - the sentinel decrypts to a UUID, so the key is correct+ :invalid - the sentinel exists but does not decrypt, so the key is wrong (or unset) for this database+ :unknown - no sentinel (database predates it), or the database is marked unencrypted"+ []+ (let [raw (t2/select-one-fn :value :setting :key "encryption-check")]+ (cond+ (or (nil? raw) (= raw "unencrypted")) :unknown+ (string/valid-uuid? (encryption/maybe-decrypt raw)) :valid+ :else :invalid)))+(defn- reencrypt-encrypted-json-column!- "Re-encrypt `column` for every row in `table` using `encrypt-str-fn`. See `encrypted-json-columns`."- [conn table column encrypt-str-fn]+ "Re-encrypt `column` for every row in `table` using `encrypt-str-fn`. See `encrypted-json-columns`.++ When `clear-undecryptable?` is true, a value that cannot be decrypted with the current key is reset to an empty+ JSON object (with a warning) instead of aborting. Only pass true when the current key is known to be correct for+ this database (see `encryption-check-status`) and the column can legitimately hold values written with some other+ key (see `clearable-when-undecryptable`): such values are equally unreadable at runtime, so clearing them loses+ nothing that was usable."+ [conn table column encrypt-str-fn clear-undecryptable?](doseq [{:keys [id value]} (t2/select [table :id [column :value]])](when (some? value)(let [decrypted (encryption/maybe-decrypt value)]- (when (encryption/possibly-encrypted-string? decrypted)- (throw (ex-info (trs "Can''t decrypt app db with MB_ENCRYPTION_SECRET_KEY")- {:table table, :id id, :column column})))- (t2/update! :conn conn table {:id id} {column (encrypt-str-fn decrypted)})))))+ (if (encryption/possibly-encrypted-string? decrypted)+ (if clear-undecryptable?+ (do+ (log/warnf "Can't decrypt %s.%s for id %s with MB_ENCRYPTION_SECRET_KEY even though the key is correct for this database; resetting the value to {}. It was likely written with a different key and has been unreadable at runtime."+ (name table) (name column) id)+ (t2/update! :conn conn table {:id id} {column (encrypt-str-fn "{}")}))+ (throw (ex-info (trs "Can''t decrypt app db with MB_ENCRYPTION_SECRET_KEY")+ {:table table, :id id, :column column})))+ (t2/update! :conn conn table {:id id} {column (encrypt-str-fn decrypted)}))))))(defn- do-encryption"Encrypt or decrypts the db using the current `MB_ENCRYPTION_SECRET_KEY` to read data.@@ -40,8 +76,14 @@(let [encrypt-str-fn (make-encrypt-fn encryption/maybe-encrypt)encrypt-bytes-fn (make-encrypt-fn encryption/maybe-encrypt-bytes)](t2/with-transaction [conn {:datasource data-source}]- (doseq [[table column] encrypted-json-columns]- (reencrypt-encrypted-json-column! conn table column encrypt-str-fn))+ (let [check-status (encryption-check-status)]+ (when (= check-status :invalid)+ (throw (ex-info (trs "Database was encrypted with a different key than the MB_ENCRYPTION_SECRET_KEY environment contains")+ {})))+ (doseq [[table column] encrypted-json-columns]+ (reencrypt-encrypted-json-column! conn table column encrypt-str-fn+ (and (= check-status :valid)+ (contains? clearable-when-undecryptable [table column])))))(doseq [[key value] (t2/select-fn->fn :key :value :model/Setting)](case key"settings-last-updated" (let [current-timestamp-as-string-honeysql (h2x/cast (if (= db-type :mysql) :char :text)diff --git a/test/metabase/cmd/rotate_encryption_key_test.clj b/test/metabase/cmd/rotate_encryption_key_test.cljindex 36906b574e1..fad235ce126 100644--- a/test/metabase/cmd/rotate_encryption_key_test.clj+++ b/test/metabase/cmd/rotate_encryption_key_test.clj@@ -167,3 +167,77 @@(is (mt/secret-value-equals? secret-val (t2/select-one-fn :value :model/Secret :id @secret-id-enc))))(testing "short keys fail to rotate"(is (thrown? Throwable (rotate-encryption-key! "short")))))))))))++(defn- set-encryption-check-raw!+ "Set the raw value of the `encryption-check` sentinel setting, replacing any existing row."+ [value]+ (t2/delete! :setting :key "encryption-check")+ (t2/insert! :setting {:key "encryption-check" :value value}))++(defn- insert-user-with-raw-settings!+ "Insert a row directly into the `core_user` table (bypassing model transforms) and return its id."+ [raw-settings]+ (first (t2/insert-returning-pks! :core_user {:email (str (mt/random-name) "@nowhere.test")+ :first_name "No"+ :last_name "Body"+ :password "nopassword"+ :password_salt "nosalt"+ :date_joined :%now+ :is_superuser false+ :is_active true+ :settings raw-settings})))++(defn- insert-channel-with-raw-details!+ "Insert a row directly into the `channel` table (bypassing model transforms) and return its id."+ [raw-details]+ (first (t2/insert-returning-pks! :channel {:name (mt/random-name)+ :type "channel/http"+ :details raw-details+ :active true+ :created_at :%now+ :updated_at :%now})))++(deftest decrypt-db-undecryptable-values-test+ (let [k1 "89ulvIGoiYw6mNELuOoEZphQafnF/zYe+3vT+v70D1A="+ k2 "yHa/6VEQuIItMyd5CNcgV9nXvzZcX6bWmiY0oOh6pLU="+ k2-hashed (encryption/validate-and-hash-secret-key k2)]+ (testing "when encryption-check verifies the key, a clearable column under an unknown key is reset to {} instead of aborting"+ (mt/with-empty-h2-app-db!+ (encryption-test/with-secret-key k1+ (set-encryption-check-raw! (encryption/encrypt (str (random-uuid))))+ (let [good-id (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"en\"}"))+ bad-id (insert-user-with-raw-settings! (encryption/encrypt k2-hashed "{\"locale\":\"fr\"}"))]+ (mdb/decrypt-db :h2 (mdb/data-source))+ (is (= "{\"locale\":\"en\"}" (t2/select-one-fn :settings :core_user :id good-id)))+ (is (= "{}" (t2/select-one-fn :settings :core_user :id bad-id)))+ (is (= "unencrypted" (t2/select-one-fn :value :setting :key "encryption-check")))))))+ (testing "when encryption-check does not decrypt, decryption aborts before touching any rows"+ (mt/with-empty-h2-app-db!+ (let [user-id (encryption-test/with-secret-key k1+ (set-encryption-check-raw! (encryption/encrypt (str (random-uuid))))+ (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"en\"}")))+ raw-settings (t2/select-one-fn :settings :core_user :id user-id)]+ (encryption-test/with-secret-key k2+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Database was encrypted with a different key than the MB_ENCRYPTION_SECRET_KEY environment contains"+ (mdb/decrypt-db :h2 (mdb/data-source)))))+ (is (= raw-settings (t2/select-one-fn :settings :core_user :id user-id))))))+ (testing "without an encryption-check sentinel, an undecryptable value still aborts even in a clearable column"+ (mt/with-empty-h2-app-db!+ (encryption-test/with-secret-key k1+ (t2/delete! :setting :key "encryption-check")+ (insert-user-with-raw-settings! (encryption/encrypt k2-hashed "{\"locale\":\"fr\"}"))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Can't decrypt app db with MB_ENCRYPTION_SECRET_KEY"+ (mdb/decrypt-db :h2 (mdb/data-source)))))))+ (testing "an undecryptable value in a non-clearable column aborts even when the key is verified"+ (mt/with-empty-h2-app-db!+ (encryption-test/with-secret-key k1+ (set-encryption-check-raw! (encryption/encrypt (str (random-uuid))))+ (insert-channel-with-raw-details! (encryption/encrypt k2-hashed "{\"url\":\"http://bad\"}"))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Can't decrypt app db with MB_ENCRYPTION_SECRET_KEY"+ (mdb/decrypt-db :h2 (mdb/data-source)))))))))