Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Store setting values encrypted under the setting's name (#81751)

Alexander Polyankina9a062731bec
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 792ece6acfb..c21d01d18a2 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -349,6 +349,7 @@
:api #{metabase.app-db.cluster-lock ; TODO FIXME
metabase.app-db.core
metabase.app-db.init
+ metabase.app-db.setting
metabase.app-db.setup
metabase.app-db.transient-error}
:uses #{auth-provider
@@ -2235,8 +2236,7 @@
settings
{:team "DevEx"
- :api #{metabase.settings.core
- metabase.settings.init}
+ :api #{metabase.settings.core metabase.settings.init}
:uses #{api
app-db
config
diff --git a/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.clj b/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.clj
index cbbb33cbb69..8297273dc8d 100644
--- a/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.clj
+++ b/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.clj
@@ -22,7 +22,7 @@
expiration (t/instant (t/plus timestamp (t/seconds 300)))
nonce (random-uuid)
payload (str (.getEpochSecond timestamp) "." (.getEpochSecond expiration) "." nonce)
- encrypted (encryption/encrypt (hashed-key) payload)]
+ encrypted (encryption/encrypt payload {:secret-key (hashed-key)})]
(URLEncoder/encode encrypted "UTF-8")))
(defn validate-token
@@ -33,7 +33,7 @@
(not-empty token)
(try
(let [decoded-token (URLDecoder/decode ^String token "UTF-8")
- decrypted-payload (encryption/decrypt (hashed-key) decoded-token)
+ decrypted-payload (encryption/decrypt decoded-token {:secret-key (hashed-key)})
[_ expiration _] (str/split decrypted-payload #"\." 3)]
(t/< (t/instant) (Instant/ofEpochSecond (Long/parseLong expiration))))
(catch Exception _
diff --git a/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.clj b/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.clj
index 3efff0d4798..5ba3cdea521 100644
--- a/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.clj
@@ -5,12 +5,15 @@
[java-time.api :as t]
[metabase-enterprise.sso.integrations.token-utils :as token-utils]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util.encryption :as encryption])
(:import
(java.net URLDecoder URLEncoder)))
(set! *warn-on-reflection* true)
+(use-fixtures :once (fixtures/initialize :db))
+
(deftest generate-token-test
(testing "generate-token"
(testing "should generate a URL-encoded string"
@@ -24,7 +27,7 @@
(mt/with-temporary-setting-values [sdk-encryption-validation-key "1FlZMdousOLX9d3SSL+KuWq2+l1gfKoFM7O4ZHqKjTgabo7QdqP8US2bNPN+PqisP1QOKvesxkxOigIrvvd5OQ=="]
(let [token (token-utils/generate-token)
decoded-token (URLDecoder/decode token "UTF-8")
- decrypted (encryption/decrypt (encryption/secret-key->hash "1FlZMdousOLX9d3SSL+KuWq2+l1gfKoFM7O4ZHqKjTgabo7QdqP8US2bNPN+PqisP1QOKvesxkxOigIrvvd5OQ==") decoded-token)
+ decrypted (encryption/decrypt decoded-token {:secret-key (encryption/secret-key->hash "1FlZMdousOLX9d3SSL+KuWq2+l1gfKoFM7O4ZHqKjTgabo7QdqP8US2bNPN+PqisP1QOKvesxkxOigIrvvd5OQ==")})
[ts exp nonce] (str/split decrypted #"\." 3)
timestamp (Long/parseLong ts)
expiration (Long/parseLong exp)]
@@ -49,7 +52,7 @@
expiration (t/instant (t/plus now (t/seconds 300)))
nonce (random-uuid)
payload (str (.getEpochSecond now) "." (.getEpochSecond expiration) "." nonce)
- encrypted (encryption/encrypt encryption-key payload)
+ encrypted (encryption/encrypt payload {:secret-key encryption-key})
token (URLEncoder/encode encrypted "UTF-8")]
(is (true? (token-utils/validate-token token)))))
(testing "returns false for expired token"
@@ -57,7 +60,7 @@
expiration (t/instant (t/minus now (t/seconds 10))) ;; 10 seconds in the past
nonce (random-uuid)
payload (str (.getEpochSecond now) "." (.getEpochSecond expiration) "." nonce)
- encrypted (encryption/encrypt encryption-key payload)
+ encrypted (encryption/encrypt payload {:secret-key encryption-key})
token (URLEncoder/encode encrypted "UTF-8")]
(is (false? (token-utils/validate-token token)))))
(testing "returns false for nil token"
@@ -71,6 +74,6 @@
expiration (t/instant (t/plus now (t/seconds 300)))
nonce (random-uuid)
payload (str (.getEpochSecond now) "." (.getEpochSecond expiration) "." nonce)
- encrypted (encryption/encrypt encryption-key payload)
+ encrypted (encryption/encrypt payload {:secret-key encryption-key})
token (URLEncoder/encode (str encrypted "tampered") "UTF-8")]
(is (false? (token-utils/validate-token token)))))))))
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index a6713d7df63..8e539237ba2 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -1,5 +1,4 @@
(ns metabase.driver.bigquery-cloud-sdk
- (:refer-clojure :exclude [mapv some empty? not-empty])
(:require
[clojure.core.async :as a]
[clojure.set :as set]
@@ -29,7 +28,7 @@
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [every? mapv some empty? not-empty]]
+ [metabase.util.performance :as perf]
;; sync fix-ups read and write Database/Table/Field rows directly; the metadata provider is read-only
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2])
@@ -98,7 +97,7 @@
chosen by whoever supplied the credentials rather than a fixed Google endpoint."
[service-account-json]
(or (when-not (str/blank? service-account-json)
- (not-empty (get (json/decode service-account-json) "token_uri")))
+ (perf/not-empty (get (json/decode service-account-json) "token_uri")))
default-token-uri))
(defmethod driver/connection-hosts :bigquery-cloud-sdk
@@ -131,7 +130,7 @@
(.setTransportOptions transport-options))]
(when universe-domain
(.setUniverseDomain bq-bldr universe-domain))
- (when-let [host (not-empty (:host details))]
+ (when-let [host (perf/not-empty (:host details))]
(.setHost bq-bldr host))
(.. bq-bldr build getService)))
@@ -170,7 +169,7 @@
[{:keys [dataset-filters-type dataset-filters-patterns]}]
(when (= "inclusion" dataset-filters-type)
(let [segments (map str/trim (str/split (str dataset-filters-patterns) #","))]
- (when (every? #(re-matches exact-dataset-id-re %) segments)
+ (when (perf/every? #(re-matches exact-dataset-id-re %) segments)
;; a scan yields each dataset once however many segments matched it
(distinct segments)))))
@@ -434,7 +433,7 @@
[dataset-id {data-type :data_type field-path-str :field_path table-name :table_name}]
(let [field-path (str/split field-path-str #"\.")
[database-type base-type] (raw-type->database+base-type data-type)]
- (when-let [nfc-path (not-empty (pop field-path))]
+ (when-let [nfc-path (perf/not-empty (pop field-path))]
{:name (peek field-path)
:table-name table-name
:table-schema dataset-id
@@ -748,9 +747,9 @@
lifted by matching the names in `fields` to the names in the table schema."
[^Table bq-table fields rff]
(let [^Schema schema (.. bq-table getDefinition getSchema)
- field-idxs (mapv :database_position fields)
+ field-idxs (perf/mapv :database_position fields)
all-parsers (get-field-parsers schema)
- parsers (mapv all-parsers field-idxs)
+ parsers (perf/mapv all-parsers field-idxs)
probe (list-sample-page bq-table (min (long initial-page-rows) table-rows-sample/max-sample-rows) nil)]
(transduce
(comp (take table-rows-sample/max-sample-rows)
@@ -996,7 +995,7 @@
(dissoc :database-type :database-position)))
cols {:cols columns}
results (eduction (map (fn [^FieldValueList row]
- (mapv parse-field-value row parsers)))
+ (perf/mapv parse-field-value row parsers)))
(reducible-bigquery-results page cancel-chan attempt-job-cancel-fn
(adaptive-query-next-page job)))]
(respond cols results)))
@@ -1204,7 +1203,7 @@
(defmethod driver/normalize-db-details :bigquery-cloud-sdk
[_driver database]
(let [details (driver.conn/default-details database)]
- (when-not (empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))
+ (when-not (perf/empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))
(log/errorf (str "Database ID %d, which was migrated from the legacy :bigquery driver to :bigquery-cloud-sdk, has"
" one or more OAuth style authentication scheme parameters saved to db-details, which cannot"
" be automatically migrated to the newer driver (since it *requires* service-account-json instead);"
@@ -1237,7 +1236,7 @@
(defn- clustering-clause
"Inline `CLUSTER BY col, ...` clause for a table's `indexes`, or nil when there's no clustering."
[indexes]
- (when-let [{:keys [columns]} (some #(when (= :clustering (:kind %)) %) indexes)]
+ (when-let [{:keys [columns]} (perf/some #(when (= :clustering (:kind %)) %) indexes)]
(let [cols (str/join ", " (map #(sql.u/quote-name :bigquery-cloud-sdk :field (:name %)) columns))]
(format "CLUSTER BY %s" cols))))
@@ -1253,7 +1252,7 @@
(let [definition (.getDefinition bq-table)]
(when (instance? StandardTableDefinition definition)
(when-let [^Clustering clustering (.getClustering ^StandardTableDefinition definition)]
- (not-empty (vec (.getFields clustering))))))))))
+ (perf/not-empty (vec (.getFields clustering))))))))))
(defmethod driver/fetch-table-indexes :bigquery-cloud-sdk
[_driver database schema table]
@@ -1370,17 +1369,17 @@
;; rather than waiting many minutes, we trade torward consistency by using SQL DML, whose table metadata
;; is consistent, and we do not see cached non-existence and things like that causing trouble.
(let [database (t2/select-one :model/Database db-id)
- col-kws (mapv (comp keyword name :name) columns)
+ col-kws (perf/mapv (comp keyword name :name) columns)
num-cols (count col-kws)
;; bigquery allows 10k query parameters per request
max-rows (max 1 (quot 10000 num-cols))
chunk-size (min (or driver/*insert-chunk-rows* 1000) max-rows)]
(doseq [chunk (partition-all chunk-size data)
:let [lift #(if (coll? %) [:lift %] %)
- lift-tuple #(mapv lift %)]]
+ lift-tuple #(perf/mapv lift %)]]
(let [[sql & params] (sql.qp/format-honeysql driver {:insert-into table-name
:columns col-kws
- :values (mapv lift-tuple chunk)})]
+ :values (perf/mapv lift-tuple chunk)})]
(driver/execute-raw-queries! driver database [[sql params]])))))
(defmethod driver/execute-raw-queries! :bigquery-cloud-sdk
@@ -1445,7 +1444,7 @@
driver-api/database
driver.conn/effective-details
list-datasets
- (some #{schema}))))
+ (perf/some #{schema}))))
(defmethod driver/table-name-length-limit :bigquery-cloud-sdk
[_driver]
diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yaml
index ded4a7c9ef3..dda4867225e 100644
--- a/resources/migrations/058_update_migrations.yaml
+++ b/resources/migrations/058_update_migrations.yaml
@@ -1660,6 +1660,28 @@ databaseChangeLog:
- customChange:
class: metabase.app_db.custom_migrations.EncryptSetterNoneSettingsV58
+ - changeSet:
+ id: v58.2026-09-01T00:00:00
+ author: ranquild
+ comment: >-
+ Add setting.value_with_aad, which stores a setting's value encrypted under additional authenticated data naming
+ the setting (setting.<key>), so that a value moved between rows fails to decrypt: the bare value in
+ setting.value, or its ciphertext, says nothing about which setting it belongs to, and a direct DB write could
+ give one setting another's value. value is kept: this version writes both columns and reads only
+ value_with_aad, so a version that predates the column keeps working through a rolling upgrade, and a rollback
+ is just dropping value_with_aad again. The column is filled in from value after migrations, by
+ metabase.app-db.setting/migrate-settings!, which also repairs rows such a version has written since.
+ changes:
+ - addColumn:
+ tableName: setting
+ columns:
+ - column:
+ name: value_with_aad
+ type: ${text.type}
+ remarks: "The setting's value, encrypted under additional authenticated data naming the setting"
+ constraints:
+ nullable: true
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index 6ed291214ce..24667337c76 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -73,6 +73,7 @@
spec]
[metabase.app-db.query
compile
+ current-timestamp-string
isa
join
qualify
diff --git a/src/metabase/app_db/encryption.clj b/src/metabase/app_db/encryption.clj
index 46daa382f11..0dcc4045863 100644
--- a/src/metabase/app_db/encryption.clj
+++ b/src/metabase/app_db/encryption.clj
@@ -1,8 +1,9 @@
(ns metabase.app-db.encryption
(:require
+ [metabase.app-db.query :as mdb.query]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.util :as u]
[metabase.util.encryption :as encryption]
- [metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -72,7 +73,10 @@
:invalid - the sentinel exists but does not decrypt (wrong or unset key, corruption)
:absent - no sentinel: a `setting` table that does not exist yet (before migrations on a fresh database), no
row, or the plaintext \"unencrypted\" marker (inserted by the v53 migration, and written back when
- the database is decrypted) -- an explicit statement of the same thing a missing row means"
+ the database is decrypted) -- an explicit statement of the same thing a missing row means
+
+ Read raw, from the legacy `value` column: this runs before migrations, and `value` is the one column every version
+ writes the sentinel to."
[]
(let [raw (u/ignore-exceptions (t2/select-one-fn :value :setting :key encryption-check-key))]
(cond
@@ -133,8 +137,8 @@
This only runs in the one-shot \"no sentinel but content exists\" state, so it can afford to stream every column
fully (stopping at the first value that does not decrypt) rather than sampling; a partially encrypted column can
- therefore never read as `:decryptable`. The `setting` table is not counted: whether a given setting is encrypted at
- rest is decided per setting, so its values prove nothing about the database."
+ therefore never read as `:decryptable`. The `setting` table is not counted: whether a setting's legacy `value` is
+ encrypted at rest is decided per setting, and `value_with_aad` only decrypts under each row's own AAD."
[]
(reduce (fn [acc [table column decryptable?]]
(case (column-content-status table column decryptable?)
@@ -146,21 +150,31 @@
(map #(conj % (comp encryption/decryptable-bytes? maybe-blob->bytes)) encrypted-bytes-columns))))
(defn- replace-encryption-check!
- "Replace the `encryption-check` sentinel on `conn`: with a fresh UUID encrypted by `encrypt-fn`, or with the
- plaintext \"unencrypted\" marker when `encrypt-fn` is nil (the database is being decrypted)."
- [conn encrypt-fn]
- (t2/delete! :conn conn :setting :key encryption-check-key)
- (t2/insert! :conn conn :setting {:key encryption-check-key
- :value (if encrypt-fn
- (encrypt-fn (str (random-uuid)))
- "unencrypted")}))
+ "Replace the `encryption-check` sentinel on `conn`: with a fresh UUID encrypted by `encrypt-fn` -- and, in
+ `value_with_aad`, by `encrypt-setting-fn`, a function of a string and the setting key -- or with the plaintext
+ \"unencrypted\" marker in both columns when they are nil (the database is being decrypted). Written to
+ `value_with_aad` and to the legacy `value` both, so that a version predating `value_with_aad` reads the same answer
+ from the same database."
+ [conn encrypt-fn encrypt-setting-fn]
+ (let [sentinel (if encrypt-fn (str (random-uuid)) "unencrypted")]
+ (t2/delete! :conn conn :setting :key encryption-check-key)
+ (t2/insert! :conn conn :setting {:key encryption-check-key
+ :value (cond-> sentinel encrypt-fn encrypt-fn)
+ :value_with_aad (cond-> sentinel encrypt-setting-fn (encrypt-setting-fn encryption-check-key))})))
+
+(defn- encrypt-setting
+ "A function of a string and a setting key that encrypts the string the way `setting.value_with_aad` holds it: under
+ that setting's AAD, and under `secret-key` when given, the current MB_ENCRYPTION_SECRET_KEY otherwise."
+ [secret-key]
+ (fn [s setting-key]
+ (encryption/maybe-encrypt s {:secret-key secret-key, :aad (mdb.setting/setting-aad setting-key)})))
(defn- write-encryption-check!
"Record that the database is encrypted under the current MB_ENCRYPTION_SECRET_KEY by replacing the `encryption-check`
sentinel with a fresh UUID encrypted under it. Only ever writes the sentinel -- never touches any other row."
[]
(t2/with-transaction [conn]
- (replace-encryption-check! conn encryption/encrypt)))
+ (replace-encryption-check! conn encryption/encrypt (encrypt-setting nil))))
(def ^:private EncryptionState
[:enum :encrypted :unencrypted :fresh :pre-sentinel :missing-key :wrong-key :not-decryptable])
@@ -302,15 +316,24 @@
(when (and (string? value)
(not (encryption/decryptable-string? value)))
(t2/update! :conn conn table {:id id} {column (encryption/encrypt value)})))
- (t2/reducible-select [table :id [column :value]] {:where [:!= column nil]}))))))
+ (t2/reducible-select [table :id [column :value]] {:where [:!= column nil]})))
+ ;; `setting.value_with_aad` is bound to its row, so it is checked and encrypted under each row's own AAD
+ (let [encrypt-setting-fn (encrypt-setting nil)]
+ (run! (fn [{:keys [key value_with_aad]}]
+ (when (and (string? value_with_aad)
+ (not (encryption/decryptable-string? value_with_aad {:aad (mdb.setting/setting-aad key)})))
+ (t2/update! :conn conn :setting {:key key} {:value_with_aad (encrypt-setting-fn value_with_aad key)})))
+ (t2/reducible-select [:setting :key :value_with_aad] {:where [:!= :value_with_aad nil]}))))))
(defn- do-encryption
- "Encrypt or decrypts the db using the current `MB_ENCRYPTION_SECRET_KEY` to read data.
-
- The passed make-encrypt-fn is used to generate the encryption/decryption function to use by passing versions of encryption/maybe-encrypt to it."
- [db-type data-source encrypting? make-encrypt-fn]
- (let [encrypt-str-fn (make-encrypt-fn encryption/maybe-encrypt)
- encrypt-bytes-fn (make-encrypt-fn encryption/maybe-encrypt-bytes)]
+ "Encrypt or decrypt the db using the current `MB_ENCRYPTION_SECRET_KEY` to read data.
+
+ When `encrypting?`, every value is re-encrypted under `to-key` (already hashed), or under the current
+ MB_ENCRYPTION_SECRET_KEY when that is nil; otherwise every value is written back decrypted."
+ [db-type data-source encrypting? to-key]
+ (let [encrypt-str-fn (if encrypting? #(encryption/maybe-encrypt % {:secret-key to-key}) identity)
+ encrypt-bytes-fn (if encrypting? #(encryption/maybe-encrypt-bytes % {:secret-key to-key}) identity)
+ encrypt-setting-fn (if encrypting? (encrypt-setting to-key) (fn [s _setting-key] s))]
(t2/with-transaction [conn {:datasource data-source}]
(let [check-status (encryption-check-status)]
(when (= check-status :invalid)
@@ -324,16 +347,23 @@
;; a setting that is plaintext at rest while a key is configured (e.g. one newly designated encrypted but not yet
;; re-encrypted) is exactly what this operation exists to fix, so we decrypt it leniently here rather than reject
;; it. A value that looks encrypted but can't be decrypted with the current key still aborts.
- (doseq [[key value] (t2/select-fn->fn :key :value :setting)]
+ ;;
+ ;; Both columns are re-encrypted: `value_with_aad` under each row's own AAD, and `value` because a version
+ ;; predating that column reads it, so a rollback must not land on ciphertext under the old key.
+ (doseq [{:keys [key value value_with_aad]} (t2/select :setting)]
(case key
- "settings-last-updated" (let [current-timestamp-as-string-honeysql (h2x/cast (if (= db-type :mysql) :char :text)
- (h2x/current-datetime-honeysql-form db-type))]
- (t2/update! :conn conn :setting {:key key} {:value current-timestamp-as-string-honeysql}))
+ "settings-last-updated" (let [now (mdb.query/current-timestamp-string db-type)]
+ (t2/update! :conn conn :setting {:key key}
+ {:value now
+ :value_with_aad (encrypt-setting-fn now key)}))
"encryption-check" nil
- (t2/update! :conn conn :setting
- {:key key}
- {:value (encrypt-str-fn (encryption/maybe-decrypt-accepting-plaintext value))})))
- (replace-encryption-check! conn (when encrypting? encrypt-str-fn))
+ (let [aad-opts {:aad (mdb.setting/setting-aad key)}
+ changes (cond-> {}
+ (seq value) (assoc :value (encrypt-str-fn (encryption/maybe-decrypt-accepting-plaintext value)))
+ (seq value_with_aad) (assoc :value_with_aad (encrypt-setting-fn (encryption/maybe-decrypt-accepting-plaintext value_with_aad aad-opts) key)))]
+ (when (seq changes)
+ (t2/update! :conn conn :setting {:key key} changes)))))
+ (replace-encryption-check! conn (when encrypting? encrypt-str-fn) (when encrypting? encrypt-setting-fn))
(doseq [[table column] encrypted-bytes-columns]
(reencrypt-encrypted-bytes-column! conn table column encrypt-bytes-fn))
(t2/delete! :conn conn :model/QueryCache))))
@@ -346,12 +376,9 @@
(throw (ex-info "Cannot encrypt database with an empty key" {})))
(when (and (nil? to-key) (not (encryption/default-encryption-enabled?)))
(throw (ex-info "Cannot encrypt database: MB_ENCRYPTION_SECRET_KEY is not set" {})))
- (do-encryption db-type data-source true (fn [maybe-encrypt-fn]
- (if
- (nil? to-key) maybe-encrypt-fn
- (partial maybe-encrypt-fn (encryption/validate-and-hash-secret-key to-key))))))
+ (do-encryption db-type data-source true (some-> to-key encryption/validate-and-hash-secret-key)))
(defn decrypt-db
"Decrypts the database using the current `MB_ENCRYPTION_SECRET_KEY` to read existing data"
[db-type data-source]
- (do-encryption db-type data-source false (constantly identity)))
+ (do-encryption db-type data-source false nil))
diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.clj
index 70c019adfe8..5580cdef999 100644
--- a/src/metabase/app_db/query.clj
+++ b/src/metabase/app_db/query.clj
@@ -24,6 +24,7 @@
[honey.sql :as sql]
[metabase.app-db.format :as app-db.format]
[metabase.util :as u]
+ [metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]
@@ -116,6 +117,14 @@
e))))]
sql-args))
+(defn current-timestamp-string
+ "The application DB's own current timestamp, as a string. Read from the DB rather than from this machine's clock,
+ for the `settings-last-updated` marker that instances compare against each other."
+ ^String [db-type]
+ ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)
+ (let [cast-form (h2x/cast (if (= db-type :mysql) :char :text) (h2x/current-datetime-honeysql-form db-type))]
+ (-> (t2/query-one {:select [[cast-form :timestamp]]}) :timestamp)))
+
;;; TODO -- we should mark this deprecated and tell people to use [[toucan2.core/query]] directly instead
(defn query
"Replacement for [[toucan.db/query]] -- uses Honey SQL 2 instead of Honey SQL 1, to ease the transition to the
diff --git a/src/metabase/app_db/setting.clj b/src/metabase/app_db/setting.clj
new file mode 100644
index 00000000000..13016abdf2c
--- /dev/null
+++ b/src/metabase/app_db/setting.clj
@@ -0,0 +1,50 @@
+(ns metabase.app-db.setting
+ "How the application DB stores a setting's value: `setting.value_with_aad` holds it encrypted under additional
+ authenticated data naming the setting, so that a value moved between rows -- whose ciphertext alone says nothing
+ about which setting it belongs to -- fails to decrypt. The legacy `value` column holds it bare, for versions that
+ predate the other. Lives here rather than with the Setting model because the encryption tooling here writes two rows
+ the model never does: the `settings-last-updated` marker and the `encryption-check` sentinel."
+ (:require
+ [buddy.core.codecs :as codecs]
+ [metabase.util :as u]
+ [metabase.util.encryption :as encryption]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^{:arglists '(^bytes [setting-key])} setting-aad
+ "The additional authenticated data a setting's value is encrypted under: `setting.<key>`."
+ (memoize (fn ^bytes [setting-key] (codecs/to-bytes (str "setting." setting-key)))))
+
+(defn migrate-settings!
+ "Give every setting row that has no `value_with_aad` one, from the legacy `value` column beside it, stored the way
+ the Setting model stores one. Runs from `metabase.app-db.setup/setup-db!` after migrations and before anything reads
+ a setting, whenever the database's encryption state says every row can be read -- a row this cannot decrypt would
+ otherwise be stored as if it were a value.
+
+ Only an empty `value_with_aad` is filled in: a row a version predating the column wrote, on the first start after
+ the upgrade or while such a version runs alongside this one. A row that has one is left exactly as it is, whatever
+ `value` says beside it -- nothing in this version reads `value`, and a version that does reconciles it at its own
+ startup.
+
+ Reads and writes the table directly, never through the model: the model's read is the strict one this fills in
+ for, and the cloud-migration guard on Toucan DML reads `read-only-mode` through it before every update -- a row that
+ may itself be among those being filled in."
+ []
+ (let [filled (atom 0)]
+ (t2/with-transaction [_conn]
+ (doseq [{:keys [key value]} (t2/select :setting
+ {:where [:and
+ [:= :value_with_aad nil]
+ [:not= :value nil]
+ [:not= :value ""]]
+ :for :update})
+ :let [plain (u/ignore-exceptions (encryption/maybe-decrypt-accepting-plaintext value))]
+ :when (some? plain)]
+ (swap! filled inc)
+ (t2/query {:update :setting
+ :set {:value_with_aad (encryption/maybe-encrypt plain {:aad (setting-aad key)})}
+ :where [:= :key key]})))
+ (when (pos? @filled)
+ (log/infof "Filled in the authenticated value of %d setting(s) from their value." @filled))))
diff --git a/src/metabase/app_db/setup.clj b/src/metabase/app_db/setup.clj
index f517d80cc8a..219a17d3abb 100644
--- a/src/metabase/app_db/setup.clj
+++ b/src/metabase/app_db/setup.clj
@@ -17,6 +17,7 @@
[metabase.app-db.encryption :as mdb.encryption]
[metabase.app-db.jdbc-protocols :as mdb.jdbc-protocols]
[metabase.app-db.liquibase :as liquibase]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.config.core :as config]
[metabase.util :as u]
[metabase.util.honey-sql-2]
@@ -241,6 +242,17 @@
(migrate! data-source (if auto-migrate? :up :print))
(log/info "Database Migrations Current ..." (u/emoji "✅")))
+(defn- migrate-settings!
+ "Run [[mdb.setting/migrate-settings!]] once the migrations have, in an encryption state where every row of the
+ database can be read: a caller that skips [[mdb.encryption/check-encryption]] (`enable-encryption`, `copy!`) can get
+ here with rows the key in hand cannot decrypt, and those are left for it to sort out. Here rather than at
+ application startup because every entry point that migrates the app DB comes through [[setup-db!]], and several
+ never reach `metabase.core.core/init!` -- `migrate up`, the serialization commands, `reset-password`, the
+ encryption commands -- and a JVM that skipped the repair would read every setting as nil."
+ [db-state]
+ (when (#{:encrypted :unencrypted :fresh :pre-sentinel} db-state)
+ (mdb.setting/migrate-settings!)))
+
;; TODO -- consider renaming to something like `verify-connection-and-migrate!`
(mu/defn setup-db!
"Connects to db and runs migrations. Don't use this directly, unless you know what you're doing;
@@ -276,6 +288,7 @@
(when manage-encryption-state?
(mdb.encryption/check-encryption db-state))
(run-schema-migrations! data-source auto-migrate?)
+ (migrate-settings! db-state)
(when manage-encryption-state?
(mdb.encryption/record-encryption-state! db-state))))))
:done))
diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.clj
index 644f9d4261a..d0afedefc6e 100644
--- a/src/metabase/core/core.clj
+++ b/src/metabase/core/core.clj
@@ -202,8 +202,6 @@
;; and the test suite can take 2x longer. this is really unfortunate because it could lead to some false
;; negatives, but for now there's not much we can do
(mdb/setup-db! :create-sample-content? (not config/is-test?))
- ;; runs before anything reads settings -- see its docstring
- (setting/migrate-encrypted-settings!)
(mdb/encrypt-plaintext-columns!)
;; In OSS, convert any Data Analysts group with members to a normal visible group
(perms/sync-data-analyst-group-for-oss!)
diff --git a/src/metabase/settings/core.clj b/src/metabase/settings/core.clj
index 629bb31b92b..1a2233c0936 100644
--- a/src/metabase/settings/core.clj
+++ b/src/metabase/settings/core.clj
@@ -104,7 +104,6 @@
log-deprecated-env-var-usage!
get-value-of-type
has-advanced-setting-access?
- migrate-encrypted-settings!
obfuscate-value
obfuscated-value?
read-setting
diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.clj
index 47ceeeb0191..8d88301fd4c 100644
--- a/src/metabase/settings/models/setting.clj
+++ b/src/metabase/settings/models/setting.clj
@@ -10,6 +10,7 @@
[malli.core :as mc]
[medley.core :as m]
[metabase.api.common :as api]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.config.core :as config]
[metabase.events.core :as events]
[metabase.models.serialization :as serdes]
@@ -263,6 +264,12 @@
registered-settings
(atom {}))
+(defonce ^:private ^{:doc "Map of `:deprecated-name` (as a string) -> the name of the setting that declares it. A row
+ stored under a name a setting used to have is still that setting's, and [[db-or-cache-value]] reads it as a
+ fallback, so it has to resolve to the setting like the current name does."}
+ settings-by-deprecated-name
+ (atom {}))
+
(defprotocol ^:private Resolvable
(resolve-setting [setting-definition-or-name]
"Resolve the definition map for a Setting. `setting-definition-or-name` map be a map, keyword, or string."))
@@ -297,12 +304,13 @@
;; references for now
(defmethod setting.cache/call-on-change :default
[old new]
- (let [rs @registered-settings
- [d1 d2] (data/diff old new)]
- (doseq [changed-setting (into (set (keys d1))
- (set (keys d2)))]
- (when-let [on-change (get-in rs [(keyword changed-setting) :on-change])]
- (on-change (core/get old changed-setting) (core/get new changed-setting))))))
+ (when (some? new)
+ (let [rs @registered-settings
+ [d1 d2] (data/diff old new)]
+ (doseq [changed-setting (into (set (keys d1))
+ (set (keys d2)))]
+ (when-let [on-change (get-in rs [(keyword changed-setting) :on-change])]
+ (on-change (core/get old changed-setting) (core/get new changed-setting)))))))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | get |
@@ -341,12 +349,14 @@
(defn- maybe-resolve-setting
"Like [[resolve-setting]] but returns nil for a setting with no code definition (e.g. one written straight to the DB
- in a test) instead of throwing."
+ in a test) instead of throwing. A name a setting used to go by resolves to that setting."
[setting-or-name]
(try (resolve-setting setting-or-name)
(catch clojure.lang.ExceptionInfo e
(when-not (::unknown-setting-error (ex-data e))
- (throw e)))))
+ (throw e))
+ (when-let [current-name (@settings-by-deprecated-name (name setting-or-name))]
+ (@registered-settings current-name)))))
(defn- encrypts? [setting-or-name]
(not= :no (:encryption (resolve-setting setting-or-name))))
@@ -1185,6 +1195,8 @@
(throw (ex-info (tru "Setting {0} uses :enabled-for-db?, but is not limited to only database-local values"
setting-name)
{:setting setting})))
+ (when-let [deprecated-name (:deprecated-name setting)]
+ (swap! settings-by-deprecated-name assoc (name deprecated-name) setting-name))
(swap! registered-settings assoc setting-name <>))))
;;; +----------------------------------------------------------------------------------------------------------------+
@@ -1735,85 +1747,58 @@
(name (:name invalid-setting))
(ex-message (:parse-error invalid-setting))))))
-(defn migrate-encrypted-settings!
- "Reconcile the at-rest encryption of every registered setting's stored value with its declared `:encryption`, in
- both directions: a `:encryption :no` setting whose row is encrypted is decrypted, and a setting that encrypts whose
- row is plaintext is encrypted. Rows of unregistered settings are left alone, and whether a value is encrypted is
- decided by [[encryption/decryptable-string?]] (actually decrypting), never by shape.
-
- Runs on every startup, before anything restores the settings cache. The encrypting half is what makes the strict
- decrypting read survive a downgrade: a boot of an older version decrypts (in this very function, as it existed
- there) or re-writes as plaintext every row its registry knew as `:encryption :no`, after the one-shot encryption
- migrations have already run -- and such a row would otherwise fail the strict read and take the whole settings
- cache down with it. Works around the standard getters/setters deliberately: values are rewritten byte-identical
- modulo encryption. No-op when MB_ENCRYPTION_SECRET_KEY is not set.
-
- Runs with the settings cache disabled: any setting consulted while this runs (e.g. `read-only-mode`, which the
- cloud-migration DML guard reads on every write this issues) is read directly from the DB. Restoring the cache
- strictly decrypts every row -- including the very rows this function exists to repair -- so going through it here
- would fail the repair on exactly the state it is repairing."
- []
- (when (encryption/default-encryption-enabled?)
- (binding [config/*disable-setting-cache* true]
- (let [{encrypting true, plaintext false} (group-by (comp boolean encrypts?) (vals @registered-settings))]
- (t2/with-transaction [_conn]
- (doseq [{v :value k :key}
- (t2/select :setting {:for :update :where [:and
- [:in :key (map setting-name plaintext)]
- ;; these are *definitely* decrypted already, let's not bother looking
- [:not [:in :value ["true" "false"]]]]})
- :when (encryption/decryptable-string? v)]
- (t2/update! :setting :key k {:value (encryption/decrypt v)}))
- (doseq [{v :value k :key}
- (t2/select :setting {:for :update :where [:and
- [:in :key (map setting-name encrypting)]
- [:!= :value nil]]})
- :when (not (encryption/decryptable-string? v))]
- (t2/update! :setting :key k {:value (encryption/encrypt v)})))))))
-
-(defn- maybe-encrypt [setting-model]
- ;; In tests, sometimes we need to insert/update settings that don't have definitions in the code and therefore can't
- ;; be resolved. Fall back to maybe-encrypting these.
- ;; Don't do any automatic handling of the "encryption-check" special setting used by mdb.encryption
- (if (= "encryption-check" (:key setting-model))
- setting-model
- (let [resolved (maybe-resolve-setting (:key setting-model))]
- (cond-> setting-model
- (or (nil? resolved)
- (encrypts? resolved))
- (update :value encryption/maybe-encrypt)))))
+(defn- write-setting-value
+ "Store a Setting's `:value` in `:value_with_aad`, encrypted under additional authenticated data naming the setting
+ (see [[mdb.setting/setting-aad]]) whenever MB_ENCRYPTION_SECRET_KEY is set -- every setting's, whatever its
+ `:encryption` says. That flag describes the legacy `value` column, which is written here too, exactly as it was
+ before `value_with_aad` existed: nothing in this version reads it, but a version that predates the column does, and
+ keeping it current is what lets that version run alongside this one and what makes rolling back to it lossless.
+
+ A key with no `defsetting` is refused: [[read-setting-value]] would read the row back as no value at all, so there is
+ no way to write one that means anything."
+ [setting]
+ (let [setting-key (:key setting)
+ value (:value setting)
+ resolved (or (maybe-resolve-setting setting-key)
+ (throw (ex-info (tru "Unknown setting: {0}" setting-key)
+ {:setting-key setting-key})))]
+ (assoc setting
+ :value (cond-> value (encrypts? resolved) encryption/maybe-encrypt)
+ :value_with_aad (some-> value (encryption/maybe-encrypt {:aad (mdb.setting/setting-aad setting-key)})))))
+
+(defn- read-setting-value
+ "Take a Setting's `:value` from the `:value_with_aad` it is stored in: decrypted under the additional authenticated
+ data naming this setting, strictly, with [[encryption/maybe-decrypt]]. With MB_ENCRYPTION_SECRET_KEY set that column
+ is ciphertext for every setting, so a plaintext value -- forged via a direct DB write, or left by a row that has never
+ been through `enable-encryption` -- is rejected rather than trusted, and so is a ciphertext moved here from another
+ setting's row, since it was authenticated under that setting's name.
+
+ Two rows read as no value at all. One with no `value_with_aad` is a row only a version predating the column has ever
+ written. One whose key has no `defsetting` -- a retired setting, one belonging to an edition this instance is not
+ running, or a row written straight to the DB in a test -- is not read at all, not even decrypted: nothing can ask
+ for such a setting by name, so there is no value to produce and no reason to touch what is stored there."
+ [setting]
+ (let [setting-key (:key setting)]
+ (if (maybe-resolve-setting setting-key)
+ (try
+ (assoc setting :value (some-> (:value_with_aad setting)
+ (encryption/maybe-decrypt {:aad (mdb.setting/setting-aad setting-key)})))
+ (catch Throwable e
+ (throw (ex-info (format "Error reading setting \"%s\": %s" setting-key (ex-message e))
+ {:setting-key setting-key}
+ e))))
+ (assoc setting :value nil))))
(t2/define-before-update :model/Setting
[setting]
- (maybe-encrypt setting))
+ (write-setting-value setting))
(t2/define-before-insert :model/Setting
[setting]
- (maybe-encrypt setting))
-
-(defn- decrypt-setting-value-on-read
- "Decrypt a Setting's `:value` on read. A setting whose `:encryption` is not `:no` is stored encrypted at rest, so it
- is read strictly with [[encryption/maybe-decrypt]]: a plaintext value — forged via a direct DB write, or a legacy row
- from before the setting became encrypted — is rejected rather than trusted. A `:no` setting (or one with no code
- definition, e.g. in tests) is intentionally plaintext, so it is read leniently with
- [[encryption/maybe-decrypt-accepting-plaintext]], which returns a plaintext value unchanged."
- [setting]
- (let [resolved (maybe-resolve-setting (:key setting))
- decrypt (if (or (nil? resolved) (not (encrypts? resolved)))
- encryption/maybe-decrypt-accepting-plaintext
- encryption/maybe-decrypt)]
- (try
- (update setting :value decrypt)
- (catch Throwable e
- (throw (ex-info (format "Error decrypting setting \"%s\": %s" (:key setting) (ex-message e))
- {:setting-key (:key setting)}
- e))))))
+ (write-setting-value setting))
(t2/define-after-select :model/Setting
[setting]
- ;; Skip aggregate results (e.g. a `count` row) that carry no `:key` to resolve or `:value` to decrypt, and don't do
- ;; any automatic handling of the "encryption-check" special setting used by mdb.encryption.
- (if (or (nil? (:key setting))
- (= "encryption-check" (:key setting)))
- setting
- (decrypt-setting-value-on-read setting)))
+ ;; Skip aggregate results (e.g. a `count` row) that carry no `:key` to resolve the setting by.
+ (cond-> setting
+ (some? (:key setting)) read-setting-value))
diff --git a/src/metabase/settings/models/setting/cache.clj b/src/metabase/settings/models/setting/cache.clj
index 778afbfa3ff..85fb9922dd9 100644
--- a/src/metabase/settings/models/setting/cache.clj
+++ b/src/metabase/settings/models/setting/cache.clj
@@ -5,8 +5,9 @@
[clojure.core :as core]
[clojure.string :as str]
[metabase.app-db.core :as mdb]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.util :as u]
- [metabase.util.honey-sql-2 :as h2x]
+ [metabase.util.encryption :as encryption]
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
@@ -72,18 +73,19 @@
"Update the value of `settings-last-updated` in the DB; if the row does not exist, insert one."
[]
(log/debug "Updating value of settings-last-updated in DB...")
- ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)
- (let [current-timestamp-as-string-honeysql (h2x/cast (if (= (mdb/db-type) :mysql) :char :text)
- (h2x/current-datetime-honeysql-form (mdb/db-type)))]
+ ;; Written raw, not through `:model/Setting`, so that `value` gets the plaintext timestamp a version predating
+ ;; `value_with_aad` compares in SQL. `value_with_aad` is encrypted under the marker's AAD like any other setting's.
+ (let [value (mdb/current-timestamp-string (mdb/db-type))
+ value-with-aad (encryption/maybe-encrypt value {:aad (mdb.setting/setting-aad settings-last-updated-key)})]
;; attempt to UPDATE the existing row. If no row exists, `t2/update!` will return 0...
- (or (pos? (t2/update! :setting {:key settings-last-updated-key} {:value current-timestamp-as-string-honeysql}))
+ (or (pos? (t2/update! :setting {:key settings-last-updated-key} {:value value, :value_with_aad value-with-aad}))
;; ...at which point we will try to INSERT a new row. Note that it is entirely possible two instances can both
;; try to INSERT it at the same time; one instance would fail because it would violate the PK constraint on
;; `key`, and throw a SQLException. As long as one instance updates the value, we are fine, so we can go ahead
;; and ignore that Exception if one is thrown.
(try
- ;; Use `simple-insert!` because we do *not* want to trigger pre-insert behavior, such as encrypting `:value`
- (t2/insert! (t2/table-name (t2/resolve-model :model/Setting)) :key settings-last-updated-key, :value current-timestamp-as-string-honeysql)
+ (t2/insert! (t2/table-name (t2/resolve-model :model/Setting))
+ :key settings-last-updated-key, :value value, :value_with_aad value-with-aad)
(catch java.sql.SQLException e
;; go ahead and log the whole SQLException message chain anyway on the off chance that it *wasn't* just a
;; race condition issue
@@ -118,10 +120,10 @@
(when-let [last-known-update (cache-last-updated-at)]
;; compare it to the value in the DB. This is done be seeing whether a row exists
;; WHERE value > <local-value>
- (u/prog1 (t2/select-one-fn :value :model/Setting
- {:where [:and
- [:= :key settings-last-updated-key]
- [:> :value last-known-update]]})
+ ;; compared here rather than in SQL: what is stored is ciphertext, so only the decrypted timestamps can be ordered
+ (u/prog1 (when-let [db-value (t2/select-one-fn :value :model/Setting :key settings-last-updated-key)]
+ (when (pos? (compare db-value last-known-update))
+ db-value))
(log/trace "last known Settings update: " (pr-str last-known-update))
(log/trace "actual last Settings update:" (pr-str <>))
(when <>
diff --git a/src/metabase/settings/settings.clj b/src/metabase/settings/settings.clj
index 4aa09bbfc84..46a9a769f6e 100644
--- a/src/metabase/settings/settings.clj
+++ b/src/metabase/settings/settings.clj
@@ -1,8 +1,24 @@
(ns metabase.settings.settings
(:require
[metabase.config.core :as config]
+ [metabase.settings.models.setting :refer [defsetting]]
[metabase.util.malli :as mu]))
+(defsetting settings-last-updated
+ "When any Setting was last changed, as a timestamp string the application DB produced. Instances compare it against
+ their own copy to notice that another instance has changed a Setting and their cache is stale.
+
+ Written by [[metabase.settings.models.setting.cache/update-settings-last-updated!]] rather than through this
+ Setting, which is why it has no setter: its value has to come from the application DB's clock, not from any one
+ instance's. It is declared here so that it is a registered Setting like any other, and so its row is read like any
+ other."
+ :visibility :internal
+ :type :string
+ :encryption :no
+ :setter :none
+ :audit :never
+ :export? false)
+
(mu/defn application-name-for-setting-descriptions
"Returns the value of the [[application-name]] setting so setting docstrings can be generated during the compilation
stage. Use this instead of `application-name` in descriptions, otherwise the `application-name` setting's
diff --git a/src/metabase/util/encryption.clj b/src/metabase/util/encryption.clj
index 42eaab52bac..27979295c30 100644
--- a/src/metabase/util/encryption.clj
+++ b/src/metabase/util/encryption.clj
@@ -74,55 +74,68 @@
"\n"
"For more information, see https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html"))
+(defn- opts->secret-key
+ "The 64-byte key `opts` name, or the hashed MB_ENCRYPTION_SECRET_KEY when they name none."
+ ^bytes [{:keys [^bytes secret-key]}]
+ (or secret-key default-secret-key))
+
+(defn- cipher-opts
+ "Buddy's options for the AEAD scheme, with the `:aad` from `opts`, if any. Additional authenticated data is covered
+ by the ciphertext's MAC without being encrypted: a value encrypted under some AAD only decrypts under the same AAD,
+ so binding a value to the row it belongs to is a matter of naming the row here."
+ [{:keys [^bytes aad]}]
+ (cond-> {:algorithm :aes256-cbc-hmac-sha512}
+ aad (assoc :aad aad)))
+
+;;; Every function below takes an optional trailing `opts` map, with `:secret-key` (a 64-byte hashed key, as
+;;; [[secret-key->hash]] makes one; by default the hashed MB_ENCRYPTION_SECRET_KEY) and, for the string and byte-array
+;;; functions, `:aad` (bytes the ciphertext is bound to -- see [[cipher-opts]]). `:secret-key` is what lets tests pass
+;;; a key directly instead of using `with-redefs`, and what key rotation uses to write under the key it rotates to.
+
(defn encrypt-bytes
- "Encrypt bytes `b` using a `secret-key` (a 64-byte byte array), by default is the hashed value of
- `MB_ENCRYPTION_SECRET_KEY`."
+ "Encrypt bytes `b`."
{:added "0.41.0"}
- (^String [^bytes b]
- (encrypt-bytes default-secret-key b))
- (^String [^String secret-key, ^bytes b]
+ (^bytes [^bytes b]
+ (encrypt-bytes b nil))
+ (^bytes [^bytes b opts]
(let [initialization-vector (nonce/random-bytes 16)]
- (->> (crypto/encrypt b
- secret-key
- initialization-vector
- {:algorithm :aes256-cbc-hmac-sha512})
+ (->> (crypto/encrypt b (opts->secret-key opts) initialization-vector (cipher-opts opts))
(concat initialization-vector)
byte-array))))
(defn encrypt
- "Encrypt string `s` as hex bytes using a `secret-key` (a 64-byte byte array), which by default is the hashed value of
- `MB_ENCRYPTION_SECRET_KEY`."
+ "Encrypt string `s`, returning the ciphertext as base64."
(^String [^String s]
- (encrypt default-secret-key s))
- (^String [^String secret-key, ^String s]
- (->> (codecs/to-bytes s)
- (encrypt-bytes secret-key)
- codec/base64-encode)))
+ (encrypt s nil))
+ (^String [^String s opts]
+ (-> (codecs/to-bytes s)
+ (encrypt-bytes opts)
+ codec/base64-encode)))
(defn decrypt-bytes
- "Decrypt bytes `b` using a `secret-key` (a 64-byte byte array), which by default is the hashed value of
- `MB_ENCRYPTION_SECRET_KEY`."
+ "Decrypt bytes `b`."
{:added "0.41.0"}
- (^String [^bytes b]
- (decrypt-bytes default-secret-key b))
- (^String [secret-key, ^bytes b]
+ (^bytes [^bytes b]
+ (decrypt-bytes b nil))
+ (^bytes [^bytes b opts]
(let [[initialization-vector message] (split-at 16 b)]
(crypto/decrypt (byte-array message)
- secret-key
+ (opts->secret-key opts)
(byte-array initialization-vector)
- {:algorithm :aes256-cbc-hmac-sha512}))))
+ (cipher-opts opts)))))
(defn encrypt-stream
"Wraps a plaintext input stream into an input stream that encrypts it using AES256 CBC.
- The encryption format is slightly different for streams vs. fixed length data"
+ The encryption format is slightly different for streams vs. fixed length data, and carries no AAD."
{:added "0.53.0"}
(^InputStream [^InputStream input-stream]
- (encrypt-stream default-secret-key input-stream))
- (^InputStream [secret-key ^InputStream input-stream]
- (let [spec aes-streaming-spec
+ (encrypt-stream input-stream nil))
+ (^InputStream [^InputStream input-stream opts]
+ (let [secret-key (opts->secret-key opts)
+ spec aes-streaming-spec
spec-header (codecs/to-bytes (format "%-32s" spec))
- cipher (Cipher/getInstance spec)
- iv (nonce/random-bytes 16)]
+ cipher (Cipher/getInstance spec)
+ iv (nonce/random-bytes 16)]
(.init cipher Cipher/ENCRYPT_MODE (SecretKeySpec. (bytes/slice secret-key 32 64) "AES") (IvParameterSpec. iv))
(SequenceInputStream. (ByteArrayInputStream. (bytes/concat spec-header iv)) (CipherInputStream. input-stream cipher)))))
@@ -130,28 +143,29 @@
"Encrypts a byte-array in a way that can be used to read it with decrypt-stream instead of decrypt."
{:added "0.53.0"}
(^bytes [^bytes input]
- (encrypt-for-stream default-secret-key input))
- (^bytes [secret-key ^bytes input]
- (with-open [encrypted (encrypt-stream secret-key (ByteArrayInputStream. input))]
+ (encrypt-for-stream input nil))
+ (^bytes [^bytes input opts]
+ (with-open [encrypted (encrypt-stream (ByteArrayInputStream. input) opts)]
(.readAllBytes encrypted))))
(defn maybe-decrypt-stream
"Wraps a possibly-encrypted input stream into a new input stream that decrypts it if necessary."
{:added "0.53.0"}
(^InputStream [^InputStream input-stream]
- (maybe-decrypt-stream default-secret-key input-stream))
- (^InputStream [secret-key ^InputStream input-stream]
- (let [spec-array (byte-array 32)
+ (maybe-decrypt-stream input-stream nil))
+ (^InputStream [^InputStream input-stream opts]
+ (let [secret-key (opts->secret-key opts)
+ spec-array (byte-array 32)
spec-array-length (.read input-stream spec-array)
- spec (str/trim (codecs/bytes->str spec-array))]
+ spec (str/trim (codecs/bytes->str spec-array))]
(cond
(= spec-array-length -1)
input-stream
(and (= spec-array-length 32) (= spec aes-streaming-spec))
(let [cipher (Cipher/getInstance spec)
- iv (byte-array 16)
- _ (.read input-stream iv)]
+ iv (byte-array 16)
+ _ (.read input-stream iv)]
(.init cipher Cipher/DECRYPT_MODE (SecretKeySpec. (bytes/slice secret-key 32 64) "AES") (IvParameterSpec. iv))
(CipherInputStream. input-stream cipher))
@@ -161,42 +175,42 @@
input-stream)))))
(defn decrypt
- "Decrypt string `s` using a `secret-key` (a 64-byte byte array), by default the hashed value of
- `MB_ENCRYPTION_SECRET_KEY`."
+ "Decrypt base64 ciphertext `s` back to the string it was made from."
(^String [^String s]
- (decrypt default-secret-key s))
- (^String [secret-key, ^String s]
- (codecs/bytes->str (decrypt-bytes secret-key (codec/base64-decode s)))))
+ (decrypt s nil))
+ (^String [^String s opts]
+ (codecs/bytes->str (decrypt-bytes (codec/base64-decode s) opts))))
(defn maybe-encrypt
- "If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of `s`; otherwise return `s` as-is."
+ "If there is a key (MB_ENCRYPTION_SECRET_KEY, or `:secret-key` in `opts`), return an encrypted version of `s`;
+ otherwise return `s` as-is."
(^String [^String s]
- (maybe-encrypt default-secret-key s))
- (^String [secret-key, ^String s]
- (if secret-key
+ (maybe-encrypt s nil))
+ (^String [^String s opts]
+ (if (opts->secret-key opts)
(when (seq s)
- (encrypt secret-key s))
+ (encrypt s opts))
s)))
(defn maybe-encrypt-bytes
- "If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of the given bytes `b`; otherwise return `b`
- as-is."
+ "If there is a key, return an encrypted version of the given bytes `b`; otherwise return `b` as-is."
{:added "0.41.0"}
(^bytes [^bytes b]
- (maybe-encrypt-bytes default-secret-key b))
- (^bytes [secret-key, ^bytes b]
- (if secret-key
+ (maybe-encrypt-bytes b nil))
+ (^bytes [^bytes b opts]
+ (if (opts->secret-key opts)
(when (seq b)
- (encrypt-bytes secret-key b))
+ (encrypt-bytes b opts))
b)))
(defn maybe-encrypt-for-stream
- "If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of `s` that can be used to stream the data; otherwise return `s` as-is."
+ "If there is a key, return an encrypted version of `s` that can be used to stream the data; otherwise return `s`
+ as-is."
(^bytes [^bytes s]
- (maybe-encrypt-for-stream default-secret-key s))
- (^bytes [secret-key, ^bytes s]
- (if secret-key
- (encrypt-for-stream secret-key s)
+ (maybe-encrypt-for-stream s nil))
+ (^bytes [^bytes s opts]
+ (if (opts->secret-key opts)
+ (encrypt-for-stream s opts)
s)))
(def ^:private ^:const aes256-tag-length 32)
@@ -238,81 +252,76 @@
(possibly-encrypted-bytes? b)))))
(defn decryptable-bytes?
- "Whether `b` is encrypted with `secret-key`: it has the shape of ciphertext *and* decrypts. The ciphertext is
- authenticated, so a wrong key, corruption, and plaintext that merely looks like ciphertext all yield false. Always
- false when no key is set. Use this, not [[possibly-encrypted-bytes?]], wherever the answer decides whether a value is
- encrypted, decrypted, or left alone."
- ([^bytes b] (decryptable-bytes? default-secret-key b))
- ([secret-key ^bytes b]
- (boolean (and secret-key
+ "Whether `b` is encrypted with the key (and under the `:aad`, if any): it has the shape of ciphertext *and* decrypts.
+ The ciphertext is authenticated, so a wrong key, a wrong AAD, corruption, and plaintext that merely looks like
+ ciphertext all yield false. Always false when there is no key. Use this, not [[possibly-encrypted-bytes?]], wherever
+ the answer decides whether a value is encrypted, decrypted, or left alone."
+ ([^bytes b] (decryptable-bytes? b nil))
+ ([^bytes b opts]
+ (boolean (and (opts->secret-key opts)
(possibly-encrypted-bytes? b)
- (u/ignore-exceptions (decrypt-bytes secret-key b) true)))))
+ (u/ignore-exceptions (decrypt-bytes b opts) true)))))
(defn decryptable-string?
- "Whether `s` is encrypted with `secret-key`: it has the shape of ciphertext *and* decrypts. See [[decryptable-bytes?]]."
- ([^String s] (decryptable-string? default-secret-key s))
- ([secret-key ^String s]
- (boolean (and secret-key
+ "Whether `s` is encrypted with the key (and under the `:aad`, if any): it has the shape of ciphertext *and* decrypts.
+ See [[decryptable-bytes?]]."
+ ([^String s] (decryptable-string? s nil))
+ ([^String s opts]
+ (boolean (and (opts->secret-key opts)
(possibly-encrypted-string? s)
- (u/ignore-exceptions (decrypt secret-key s) true)))))
+ (u/ignore-exceptions (decrypt s opts) true)))))
(defn maybe-decrypt-accepting-plaintext
- "Plaintext-tolerant decrypt of a String `s`. If `MB_ENCRYPTION_SECRET_KEY` is set and `s` is encrypted, decrypt it;
- a value that is stored as plaintext is returned as-is. A value that *looks* encrypted but cannot be decrypted with
- the current key (wrong key, tampering, corruption) throws — it must not be silently returned, or a re-encrypting
- caller would double-encrypt it into something permanently unrecoverable. This differs from the strict
- [[maybe-decrypt]] only in tolerating genuine plaintext; use it for values that may legitimately be plaintext at rest
- (rows written before encryption was enabled, key rotation, settings, and migration reads).
-
- `secret-key` is accepted as an argument so tests can pass it directly instead of using `with-redefs` to run in
- parallel."
- (^String [^String s] (maybe-decrypt-accepting-plaintext default-secret-key s))
- (^String [secret-key ^String s]
+ "Plaintext-tolerant decrypt of a String `s`. If there is a key and `s` is encrypted, decrypt it; a value that is
+ stored as plaintext is returned as-is. A value that *looks* encrypted but cannot be decrypted with the key (wrong
+ key, wrong AAD, tampering, corruption) throws -- it must not be silently returned, or a re-encrypting caller would
+ double-encrypt it into something permanently unrecoverable. This differs from the strict [[maybe-decrypt]] only in
+ tolerating genuine plaintext; use it for values that may legitimately be plaintext at rest (rows written before
+ encryption was enabled, key rotation, settings, and migration reads)."
+ (^String [^String s] (maybe-decrypt-accepting-plaintext s nil))
+ (^String [^String s opts]
(cond
- (nil? secret-key) s
- (possibly-encrypted-string? s) (decrypt secret-key s)
+ (nil? (opts->secret-key opts)) s
+ (possibly-encrypted-string? s) (decrypt s opts)
:else s)))
(defn maybe-decrypt-bytes-accepting-plaintext
"Plaintext-tolerant counterpart to [[maybe-decrypt-accepting-plaintext]] for a byte array `b` (e.g. secret values):
an encrypted value is decrypted, a plaintext value is returned as-is, and a value that looks encrypted but cannot be
- decrypted with the current key throws rather than being returned (returning it would let a re-encrypting caller
+ decrypted with the key throws rather than being returned (returning it would let a re-encrypting caller
double-encrypt it into something permanently unrecoverable)."
- (^bytes [^bytes b] (maybe-decrypt-bytes-accepting-plaintext default-secret-key b))
- (^bytes [secret-key ^bytes b]
+ (^bytes [^bytes b] (maybe-decrypt-bytes-accepting-plaintext b nil))
+ (^bytes [^bytes b opts]
(cond
- (nil? secret-key) b
- (possibly-encrypted-bytes? b) (decrypt-bytes secret-key b)
- :else b)))
+ (nil? (opts->secret-key opts)) b
+ (possibly-encrypted-bytes? b) (decrypt-bytes b opts)
+ :else b)))
(defn maybe-decrypt
- "Strict decrypt of a String `s`. When `MB_ENCRYPTION_SECRET_KEY` is set, `s` must be an encrypted value that decrypts
- with the current key: a value that is not encrypted throws (it was written outside the encrypting path — a plaintext
- value cannot stand in for an encrypted one), and a value that is encrypted but cannot be decrypted with the current
- key (wrong key, tampering, or corruption) also throws rather than being trusted. When no key is set, `s` is returned
- as-is (there is no key to decrypt with). For values that may legitimately be plaintext at rest, use
- [[maybe-decrypt-accepting-plaintext]].
-
- `secret-key` is accepted as an argument so tests can pass it directly instead of using `with-redefs` to run in
- parallel."
- (^String [^String s] (maybe-decrypt default-secret-key s))
- (^String [secret-key ^String s]
+ "Strict decrypt of a String `s`. When there is a key, `s` must be an encrypted value that decrypts with it (and
+ under the `:aad`, if any): a value that is not encrypted throws (it was written outside the encrypting path -- a
+ plaintext value cannot stand in for an encrypted one), and a value that is encrypted but cannot be decrypted with
+ the key (wrong key, wrong AAD, tampering, or corruption) also throws rather than being trusted. When there is no
+ key, `s` is returned as-is (there is no key to decrypt with). For values that may legitimately be plaintext at rest,
+ use [[maybe-decrypt-accepting-plaintext]]."
+ (^String [^String s] (maybe-decrypt s nil))
+ (^String [^String s opts]
(cond
- (nil? secret-key) s
+ (nil? (opts->secret-key opts)) s
(nil? s) s
- (possibly-encrypted-string? s) (decrypt secret-key s)
+ (possibly-encrypted-string? s) (decrypt s opts)
:else (throw (ex-info "Expected an encrypted value but the stored value is not encrypted."
{:type ::not-encrypted})))))
(defn maybe-decrypt-bytes
"Strict counterpart to [[maybe-decrypt-bytes-accepting-plaintext]] for a byte array `b`: a value that is not encrypted,
- or that cannot be decrypted with the current key, throws rather than being trusted. When no key is set, `b` is returned
+ or that cannot be decrypted with the key, throws rather than being trusted. When there is no key, `b` is returned
as-is."
- (^bytes [^bytes b] (maybe-decrypt-bytes default-secret-key b))
- (^bytes [secret-key ^bytes b]
+ (^bytes [^bytes b] (maybe-decrypt-bytes b nil))
+ (^bytes [^bytes b opts]
(cond
- (nil? secret-key) b
- (nil? b) b
- (possibly-encrypted-bytes? b) (decrypt-bytes secret-key b)
- :else (throw (ex-info "Expected an encrypted value but the stored value is not encrypted."
- {:type ::not-encrypted})))))
+ (nil? (opts->secret-key opts)) b
+ (nil? b) b
+ (possibly-encrypted-bytes? b) (decrypt-bytes b opts)
+ :else (throw (ex-info "Expected an encrypted value but the stored value is not encrypted."
+ {:type ::not-encrypted})))))
diff --git a/test/metabase/app_db/query_test.clj b/test/metabase/app_db/query_test.clj
index e5195552079..b72b44a317f 100644
--- a/test/metabase/app_db/query_test.clj
+++ b/test/metabase/app_db/query_test.clj
@@ -8,6 +8,7 @@
[metabase.app-db.query :as mdb.query]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.test :as qp]
+ [metabase.settings.models.setting :as setting]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
@@ -19,6 +20,25 @@
(use-fixtures :once (fixtures/initialize :db))
+(defn- random-setting-name!
+ "Register a Setting under a random name and return it. The Setting model refuses to write a key it cannot resolve to
+ a definition, and these tests need a fresh key per racing thread -- more than one `defsetting` could supply."
+ []
+ (let [setting-name (keyword (str "query-test-setting-" (random-uuid)))]
+ (setting/register-setting! {:name setting-name
+ :namespace (ns-name *ns*)
+ :type :string
+ :encryption :no
+ :visibility :internal})
+ (name setting-name)))
+
+(defn- random-value!
+ "A fresh value to store in `column`: the `key` column holds setting names, the `value` column anything."
+ [column]
+ (if (= column :key)
+ (random-setting-name!)
+ (str (random-uuid))))
+
(defn- verify-same-query
"Ensure that the formatted native query derived from an mbql query produce the same results."
[q]
@@ -85,7 +105,7 @@
(deftest select-or-insert!-test
;; We test both a case where the database protects against duplicates, and where it does not.
- ;; Using Setting is perfect because it has only two required fields - (the primary) key & value (with no constraint).
+ ;; Using the `setting` table is perfect because it has only two required fields - (the primary) key & value (with
;;
;; In the `:key` case using the `idempotent-insert!` rather than an `or` prevents the from application throwing an
;; exception when there are race conditions. For `:value` it prevents us silently inserting duplicates.
@@ -95,7 +115,7 @@
(doseq [search-col columns]
(testing (format "When the search column %s a uniqueness constraint in the db"
(if (= :key search-col) "has" "does not have"))
- (let [search-value (str (random-uuid))
+ (let [search-value (random-value! search-col)
other-col (first (remove #{search-col} columns))]
(try
;; ensure there is no database detritus to trip us up
@@ -108,7 +128,7 @@
;; Make sure all the threads are in the mutating path
(.countDown latch)
(.await latch)
- {other-col (str (random-uuid))})))
+ {other-col (random-value! other-col)})))
results (set (mt/repeat-concurrently threads thunk))
n (count results)
latest (t2/select-one :model/Setting search-col search-value)]
@@ -137,15 +157,15 @@
(deftest updated-or-insert!-test
;; We test both a case where the database protects against duplicates, and where it does not.
- ;; Using Setting is perfect because it has only two required fields - (the primary) key & value (with no constraint).
+ ;; Using the `setting` table is perfect because it has only two required fields - (the primary) key & value (with
(let [columns [:key :value]]
(doseq [search-col columns]
(testing (format "When the search column %s a uniqueness constraint in the db"
(if (= :key search-col) "has" "does not have"))
(doseq [already-exists? [true false]]
- (let [search-value (str (random-uuid))
+ (let [search-value (random-value! search-col)
other-col (first (remove #{search-col} columns))
- other-value (str (random-uuid))]
+ other-value (random-value! other-col)]
(try
;; ensure there is no database detritus to trip us up
(t2/delete! :model/Setting search-col search-value)
@@ -154,7 +174,7 @@
(let [threads 5
latch (CountDownLatch. threads)
thunk (fn []
- (u/prog1 (str (random-uuid))
+ (u/prog1 (random-value! other-col)
(mdb.query/update-or-insert! :model/Setting {search-col search-value}
(fn [_]
;; Make sure all the threads are in the mutating path
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index de6f3b6f9f1..6e83d794280 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -2559,88 +2559,88 @@
(deftest ^:mb/old-migrations-test populate-enabled-embedding-settings-works
(testing "Check that embedding settings are nil when enable-embedding is nil"
(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]
- (t2/delete! :model/Setting :key "enable-embedding")
+ (t2/delete! :setting :key "enable-embedding")
(migrate!)
- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-interactive")))
- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-static")))
- (is (= nil (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk")))))
+ (is (= nil (t2/select-one :setting :key "enable-embedding-interactive")))
+ (is (= nil (t2/select-one :setting :key "enable-embedding-static")))
+ (is (= nil (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk"))))))
(testing "Check that embedding settings are true when enable-embedding is true"
(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]
- (t2/delete! :model/Setting :key "enable-embedding")
- (t2/insert! :model/Setting {:key "enable-embedding" :value "true"})
+ (t2/delete! :setting :key "enable-embedding")
+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "true")})
(migrate!)
- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))
- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))
- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk")))))
+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))
+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))
+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk"))))))
(testing "Check that embedding settings are false when enable-embedding is false"
(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]
- (t2/delete! :model/Setting :key "enable-embedding")
- (t2/insert! :model/Setting {:key "enable-embedding" :value "false"})
+ (t2/delete! :setting :key "enable-embedding")
+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "false")})
(migrate!)
- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))
- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))
- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk"))))))
+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))
+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))
+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk")))))))
(deftest ^:mb/old-migrations-test populate-enabled-embedding-settings-encrypted-works
(testing "With encryption turned on > "
(mt/with-temp-env-var-value! [MB_ENCRYPTION_SECRET_KEY "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"]
(testing "Check that embedding settings are nil when enable-embedding is nil"
(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]
- (t2/delete! :model/Setting :key "enable-embedding")
+ (t2/delete! :setting :key "enable-embedding")
(migrate!)
- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-interactive")))
- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-static")))
- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-sdk")))))
+ (is (= nil (t2/select-one :setting :key "enable-embedding-interactive")))
+ (is (= nil (t2/select-one :setting :key "enable-embedding-static")))
+ (is (= nil (t2/select-one :setting :key "enable-embedding-sdk")))))
(testing "Check that embedding settings are true when enable-embedding is true"
(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]
- (t2/delete! :model/Setting :key "enable-embedding")
- (t2/insert! :model/Setting {:key "enable-embedding" :value "true"})
+ (t2/delete! :setting :key "enable-embedding")
+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "true")})
(migrate!)
- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))
- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))
- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk")))))
+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))
+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))
+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk"))))))
(testing "Check that embedding settings are false when enable-embedding is false"
(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]
- (t2/delete! :model/Setting :key "enable-embedding")
- (t2/insert! :model/Setting {:key "enable-embedding" :value "false"})
+ (t2/delete! :setting :key "enable-embedding")
+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "false")})
(migrate!)
- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))
- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))
- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk"))))))))
+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))
+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))
+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk")))))))))
(deftest ^:mb/old-migrations-test populate-embedding-origin-settings-works
(testing "Check that embedding-origins are unset when embedding-app-origin is unset"
(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]
- (t2/delete! :model/Setting :key "embedding-app-origin")
+ (t2/delete! :setting :key "embedding-app-origin")
(migrate!)
- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-interactive")))
- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-sdk"))))))
+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-interactive")))
+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-sdk"))))))
(deftest ^:mb/old-migrations-test populate-embedding-origin-settings-works-2
(testing "Check that embedding-origins settings are propigated when embedding-app-origin is set to some value"
(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]
- (t2/delete! :model/Setting :key "embedding-app-origin")
- (t2/insert! :model/Setting {:key "embedding-app-origin" :value "1.2.3.4:5555"})
- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origin")))
+ (t2/delete! :setting :key "embedding-app-origin")
+ (t2/insert! :setting {:key "embedding-app-origin" :value (encryption/maybe-encrypt "1.2.3.4:5555")})
+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origin"))))
(migrate!)
- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origins-interactive"))))))
+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origins-interactive")))))))
(deftest ^:mb/old-migrations-test populate-embedding-origin-settings-encrypted-works
(testing "With encryption turned on > "
(mt/with-temp-env-var-value! [MB_ENCRYPTION_SECRET_KEY "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"]
(testing "Check that embedding-origins are unset when embedding-app-origin is unset"
(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]
- (t2/delete! :model/Setting :key "embedding-app-origin")
+ (t2/delete! :setting :key "embedding-app-origin")
(migrate!)
- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-interactive")))
- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-sdk")))))
+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-interactive")))
+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-sdk")))))
(testing "Check that embedding-origins settings are propigated when embedding-app-origin is set to some value"
(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]
- (t2/delete! :model/Setting :key "embedding-app-origin")
- (t2/insert! :model/Setting {:key "embedding-app-origin" :value "1.2.3.4:5555"})
- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origin")))
+ (t2/delete! :setting :key "embedding-app-origin")
+ (t2/insert! :setting {:key "embedding-app-origin" :value (encryption/maybe-encrypt "1.2.3.4:5555")})
+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origin"))))
(migrate!)
- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origins-interactive"))))))))
+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origins-interactive")))))))))
;;;
;;; 53+ tests should go below this line please <3
diff --git a/test/metabase/app_db/setup_test.clj b/test/metabase/app_db/setup_test.clj
index badd77ec74e..48999f9bff1 100644
--- a/test/metabase/app_db/setup_test.clj
+++ b/test/metabase/app_db/setup_test.clj
@@ -4,12 +4,16 @@
[clojure.test :refer :all]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.connection-pool-setup :as mdb.connection-pool-setup]
+ [metabase.app-db.core :as mdb]
[metabase.app-db.data-source :as mdb.data-source]
[metabase.app-db.liquibase :as liquibase]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.app-db.setup :as mdb.setup]
[metabase.app-db.test-util :as mdb.test-util]
[metabase.driver :as driver]
[metabase.test :as mt]
+ [metabase.util.encryption :as encryption]
+ [metabase.util.encryption-test :as encryption-test]
[toucan2.core :as t2])
(:import
(liquibase.changelog ChangeSet)))
@@ -226,3 +230,35 @@
{:delete [:field]
:from [[:metabase_field :field]]
:where [:= :field.id 0]}))))))
+
+(deftest setup-db-leaves-settings-alone-without-their-key-test
+ (testing "setup-db! fills in setting.details, except in an encryption state where the rows cannot be read"
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
+ (let [ciphertext (encryption-test/with-secret-key "ABCDEFGH12345678"
+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)
+ (t2/insert! :model/Setting {:key "site-name", :value "Sad Can"})
+ (t2/select-one-fn :value_with_aad :setting :key "site-name"))]
+ (is (encryption/possibly-encrypted-string? ciphertext))
+ (testing "with no key the state is :missing-key, and a repair would only wrap the ciphertext as a value"
+ (reset! (:status mdb.connection/*application-db*) ::not-set-up)
+ (mdb/setup-db! :create-sample-content? false :manage-encryption-state? false)
+ (is (= ciphertext (t2/select-one-fn :value_with_aad :setting :key "site-name"))))))))
+
+(deftest encryption-check-status-falls-back-to-value-test
+ (testing "the sentinel is read from `value`, whatever `value_with_aad` holds -- an older version rewrites only `value`"
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
+ (encryption-test/with-secret-key "ABCDEFGH12345678"
+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)
+ (is (= :valid (mdb/encryption-check-status)))
+ (testing "an authenticated value left under a key a rotation on an older version has since replaced"
+ (let [old-key (encryption/secret-key->hash "12345678ABCDEFGH")]
+ (t2/update! :setting :key "encryption-check"
+ {:value_with_aad (encryption/encrypt (str (random-uuid))
+ {:secret-key old-key
+ :aad (mdb.setting/setting-aad "encryption-check")})})
+ (is (= :valid (mdb/encryption-check-status)))))
+ (testing "the unencrypted marker an older version's remove-encryption writes to value alone"
+ (t2/update! :setting :key "encryption-check" {:value "unencrypted"})
+ (is (= :absent (mdb/encryption-check-status))))))))
diff --git a/test/metabase/cmd/dump_to_h2_test.clj b/test/metabase/cmd/dump_to_h2_test.clj
index 79aa7d930b8..bc460106e19 100644
--- a/test/metabase/cmd/dump_to_h2_test.clj
+++ b/test/metabase/cmd/dump_to_h2_test.clj
@@ -15,6 +15,7 @@
[metabase.cmd.test-util :as cmd.test-util]
[metabase.config.core :as config]
[metabase.driver :as driver]
+ [metabase.settings.core :refer [defsetting]]
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.util.encryption-test :as encryption-test]
@@ -23,6 +24,11 @@
(set! *warn-on-reflection* true)
+(defsetting dump-to-h2-test-setting
+ "Test setting -- a setting row to check is dumped decrypted or encrypted as asked."
+ :visibility :internal
+ :encryption :when-encryption-key-set)
+
(deftest dump-deletes-target-db-files-tests
;; test fails when the application db is anything but H2 presently
;; TODO: make this test work with postgres / mysql / mariadb
@@ -79,7 +85,7 @@
;; does) so no encrypted column is left plaintext for the strict model reads
;; the update and dump below trigger
(mdb.encryption/encrypt-db driver/*driver* (:data-source mdb.connection/*application-db*) nil)
- (t2/insert! :model/Setting {:key "my-site-admin", :value "baz"})
+ (t2/insert! :model/Setting {:key "dump-to-h2-test-setting", :value "baz"})
(t2/update! :model/Database 1 {:details {:db "/tmp/test.db"}})
(dump-to-h2/dump-to-h2! h2-file-plaintext {:dump-plaintext? true})
(dump-to-h2/dump-to-h2! h2-file-enc {:dump-plaintext? false})
@@ -87,7 +93,7 @@
(testing "decodes settings and dashboard.details"
(with-open [target-conn (.getConnection (copy.h2/h2-data-source h2-file-plaintext))]
(is (= "baz" (:value (first (jdbc/query {:connection target-conn}
- "select \"VALUE\" from SETTING where \"KEY\"='my-site-admin';")))))
+ "select \"VALUE\" from SETTING where \"KEY\"='dump-to-h2-test-setting';")))))
(is (= "{\"db\":\"/tmp/test.db\"}"
(:details (first (jdbc/query {:connection target-conn}
"select details from metabase_database where id=1;")))))))
@@ -95,7 +101,7 @@
(with-open [target-conn (.getConnection (copy.h2/h2-data-source h2-file-enc))]
(is (not (= "baz"
(:value (first (jdbc/query {:connection target-conn}
- "select \"VALUE\" from SETTING where \"KEY\"='my-site-admin';"))))))
+ "select \"VALUE\" from SETTING where \"KEY\"='dump-to-h2-test-setting';"))))))
(is (not (= "{\"db\":\"/tmp/test.db\"}"
(:details (first (jdbc/query {:connection target-conn}
"select details from metabase_database where id=1;"))))))))
@@ -103,7 +109,7 @@
(with-open [target-conn (.getConnection (copy.h2/h2-data-source h2-file-default-enc))]
(is (not (= "baz"
(:value (first (jdbc/query {:connection target-conn}
- "select \"VALUE\" from SETTING where \"KEY\"='my-site-admin';"))))))
+ "select \"VALUE\" from SETTING where \"KEY\"='dump-to-h2-test-setting';"))))))
(is (not (= "{\"db\":\"/tmp/test.db\"}"
(:details (first (jdbc/query {:connection target-conn}
"select details from metabase_database where id=1;")))))))))))))))
diff --git a/test/metabase/cmd/enable_encryption_test.clj b/test/metabase/cmd/enable_encryption_test.clj
index 65d4912a23b..1c6089c5663 100644
--- a/test/metabase/cmd/enable_encryption_test.clj
+++ b/test/metabase/cmd/enable_encryption_test.clj
@@ -5,6 +5,7 @@
[metabase.app-db.core :as mdb]
[metabase.cmd.core :as cmd]
[metabase.cmd.enable-encryption :refer [enable-encryption!]]
+ [metabase.settings.core :refer [defsetting]]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util.encryption :as encryption]
@@ -15,6 +16,11 @@
(use-fixtures :once (fixtures/initialize :db))
+(defsetting enable-encryption-test-setting
+ "Test setting -- a plaintext setting row for `enable-encryption` to encrypt."
+ :visibility :internal
+ :encryption :no)
+
(defn- raw-value [k]
(t2/select-one-fn :value :setting :key k))
@@ -32,7 +38,7 @@
(encryption-test/with-secret-key nil
(mt/with-temp-empty-app-db [_conn :h2]
(mdb/setup-db! :create-sample-content? true)
- (t2/insert! :model/Setting {:key "test-setting", :value "plain value"})
+ (t2/insert! :model/Setting {:key "enable-encryption-test-setting", :value "plain value"})
;; the v53 migration's legacy plaintext marker; new code never writes it and reads it as "no sentinel"
(is (= "unencrypted" (raw-value "encryption-check")))
(encryption-test/with-secret-key "key1"
@@ -42,8 +48,8 @@
(testing "the command encrypts everything and writes the sentinel"
(enable-encryption!)
(is (encryption/decryptable-string? (raw-value "encryption-check")))
- (is (encryption/decryptable-string? (raw-value "test-setting")))
- (is (= "plain value" (t2/select-one-fn :value :model/Setting :key "test-setting")))
+ (is (encryption/decryptable-string? (raw-value "enable-encryption-test-setting")))
+ (is (= "plain value" (t2/select-one-fn :value :model/Setting :key "enable-encryption-test-setting")))
(is (encryption/decryptable-string? (t2/select-one-fn :details :metabase_database)))
(is (map? (t2/select-one-fn :details :model/Database))))
(testing "startup now succeeds"
diff --git a/test/metabase/cmd/remove_encryption_test.clj b/test/metabase/cmd/remove_encryption_test.clj
index bd20bfd48f8..aee9f91b175 100644
--- a/test/metabase/cmd/remove_encryption_test.clj
+++ b/test/metabase/cmd/remove_encryption_test.clj
@@ -5,6 +5,7 @@
[metabase.app-db.core :as mdb]
[metabase.cmd.core :as cmd]
[metabase.cmd.remove-encryption :refer [remove-encryption!]]
+ [metabase.settings.core :refer [defsetting]]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util.encryption :as encryption]
@@ -15,6 +16,11 @@
(use-fixtures :once (fixtures/initialize :db))
+(defsetting remove-encryption-test-setting
+ "Test setting -- an encrypted setting row for `remove-encryption` to decrypt."
+ :visibility :internal
+ :encryption :when-encryption-key-set)
+
(defn- raw-value [data-source keyy]
(-> (jdbc/query {:connection data-source}
["select \"VALUE\" from setting where setting.\"KEY\"=?;" keyy])
@@ -28,12 +34,12 @@
(deftest remove-encryption!-test
(testing "removing encryption"
- (encryption-test/with-secret-key "key1"
- (mt/with-temp-empty-app-db [_conn :h2]
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (encryption-test/with-secret-key "key1"
(mdb/setup-db! :create-sample-content? true)
- (t2/insert! :model/Setting {:key "test-setting", :value "unencrypted value"})
+ (t2/insert! :model/Setting {:key "remove-encryption-test-setting", :value "unencrypted value"})
(is (encryption/decryptable-string? (raw-value _conn "encryption-check")))
- (is (encryption/decryptable-string? (raw-value _conn "test-setting")))
+ (is (encryption/decryptable-string? (raw-value _conn "remove-encryption-test-setting")))
(remove-encryption!)
(is (= "unencrypted" (raw-value _conn "encryption-check")))
- (is (not (encryption/possibly-encrypted-string? (raw-value _conn "test-setting"))))))))
+ (is (not (encryption/possibly-encrypted-string? (raw-value _conn "remove-encryption-test-setting"))))))))
diff --git a/test/metabase/cmd/rotate_encryption_key_test.clj b/test/metabase/cmd/rotate_encryption_key_test.clj
index 4f1d3eaaa48..ce90229c547 100644
--- a/test/metabase/cmd/rotate_encryption_key_test.clj
+++ b/test/metabase/cmd/rotate_encryption_key_test.clj
@@ -14,6 +14,7 @@
[metabase.config.core :as config]
[metabase.driver :as driver]
[metabase.models.interface :as mi]
+ [metabase.settings.core :refer [defsetting]]
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.test.fixtures :as fixtures]
@@ -29,6 +30,16 @@
(use-fixtures :once (fixtures/initialize :db))
+(defsetting rotate-test-plaintext-setting
+ "Test setting -- a setting that is not encrypted at rest."
+ :visibility :internal
+ :encryption :no)
+
+(defsetting rotate-test-encrypted-setting
+ "Test setting -- a setting that is encrypted at rest."
+ :visibility :internal
+ :encryption :when-encryption-key-set)
+
(defn- raw-value [keyy]
(:value (first (jdbc/query {:datasource (mdb/data-source)}
[(if (= driver/*driver* :h2)
@@ -83,7 +94,7 @@
(tx/create-db! driver/*driver* {:database-name db-name}))
(binding [copy/*copy-h2-database-details* true]
(load-from-h2/load-from-h2! h2-fixture-db-file))
- (t2/insert! :model/Setting {:key "nocrypt", :value "unencrypted value"})
+ (t2/insert! :model/Setting {:key "rotate-test-plaintext-setting", :value "unencrypted value"})
(t2/insert! :model/Setting {:key "settings-last-updated", :value original-timestamp})
(let [u (first (t2/insert-returning-instances! :model/User {:email "[email protected]"
:first_name "No"
@@ -98,7 +109,7 @@
:creator_id @user-id}))]
(reset! secret-id-unenc (u/the-id secret)))
(encryption-test/with-secret-key k1
- (t2/insert! :model/Setting {:key "k1crypted", :value "encrypted with k1"})
+ (t2/insert! :model/Setting {:key "rotate-test-encrypted-setting", :value "encrypted with k1"})
(mdb.encryption/encrypt-db driver/*driver* data-source nil)
(t2/update! :model/Database 1 {:details {:db "/tmp/test.db"}})
;; other encrypted-json columns that must also be re-encrypted on rotation
@@ -114,13 +125,13 @@
(rotate-encryption-key! k1)
;; plain->newkey
(testing "for unencrypted values"
- (is (not= "unencrypted value" (raw-value "nocrypt")))
- (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "nocrypt")))
+ (is (not= "unencrypted value" (raw-value "rotate-test-plaintext-setting")))
+ (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "rotate-test-plaintext-setting")))
(is (mt/secret-value-equals? secret-val (t2/select-one-fn :value :model/Secret :id @secret-id-unenc))))
;; samekey->samekey
(testing "for values encrypted with the same key"
- (is (not= "encrypted with k1" (raw-value "k1crypted")))
- (is (= "encrypted with k1" (t2/select-one-fn :value :model/Setting :key "k1crypted")))
+ (is (not= "encrypted with k1" (raw-value "rotate-test-encrypted-setting")))
+ (is (= "encrypted with k1" (t2/select-one-fn :value :model/Setting :key "rotate-test-encrypted-setting")))
(is (mt/secret-value-equals? secret-val (t2/select-one-fn :value :model/Secret :id @secret-id-enc))))))
(testing "settings-last-updated is updated AND plaintext"
(is (not= original-timestamp (raw-value "settings-last-updated")))
@@ -129,7 +140,7 @@
(encryption-test/with-secret-key k1 (rotate-encryption-key! k2))
(testing "with new key"
(encryption-test/with-secret-key k2
- (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "nocrypt")))
+ (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "rotate-test-plaintext-setting")))
(is (= {:db "/tmp/test.db"} (t2/select-one-fn :details :model/Database :id 1)))
(is (= {:database-enable-actions true} (t2/select-one-fn :settings :model/Database :id 1)))
(is (= {:locale "en"} (t2/select-one-fn :settings :model/User :id @user-id)))
@@ -138,7 +149,7 @@
(encryption-test/with-secret-key k1
;; reading a value that looks encrypted but can't be decrypted with the current key throws rather
;; than returning the raw ciphertext, so it can never be mistaken for a real plaintext value
- (is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :value :model/Setting :key "nocrypt")))
+ (is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :value :model/Setting :key "rotate-test-plaintext-setting")))
(is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :details :model/Database :id 1)))
(is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :settings :model/Database :id 1)))
(is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :settings :model/User :id @user-id)))
@@ -165,7 +176,7 @@
(t2/update! :model/Database {:name "k3"} {:details {:db "/tmp/test.db"}})
(encryption-test/with-secret-key k2 ; with the last key that we rotated to in the test
(rotate-encryption-key! nil))
- (is (= "unencrypted value" (raw-value "nocrypt")))
+ (is (= "unencrypted value" (raw-value "rotate-test-plaintext-setting")))
;; at this point, both the originally encrypted, and the originally unencrypted secret instances
;; should be decrypted
(is (mt/secret-value-equals? secret-val (t2/select-one-fn :value :model/Secret :id @secret-id-unenc)))
@@ -213,7 +224,7 @@
(encryption-test/with-secret-key k1
(set-encryption-check-raw! (encryption/encrypt (str (random-uuid))))
(let [good-id (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"en\"}"))
- bad-id (insert-user-with-raw-settings! (encryption/encrypt k2-hashed "{\"locale\":\"fr\"}"))]
+ bad-id (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"fr\"}" {:secret-key k2-hashed}))]
(mdb/decrypt-db :h2 (mdb/data-source))
(is (= "{\"locale\":\"en\"}" (t2/select-one-fn :settings :core_user :id good-id)))
(is (= "{}" (t2/select-one-fn :settings :core_user :id bad-id)))
@@ -234,7 +245,7 @@
(mt/with-empty-h2-app-db!
(encryption-test/with-secret-key k1
(t2/delete! :setting :key "encryption-check")
- (insert-user-with-raw-settings! (encryption/encrypt k2-hashed "{\"locale\":\"fr\"}"))
+ (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"fr\"}" {:secret-key k2-hashed}))
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Can't decrypt app db with MB_ENCRYPTION_SECRET_KEY"
@@ -243,7 +254,7 @@
(mt/with-empty-h2-app-db!
(encryption-test/with-secret-key k1
(set-encryption-check-raw! (encryption/encrypt (str (random-uuid))))
- (insert-channel-with-raw-details! (encryption/encrypt k2-hashed "{\"url\":\"http://bad\"}"))
+ (insert-channel-with-raw-details! (encryption/encrypt "{\"url\":\"http://bad\"}" {:secret-key k2-hashed}))
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Can't decrypt app db with MB_ENCRYPTION_SECRET_KEY"
diff --git a/test/metabase/models/interface_test.clj b/test/metabase/models/interface_test.clj
index 2a2bca20069..ce6a6a2bb47 100644
--- a/test/metabase/models/interface_test.clj
+++ b/test/metabase/models/interface_test.clj
@@ -123,11 +123,11 @@
(is (= {:a 1}
(encryption-test/with-secret-key "qwe"
(mi/encrypted-json-out
- (encryption/encrypt (encryption/secret-key->hash "qwe") "{\"a\": 1}"))))))
+ (encryption/encrypt "{\"a\": 1}" {:secret-key (encryption/secret-key->hash "qwe")}))))))
(testing "Logs an error message when incoming data looks encrypted"
(mt/with-log-messages-for-level [messages :error]
(mi/encrypted-json-out
- (encryption/encrypt (encryption/secret-key->hash "qwe") "{\"a\": 1}"))
+ (encryption/encrypt "{\"a\": 1}" {:secret-key (encryption/secret-key->hash "qwe")}))
(is (=? [{:level :error
:e nil
:message "Could not decrypt encrypted field! Have you forgot to set MB_ENCRYPTION_SECRET_KEY?"}]
@@ -140,7 +140,7 @@
(mt/with-log-messages-for-level [messages :error]
(encryption-test/with-secret-key "qwe"
(mi/encrypted-json-out
- (encryption/encrypt (encryption/secret-key->hash "qwe") "{\"a\": 1")))
+ (encryption/encrypt "{\"a\": 1" {:secret-key (encryption/secret-key->hash "qwe")})))
(is (=? [{:level :error, :e nil, :message #"(?s)^Error parsing JSON: .*"}]
(messages)))))))
diff --git a/test/metabase/models/serialization/resolve/db_test.clj b/test/metabase/models/serialization/resolve/db_test.clj
index 331e1b2c4d4..44bb38170cc 100644
--- a/test/metabase/models/serialization/resolve/db_test.clj
+++ b/test/metabase/models/serialization/resolve/db_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase.models.serialization :as serdes]
+ [metabase.settings.models.setting.cache :as setting.cache]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -98,6 +99,7 @@
(deftest cached-export-database-fk-test
(mt/with-temp [:model/Database {db-id :id} {:name "test-db"}]
+ (setting.cache/restore-cache!)
(serdes/with-cache
(t2/with-call-count [call-count]
(is (= "test-db" (serdes/*export-database-fk* db-id)))
diff --git a/test/metabase/settings/models/setting/cache_test.clj b/test/metabase/settings/models/setting/cache_test.clj
index af29e773904..81520ae411e 100644
--- a/test/metabase/settings/models/setting/cache_test.clj
+++ b/test/metabase/settings/models/setting/cache_test.clj
@@ -2,11 +2,13 @@
(:require
[clojure.test :refer :all]
[metabase.app-db.core :as mdb]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.settings.models.setting-test :as setting-test]
[metabase.settings.models.setting.cache :as setting.cache]
[metabase.system.core :as system]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.util.encryption :as encryption]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -23,15 +25,25 @@
(defn- update-settings-last-updated-value-in-db!
"Simulate a different instance updating the value of `settings-last-updated` in the DB by updating its value without
- updating our locally cached value.."
+ updating our locally cached value.
+
+ Written raw, and to both columns, exactly as
+ [[metabase.settings.models.setting.cache/update-settings-last-updated!]] writes them -- through `:model/Setting` the
+ raw SQL below would end up inside the JSON envelope rather than being evaluated."
[]
- (t2/update! :model/Setting {:key setting.cache/settings-last-updated-key}
- {:value ^:allow-raw-sql [:raw (case (mdb/db-type)
- ;; make it one second in the future so we don't end up getting an exact match when we try to test
- ;; to see if things update below
- :h2 "cast(dateadd('second', 1, current_timestamp) AS text)"
- :mysql "cast((current_timestamp + interval 1 second) AS char)"
- :postgres "cast((current_timestamp + interval '1 second') AS text)")]}))
+ (let [ts (-> (t2/query-one
+ {:select [[^:allow-raw-sql
+ [:raw (case (mdb/db-type)
+ ;; make it one second in the future so we don't end up getting an exact match when we try to test
+ ;; to see if things update below
+ :h2 "cast(dateadd('second', 1, current_timestamp) AS text)"
+ :mysql "cast((current_timestamp + interval 1 second) AS char)"
+ :postgres "cast((current_timestamp + interval '1 second') AS text)")]
+ :timestamp]]})
+ :timestamp)]
+ (t2/update! :setting {:key setting.cache/settings-last-updated-key}
+ {:value ts
+ :value_with_aad (encryption/maybe-encrypt ts {:aad (mdb.setting/setting-aad setting.cache/settings-last-updated-key)})})))
(defn- simulate-another-instance-updating-setting! [setting-name new-value]
(if new-value
diff --git a/test/metabase/settings/models/setting_test.clj b/test/metabase/settings/models/setting_test.clj
index 9e75ed7ae78..f8e40d1c1cb 100644
--- a/test/metabase/settings/models/setting_test.clj
+++ b/test/metabase/settings/models/setting_test.clj
@@ -7,6 +7,7 @@
[medley.core :as m]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.core :as mdb]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.cloud-migration.models.cloud-migration :as cloud-migration]
[metabase.config.core :as config]
[metabase.settings.models.setting :as setting :refer [defsetting]]
@@ -26,7 +27,7 @@
(set! *warn-on-reflection* true)
;; side-effect require: registers the DML build guard exercised by
-;; [[migrate-encrypted-settings!-does-not-depend-on-settings-cache-test]]
+;; [[migrate-settings!-repairs-past-the-read-only-mode-guard-test]]
(comment cloud-migration/keep-me)
(use-fixtures :once (fixtures/initialize :db))
@@ -617,52 +618,104 @@
;;; ----------------------------------------------- Encrypted Settings -----------------------------------------------
-(defn- actual-value-in-db [setting-key]
- (-> (mdb/query {:select [:value]
+(defn- raw-setting-value-with-aad
+ "The setting's `value_with_aad` exactly as it sits in the DB, bypassing the model's decrypting read."
+ [setting-key]
+ (-> (mdb/query {:select [:value_with_aad]
:from [:setting]
:where [:= :key (name setting-key)]})
- first :value))
+ first :value_with_aad))
+
+(defn- aad-opts
+ "Encryption opts binding a value to `setting-key`, as the Setting model stores one."
+ [setting-key & {:as more}]
+ (merge {:aad (mdb.setting/setting-aad (name setting-key))} more))
+
+(defn- decrypt-setting-value
+ "Decrypt `raw`, a `value_with_aad`, under the AAD of `setting-key`."
+ [setting-key raw]
+ (encryption/decrypt raw (aad-opts setting-key)))
(deftest encrypted-settings-test
(testing "If encryption is *enabled*, make sure Settings get saved as encrypted!"
;; Setting an encryption key without running encrypt-db leaves the other encrypted settings in the shared app DB
;; stored plaintext, and restoring the whole-table settings cache strictly decrypts every one of them. Use an
- ;; isolated app DB so this test's key only meets settings it wrote itself.
+ ;; isolated app DB so this test's key only meets settings it wrote itself -- and write without a key first: every
+ ;; write under a key leaves ciphertext behind (the last-updated marker included) that nothing can read once the
+ ;; key is gone.
(mt/with-temp-empty-app-db [_conn :h2]
(mdb/setup-db! :create-sample-content? false)
+ (testing "If encryption is not enabled, of course Settings shouldn't get saved as encrypted."
+ (encryption-test/with-secret-key nil
+ (toucan-name! "Sad Can")
+ (is (= "Sad Can"
+ (raw-setting-value-with-aad :toucan-name)))))
(encryption-test/with-secret-key "ABCDEFGH12345678"
+ ;; every row so far is plaintext, which a strict read under a key rejects -- so do what `enable-encryption`
+ ;; does before the first write restores the settings cache over them
+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)
(toucan-name! "Sad Can")
- (is (u/base64-string? (actual-value-in-db :toucan-name)))
- (testing "make sure it can be decrypted as well..."
+ (is (u/base64-string? (raw-setting-value-with-aad :toucan-name)))
+ (testing "the ciphertext is bound to the setting it belongs to: it decrypts under that name and no other"
(is (= "Sad Can"
- (toucan-name)))))
- (testing "But if encryption is not enabled, of course Settings shouldn't get saved as encrypted."
- (encryption-test/with-secret-key nil
- (toucan-name! "Sad Can")
+ (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name))))
+ (is (thrown? Throwable (encryption/decrypt (raw-setting-value-with-aad :toucan-name))))
+ (is (thrown? Throwable (decrypt-setting-value :site-name (raw-setting-value-with-aad :toucan-name)))))
+ (testing "make sure it can be decrypted as well..."
(is (= "Sad Can"
- (actual-value-in-db :toucan-name))))))))
+ (toucan-name))))))))
(deftest decrypt-error-names-setting-test
(testing "a Setting row that fails the decrypting read names the setting in the message (and never the value)"
(encryption-test/with-secret-key "0123456789abcdef"
(let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo
- #"Error decrypting setting \"toucan-name\": Expected an encrypted value"
- (#'setting/decrypt-setting-value-on-read {:key "toucan-name" :value "plaintext-sekret"})))]
+ #"Error reading setting \"toucan-name\": Expected an encrypted value"
+ (#'setting/read-setting-value {:key "toucan-name" :value_with_aad "plaintext-sekret"})))]
(is (not (re-find #"sekret" (ex-message e))))
(is (= "toucan-name" (:setting-key (ex-data e))))))))
+(deftest setting-value-aad-test
+ (let [read #'setting/read-setting-value]
+ (encryption-test/with-secret-key "ABCDEFGH12345678"
+ (testing "a value round trips through the AAD naming its setting"
+ (is (= "Sad Can"
+ (:value (read {:key "test-setting-1"
+ :value_with_aad (encryption/encrypt "Sad Can" (aad-opts :test-setting-1))})))))
+ (testing "a value moved to another setting's row is rejected, and the message never carries the value"
+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Error reading setting \"test-setting-2\""
+ (read {:key "test-setting-2"
+ :value_with_aad (encryption/encrypt "Sad Can" (aad-opts :test-setting-1))})))]
+ (is (not (re-find #"Sad Can" (ex-message e))))))
+ (testing "a plaintext value is rejected while a key is set"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Expected an encrypted value"
+ (read {:key "test-setting-1" :value_with_aad "Sad Can"}))))
+ (testing "a row written only by a version predating `value_with_aad` reads as no value at all"
+ (is (= {:key "test-setting-1" :value nil :value_with_aad nil}
+ (read {:key "test-setting-1" :value "Sad Can" :value_with_aad nil}))))
+ (testing "a key with no `defsetting` reads as no value at all, without so much as a decrypt"
+ (is (= {:key "no-such-setting" :value nil :value_with_aad "not even ciphertext"}
+ (read {:key "no-such-setting" :value_with_aad "not even ciphertext"})))))
+ (testing "a key with no `defsetting` cannot be written: how it is stored is the setting's to decide"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unknown setting: no-such-setting"
+ (#'setting/write-setting-value {:key "no-such-setting" :value "Sad Can"}))))))
+
(deftest previously-encrypted-settings-test
(testing "Make sure settings that were encrypted don't cause `user-facing-info` to blow up if encyrption key changed"
(mt/with-temp-empty-app-db [_conn :h2]
(mdb/setup-db! :create-sample-content? false)
- (mt/discard-setting-changes [test-json-setting]
- (encryption-test/with-secret-key "0B9cD6++AME+A7/oR7Y2xvPRHX3cHA2z7w+LbObd/9Y="
- (test-json-setting! {:abc 123})
- (is (not= "{\"abc\":123}"
- (actual-value-in-db :test-json-setting))))
- (testing (str "If fetching the Setting fails (e.g. because key changed) `user-facing-info` should return `nil` "
- "rather than failing entirely")
- (encryption-test/with-secret-key nil
+ ;; No `discard-setting-changes`: restoring the setting means writing it back without the key it was written
+ ;; under, and a write restores the whole settings cache over the row it cannot read. The temp app DB is thrown
+ ;; away either way.
+ (encryption-test/with-secret-key "0B9cD6++AME+A7/oR7Y2xvPRHX3cHA2z7w+LbObd/9Y="
+ (test-json-setting! {:abc 123})
+ (is (not= "{\"abc\":123}"
+ (raw-setting-value-with-aad :test-json-setting))))
+ (testing (str "If fetching the Setting fails (e.g. because key changed) `user-facing-info` should return `nil` "
+ "rather than failing entirely")
+ (encryption-test/with-secret-key nil
+ (binding [config/*disable-setting-cache* true]
(is (= {:key :test-json-setting
:value nil
:is_env_setting false
@@ -708,7 +761,7 @@
(testing "make sure uncached setting still saves to the DB"
(uncached-setting! "ABCDEF")
(is (= "ABCDEF"
- (actual-value-in-db "uncached-setting"))))
+ (raw-setting-value-with-aad "uncached-setting"))))
(testing "make sure that fetching the Setting always fetches the latest value from the DB"
(uncached-setting! "ABCDEF")
(t2/update! :model/Setting {:key "uncached-setting"}
@@ -1765,72 +1818,46 @@
(testing (format "We have defined a setting for the %s validation tests" format)
(is (var? (resolve (ns-validation-setting-symbol format)))))))))
-(deftest migrate-encrypted-settings!-works
- ;; Isolated app DB: with a secret key active this mutates the at-rest encryption of every registered setting row,
- ;; which would poison the shared test DB for later tests running with a different (or no) key.
- (mt/with-temp-empty-app-db [_conn :h2]
- (mdb/setup-db! :create-sample-content? false)
- (testing "It works when a secret key is set"
- (encryption-test/with-secret-key "ABCDEFGH12345678"
- (t2/insert! :setting {:key "test-never-encrypted-setting" :value (encryption/maybe-encrypt "foobar")})
- ;; Sanity check: the value is encrypted
- (is (not= "foobar" (actual-value-in-db :test-never-encrypted-setting)))
- (setting/migrate-encrypted-settings!)
- (is (= "foobar" (actual-value-in-db :test-never-encrypted-setting)))
- (setting/migrate-encrypted-settings!)
- (is (= "foobar" (actual-value-in-db :test-never-encrypted-setting)))))
- (testing "It doesn't do anything when the secret key is not set"
+(deftest migrate-settings!-test
+ (testing "rows a version predating `value_with_aad` wrote get it filled in from `value` on startup"
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
(encryption-test/with-secret-key "ABCDEFGH12345678"
+ (t2/delete! :setting :key "toucan-name")
(t2/delete! :setting :key "test-never-encrypted-setting")
- (t2/insert! :setting {:key "test-never-encrypted-setting" :value (encryption/maybe-encrypt "foobar")}))
- (encryption-test/with-secret-key nil
- (is (not= "foobar" (actual-value-in-db :test-never-encrypted-setting)))
- (setting/migrate-encrypted-settings!)
- (is (not= "foobar" (actual-value-in-db :test-never-encrypted-setting)))))))
-
-(deftest migrate-encrypted-settings!-does-not-depend-on-settings-cache-test
- ;; The cloud-migration read-only-mode guard (a `t2.pipeline/build :before` method registered when
- ;; `metabase.cloud-migration.models.cloud-migration` loads -- required above so this holds in a targeted test run
- ;; too) runs on every DML statement and reads a setting. On a fresh JVM that read triggers a full strict
- ;; settings-cache restore, which fails on the very plaintext row this function exists to repair -- so the repair
- ;; itself must never go through the cache. Regression test for the chicken-and-egg startup crash.
+ ;; `value` only, as that version writes it: encrypted for a setting that encrypts, plaintext for one that does not
+ (t2/insert! :setting [{:key "toucan-name", :value (encryption/encrypt "Lenny")}
+ {:key "test-never-encrypted-setting", :value "foobar"}])
+ (mdb.setting/migrate-settings!)
+ (testing "an encrypted row's value is stored under its AAD"
+ (is (= "Lenny" (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name)))))
+ (testing "a plaintext row's value is too -- every setting's is, whatever its :encryption says"
+ (is (= "foobar" (decrypt-setting-value :test-never-encrypted-setting
+ (raw-setting-value-with-aad :test-never-encrypted-setting)))))
+ (testing "and the values are readable again"
+ (setting.cache/restore-cache!)
+ (is (= "Lenny" (toucan-name))))
+ (testing "a row that already has a stored value is left byte-identical, whatever `value` says beside it"
+ (toucan-name! "Sad Can")
+ (let [before (raw-setting-value-with-aad :toucan-name)]
+ (t2/update! :setting :key "toucan-name" {:value (encryption/encrypt "Bird Can")})
+ (mdb.setting/migrate-settings!)
+ (is (= before (raw-setting-value-with-aad :toucan-name)))
+ (is (= "Sad Can" (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name))))))))))
+
+(deftest migrate-settings!-writes-past-the-read-only-mode-guard-test
+ ;; The cloud-migration guard on Toucan DML (registered when `metabase.cloud-migration.models.cloud-migration`
+ ;; loads -- required above so this holds in a targeted run too) reads `read-only-mode` through the model before
+ ;; every update, and the model's read is strict. The backfill's writes must not go through it: here that row holds a
+ ;; plaintext value under a key, which the strict read rejects, and the backfill still has to get its other rows done.
(mt/with-temp-empty-app-db [_conn :h2]
(mdb/setup-db! :create-sample-content? false)
(encryption-test/with-secret-key "ABCDEFGH12345678"
- (t2/insert! :setting {:key "toucan-name" :value "Lenny"})
- (binding [config/*disable-setting-cache* false]
- ;; Simulate a fresh JVM. `setting.cache/cache*` is the atom holding this app DB's in-memory settings map; nil
- ;; means never populated, so the next cached read must do the full (strictly decrypting) restore. And
- ;; `last-update-check` is the AtomicLong nanotime of the last staleness check: zeroing it defeats the
- ;; one-minute throttle that otherwise skips the check entirely in a warm test JVM.
- (reset! (#'setting.cache/cache*) nil)
- (.set ^java.util.concurrent.atomic.AtomicLong @#'setting.cache/last-update-check 0)
- (setting/migrate-encrypted-settings!))
- (is (encryption/decryptable-string? (actual-value-in-db :toucan-name)))
- (is (= "Lenny" (encryption/decrypt (actual-value-in-db :toucan-name)))))))
-
-(deftest migrate-encrypted-settings!-encrypts-strict-settings
- ;; raw :setting (not :model/Setting) throughout: the model's before-insert would encrypt the value, and these tests
- ;; need genuinely plaintext rows at rest. Isolated app DB for the same reason as [[migrate-encrypted-settings!-works]].
- (mt/with-temp-empty-app-db [_conn :h2]
- (mdb/setup-db! :create-sample-content? false)
- (testing "a plaintext row of a setting that encrypts is encrypted at rest on startup (e.g. after a downgraded boot decrypted it)"
- (encryption-test/with-secret-key "ABCDEFGH12345678"
- (t2/insert! :setting {:key "toucan-name" :value "Lenny"})
- (is (not (encryption/decryptable-string? (actual-value-in-db :toucan-name))))
- (setting/migrate-encrypted-settings!)
- (is (encryption/decryptable-string? (actual-value-in-db :toucan-name)))
- (is (= "Lenny" (encryption/decrypt (actual-value-in-db :toucan-name))))
- (testing "already-encrypted rows are left byte-identical"
- (let [before (actual-value-in-db :toucan-name)]
- (setting/migrate-encrypted-settings!)
- (is (= before (actual-value-in-db :toucan-name)))))))
- (testing "without an encryption key nothing happens"
- (encryption-test/with-secret-key nil
- (t2/delete! :setting :key "toucan-name")
- (t2/insert! :setting {:key "toucan-name" :value "Lenny"})
- (setting/migrate-encrypted-settings!)
- (is (= "Lenny" (actual-value-in-db :toucan-name)))))))
+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)
+ (t2/insert! :setting [{:key "read-only-mode", :value "false", :value_with_aad "false"}
+ {:key "toucan-name", :value (encryption/encrypt "Lenny")}])
+ (mdb.setting/migrate-settings!)
+ (is (= "Lenny" (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name)))))))
(deftest setter-none-does-not-imply-encryption-test
(testing "`:setter :none` does not imply encryption -- it is decided by type or stated explicitly, like any setting"
diff --git a/test/metabase/setup/core_test.clj b/test/metabase/setup/core_test.clj
index 36c0b1647f1..25f2c77e1e8 100644
--- a/test/metabase/setup/core_test.clj
+++ b/test/metabase/setup/core_test.clj
@@ -5,12 +5,14 @@
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.core :as mdb]
[metabase.app-db.encryption :as mdb.encryption]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.appearance.core :as appearance]
[metabase.config.core :as config]
[metabase.driver :as driver]
[metabase.models.interface :as mi]
[metabase.query-processor.middleware.cache-backend.interface :as i]
[metabase.settings.models.setting :as setting]
+ [metabase.settings.models.setting.cache :as setting.cache]
[metabase.setup.core :as setup]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -35,6 +37,7 @@
(testing "The has-user-setup getter should cache truthy results since it can never become falsey"
;; make sure some test users are created.
(mt/initialize-if-needed! :test-users)
+ (setting.cache/restore-cache!)
(t2/with-call-count [call-count]
;; call has-user-setup several times.
(dotimes [_ 5]
@@ -56,6 +59,7 @@
(is (<= (call-count)
10)))))) ;; in dev/test we check settings for an override
(testing "Switch back to the 'normal' app DB; value should still be cached for it"
+ (setting.cache/restore-cache!)
(t2/with-call-count [call-count]
(is (true?
(setup/has-user-setup)))
@@ -155,8 +159,8 @@
(reset! (:status mdb.connection/*application-db*) ::setup-finished)
(is (= :done (mdb/setup-db! :create-sample-content? false))))))))))
(testing "Database created with encryption configured is encrypted"
- (encryption-test/with-secret-key "key2"
- (mt/with-temp-empty-app-db [_conn driver/*driver*]
+ (mt/with-temp-empty-app-db [_conn driver/*driver*]
+ (encryption-test/with-secret-key "key2"
(mdb/setup-db! :create-sample-content? true)
(is (encryption/decryptable-string? (t2/select-one-fn :value "setting" :key "encryption-check")))
(is (encryption/decryptable-string? (t2/select-one-fn :details "metabase_database")))
@@ -194,8 +198,8 @@
(deftest sentinel-state-never-triggers-encryption-test
(testing "a restart must never encrypt a pre-existing plaintext row, whatever state the sentinel is in"
- (encryption-test/with-secret-key "sentinel-state-key-1"
- (mt/with-temp-empty-app-db [_conn :h2]
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (encryption-test/with-secret-key "sentinel-state-key-1"
(mdb/setup-db! :create-sample-content? true)
(let [db-id (t2/select-one-fn :id :metabase_database)
plaintext "{\"host\":\"example.com\"}"
@@ -226,8 +230,8 @@
(deftest fresh-install-with-encryption-key-test
(testing "a database created with MB_ENCRYPTION_SECRET_KEY set stores every encrypted-at-rest value encrypted"
- (encryption-test/with-secret-key "fresh-install-test-key-1234"
- (mt/with-temp-empty-app-db [_conn :h2]
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (encryption-test/with-secret-key "fresh-install-test-key-1234"
(mdb/setup-db! :create-sample-content? true)
(testing "encrypted-at-rest columns (read raw, so no model transform can hide a plaintext value)"
(doseq [[table column] @#'mdb.encryption/encrypted-string-columns
@@ -244,4 +248,8 @@
:let [definition (get @setting/registered-settings (keyword k))]
:when (and definition (not= :no (:encryption definition)))]
(testing k
- (is (encryption/decryptable-string? v)))))))))
+ (is (encryption/decryptable-string? v)))))
+ (testing "every setting's value_with_aad, under its own setting's AAD"
+ (doseq [{k :key v :value_with_aad} (t2/select :setting {:where [:!= :value_with_aad nil]})]
+ (testing k
+ (is (encryption/decryptable-string? v {:aad (mdb.setting/setting-aad k)})))))))))
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index c7386294922..260ec4c454c 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -18,6 +18,7 @@
[mb.hawk.parallel]
[metabase.analytics.prometheus :as prometheus]
[metabase.app-db.core :as mdb]
+ [metabase.app-db.setting :as mdb.setting]
[metabase.app-db.transient-error :as transient-error]
[metabase.audit-app.core :as audit]
[metabase.classloader.core :as classloader]
@@ -45,6 +46,7 @@
[metabase.test.util.log]
[metabase.timeline.models.timeline-event :as timeline-event]
[metabase.util :as u]
+ [metabase.util.encryption :as encryption]
[metabase.util.files :as u.files]
[metabase.util.json :as json]
[metabase.util.random :as u.random]
@@ -567,21 +569,31 @@
(list `with-temp-env-var-value! '[a])
(list `with-temp-env-var-value! '[a b c]))))
+(defn- raw-setting
+ "The `setting` row for `setting-k` as it sits in the table, or nil."
+ [setting-k]
+ (t2/select-one [:setting :value :value_with_aad] :key setting-k))
+
(defn- upsert-raw-setting!
- [original-value setting-k value]
+ "Write `value` for `setting-k` straight into the table, bypassing the model and so any setter: `value` bare, and
+ `value_with_aad` the way the model stores it, so the app reads the value back. A nil `value` removes the row."
+ [original setting-k value]
(if (some? value)
- (if original-value
- (t2/update! :model/Setting setting-k {:value value})
- (t2/insert! :model/Setting :key setting-k :value value))
- (when original-value
- (t2/delete! :model/Setting :key setting-k)))
+ (let [row {:value value
+ :value_with_aad (encryption/maybe-encrypt value {:aad (mdb.setting/setting-aad setting-k)})}]
+ (if original
+ (t2/update! :setting :key setting-k row)
+ (t2/insert! :setting (assoc row :key setting-k))))
+ (when original
+ (t2/delete! :setting :key setting-k)))
(setting.cache/restore-cache!))
(defn- restore-raw-setting!
- [original-value setting-k]
- (if original-value
- (t2/update! :model/Setting setting-k {:value original-value})
- (t2/delete! :model/Setting :key setting-k))
+ "Put back the row [[raw-setting]] found, byte for byte, or remove the one written over nothing."
+ [original setting-k]
+ (if original
+ (t2/update! :setting :key setting-k original)
+ (t2/delete! :setting :key setting-k))
(setting.cache/restore-cache!))
(defn do-with-temporary-setting-value!
@@ -610,7 +622,7 @@
(if (and (not raw-setting?) (setting/env-var-value setting-k))
(do-with-temp-env-var-value! (setting/setting-env-map-name setting-k) value thunk)
(let [original-value (if raw-setting?
- (t2/select-one-fn :value :model/Setting :key setting-k)
+ (raw-setting setting-k)
(if skip-init?
(setting/read-setting setting-k)
(setting/get setting-k)))]
diff --git a/test/metabase/util/encryption_test.clj b/test/metabase/util/encryption_test.clj
index ccaf4d84a8b..6b90d5110a3 100644
--- a/test/metabase/util/encryption_test.clj
+++ b/test/metabase/util/encryption_test.clj
@@ -13,17 +13,25 @@
(set! *warn-on-reflection* true)
+(defn- clear-setting-cache!
+ "Empty the Setting cache so values have to be fetched from the DB again. Emptied rather than restored: restoring
+ reads and decrypts every setting row there and then, which fails on rows written under a key other than the one in
+ effect at the boundary -- exactly what changing keys leaves behind. The once-a-minute throttle on the check that
+ repopulates an empty cache is zeroed too, so the next read restores it whole instead of going to the DB row by row."
+ []
+ (reset! (#'setting.cache/cache*) nil)
+ (.set ^java.util.concurrent.atomic.AtomicLong @#'setting.cache/last-update-check 0))
+
(defn do-with-secret-key! [^String secret-key thunk]
- ;; flush the Setting cache so unencrypted values have to be fetched from the DB again
(initialize/initialize-if-needed! :db)
- (setting.cache/restore-cache!)
+ (clear-setting-cache!)
(try
(with-redefs [encryption/default-secret-key (when (seq secret-key)
(encryption/secret-key->hash secret-key))]
(thunk))
(finally
- ;; reset the cache again so nothing that happened during the test is persisted.
- (setting.cache/restore-cache!))))
+ ;; clear the cache again so nothing that happened during the test is persisted.
+ (clear-setting-cache!))))
(defmacro with-secret-key
"Run `body` with the encryption secret key temporarily bound to `secret-key`. Useful for testing how functions behave
@@ -50,49 +58,49 @@
(deftest ^:parallel hash-pattern-test
(is (re= #"^[0-9A-Za-z/+]+=*$"
- (encryption/encrypt secret "Hello!"))))
+ (encryption/encrypt "Hello!" {:secret-key secret}))))
(deftest ^:parallel hashing-isnt-idempotent-test
(testing "test that encrypting something twice gives you two different ciphertexts"
- (is (not= (encryption/encrypt secret "Hello!")
- (encryption/encrypt secret "Hello!")))))
+ (is (not= (encryption/encrypt "Hello!" {:secret-key secret})
+ (encryption/encrypt "Hello!" {:secret-key secret})))))
(deftest ^:parallel decrypt-test
(testing "test that we can decrypt something"
(is (= "Hello!"
- (encryption/decrypt secret (encryption/encrypt secret "Hello!"))))))
+ (encryption/decrypt (encryption/encrypt "Hello!" {:secret-key secret}) {:secret-key secret})))))
(deftest ^:parallel decrypt-bytes-test
(testing "test that we can decrypt binary data"
(let [data (byte-array (range 0 100))]
(is (= (seq data)
- (seq (encryption/decrypt-bytes secret (encryption/encrypt-bytes secret data))))))))
+ (seq (encryption/decrypt-bytes (encryption/encrypt-bytes data {:secret-key secret}) {:secret-key secret})))))))
(deftest ^:parallel exception-with-wrong-decryption-key-test
(testing "trying to decrypt something with the wrong key with `decrypt` should throw an Exception"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Message seems corrupt or manipulated"
- (encryption/decrypt secret-2 (encryption/encrypt secret "WOW"))))))
+ (encryption/decrypt (encryption/encrypt "WOW" {:secret-key secret}) {:secret-key secret-2})))))
(deftest ^:parallel maybe-decrypt-not-encrypted-test
(testing "trying to `maybe-decrypt-accepting-plaintext` something that's not encrypted should return it as-is"
(is (= "{\"a\":100}"
- (encryption/maybe-decrypt-accepting-plaintext secret "{\"a\":100}")))
+ (encryption/maybe-decrypt-accepting-plaintext "{\"a\":100}" {:secret-key secret})))
(is (= "abc"
- (encryption/maybe-decrypt-accepting-plaintext secret "abc")))))
+ (encryption/maybe-decrypt-accepting-plaintext "abc" {:secret-key secret})))))
(deftest ^:parallel maybe-decrypt-with-wrong-key-test
(testing (str "decrypting something encrypted with a different key using `maybe-decrypt-accepting-plaintext` throws "
"rather than returning the ciphertext — returning it would let a re-encrypting caller double-encrypt it")
- (let [original-ciphertext (encryption/encrypt secret "WOW")]
+ (let [original-ciphertext (encryption/encrypt "WOW" {:secret-key secret})]
(is (thrown? Throwable
- (encryption/maybe-decrypt-accepting-plaintext secret-2 original-ciphertext))))))
+ (encryption/maybe-decrypt-accepting-plaintext original-ciphertext {:secret-key secret-2}))))))
(deftest ^:parallel no-errors-for-unencrypted-test
(testing "Something obviously not encrypted should avoiding trying to decrypt it (and thus not log an error)"
(mt/with-log-messages-for-level [messages :warn]
- (encryption/maybe-decrypt-accepting-plaintext secret "abc")
+ (encryption/maybe-decrypt-accepting-plaintext "abc" {:secret-key secret})
(is (empty? (messages))))))
(def ^:private fake-ciphertext
@@ -104,39 +112,39 @@
(deftest ^:parallel possibly-encrypted-test
(testing "a value shaped like ciphertext but that cannot be decrypted with the current key throws rather than being returned as-is"
(is (thrown? Throwable
- (encryption/maybe-decrypt-accepting-plaintext secret fake-ciphertext)))))
+ (encryption/maybe-decrypt-accepting-plaintext fake-ciphertext {:secret-key secret})))))
(deftest ^:parallel decryptable-string-test
(testing "true only for ciphertext that decrypts with the given key"
- (let [ciphertext (encryption/encrypt secret "WOW")]
- (is (true? (encryption/decryptable-string? secret ciphertext)))
+ (let [ciphertext (encryption/encrypt "WOW" {:secret-key secret})]
+ (is (true? (encryption/decryptable-string? ciphertext {:secret-key secret})))
(testing "false for ciphertext under another key"
- (is (false? (encryption/decryptable-string? secret-2 ciphertext))))
+ (is (false? (encryption/decryptable-string? ciphertext {:secret-key secret-2}))))
(testing "false with no key, even for genuine ciphertext"
- (is (false? (encryption/decryptable-string? nil ciphertext))))))
+ (is (false? (encryption/decryptable-string? ciphertext {:secret-key nil}))))))
(testing "false for anything that is not ciphertext"
- (is (false? (encryption/decryptable-string? secret "WOW")))
- (is (false? (encryption/decryptable-string? secret "")))
- (is (false? (encryption/decryptable-string? secret nil)))
+ (is (false? (encryption/decryptable-string? "WOW" {:secret-key secret})))
+ (is (false? (encryption/decryptable-string? "" {:secret-key secret})))
+ (is (false? (encryption/decryptable-string? nil {:secret-key secret})))
(testing "including plaintext shaped like ciphertext, which `possibly-encrypted-string?` cannot tell apart"
(is (true? (encryption/possibly-encrypted-string? fake-ciphertext)))
- (is (false? (encryption/decryptable-string? secret fake-ciphertext))))))
+ (is (false? (encryption/decryptable-string? fake-ciphertext {:secret-key secret}))))))
(deftest ^:parallel decryptable-bytes-test
(let [plaintext (codecs/to-bytes "WOW")
- ciphertext (encryption/encrypt-bytes secret plaintext)
+ ciphertext (encryption/encrypt-bytes plaintext {:secret-key secret})
fake-bytes (byte-array 64)]
(testing "true only for ciphertext that decrypts with the given key"
- (is (true? (encryption/decryptable-bytes? secret ciphertext)))
- (is (false? (encryption/decryptable-bytes? secret-2 ciphertext)))
- (is (false? (encryption/decryptable-bytes? nil ciphertext))))
+ (is (true? (encryption/decryptable-bytes? ciphertext {:secret-key secret})))
+ (is (false? (encryption/decryptable-bytes? ciphertext {:secret-key secret-2})))
+ (is (false? (encryption/decryptable-bytes? ciphertext {:secret-key nil}))))
(testing "false for anything that is not ciphertext"
- (is (false? (encryption/decryptable-bytes? secret plaintext)))
- (is (false? (encryption/decryptable-bytes? secret (byte-array 0))))
- (is (false? (encryption/decryptable-bytes? secret nil)))
+ (is (false? (encryption/decryptable-bytes? plaintext {:secret-key secret})))
+ (is (false? (encryption/decryptable-bytes? (byte-array 0) {:secret-key secret})))
+ (is (false? (encryption/decryptable-bytes? nil {:secret-key secret})))
(testing "including bytes shaped like ciphertext"
(is (true? (encryption/possibly-encrypted-bytes? fake-bytes)))
- (is (false? (encryption/decryptable-bytes? secret fake-bytes)))))))
+ (is (false? (encryption/decryptable-bytes? fake-bytes {:secret-key secret})))))))
(deftest ^:parallel possibly-encrypted-returns-booleans-test
(testing "the shape checks never return nil"
@@ -145,63 +153,63 @@
(is (false? (encryption/possibly-encrypted-string? "not base64!")))
(is (false? (encryption/possibly-encrypted-bytes? nil)))
(is (false? (encryption/possibly-encrypted-bytes? (byte-array 3))))
- (is (true? (encryption/possibly-encrypted-string? (encryption/encrypt secret "WOW"))))))
+ (is (true? (encryption/possibly-encrypted-string? (encryption/encrypt "WOW" {:secret-key secret}))))))
(deftest ^:parallel maybe-decrypt-strict-test
(testing "strict `maybe-decrypt`"
(testing "decrypts a genuinely encrypted value"
- (is (= "WOW" (encryption/maybe-decrypt secret (encryption/encrypt secret "WOW")))))
+ (is (= "WOW" (encryption/maybe-decrypt (encryption/encrypt "WOW" {:secret-key secret}) {:secret-key secret}))))
(testing "throws on a value that is not encrypted"
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"not encrypted"
- (encryption/maybe-decrypt secret "{\"a\":100}")))
+ (encryption/maybe-decrypt "{\"a\":100}" {:secret-key secret})))
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"not encrypted"
- (encryption/maybe-decrypt secret "abc"))))
+ (encryption/maybe-decrypt "abc" {:secret-key secret}))))
(testing "throws on a value that looks encrypted but fails to decrypt (wrong key or corrupt)"
- (is (thrown? Throwable (encryption/maybe-decrypt secret fake-ciphertext)))
- (is (thrown? Throwable (encryption/maybe-decrypt secret-2 (encryption/encrypt secret "WOW")))))
+ (is (thrown? Throwable (encryption/maybe-decrypt fake-ciphertext {:secret-key secret})))
+ (is (thrown? Throwable (encryption/maybe-decrypt (encryption/encrypt "WOW" {:secret-key secret}) {:secret-key secret-2}))))
(testing "decrypts an encrypted byte array"
- (is (= "WOW" (String. ^bytes (encryption/maybe-decrypt-bytes secret (encryption/encrypt-bytes secret (.getBytes "WOW")))))))
+ (is (= "WOW" (String. ^bytes (encryption/maybe-decrypt-bytes (encryption/encrypt-bytes (.getBytes "WOW") {:secret-key secret}) {:secret-key secret})))))
(testing "passes nil through but rejects a blank (non-encrypted) string"
(is (nil? (encryption/maybe-decrypt nil)))
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"not encrypted"
- (encryption/maybe-decrypt secret ""))))))
+ (encryption/maybe-decrypt "" {:secret-key secret}))))))
(deftest ^:parallel stream-encryption-test
(testing "Can encrypt stream"
(let [input-stream (ByteArrayInputStream. (.getBytes "test string"))
- encrypted-stream (encryption/encrypt-stream secret input-stream)
+ encrypted-stream (encryption/encrypt-stream input-stream {:secret-key secret})
output-string (slurp encrypted-stream)]
(is (not= "test string" output-string))))
(testing "Can encrypt and decrypt streams"
(let [input-stream (ByteArrayInputStream. (.getBytes "test string"))]
- (with-open [encrypted-stream (encryption/encrypt-stream secret input-stream)
- decrypted-stream (encryption/maybe-decrypt-stream secret encrypted-stream)]
+ (with-open [encrypted-stream (encryption/encrypt-stream input-stream {:secret-key secret})
+ decrypted-stream (encryption/maybe-decrypt-stream encrypted-stream {:secret-key secret})]
(is (= "test string" (slurp decrypted-stream))))))
(testing "Can encrypt and decrypt a large stream"
(let [data (string/random-string 100000)
input-stream (ByteArrayInputStream. (codecs/to-bytes data))]
- (with-open [encrypted-stream (encryption/encrypt-stream secret input-stream)
- decrypted-stream (encryption/maybe-decrypt-stream secret encrypted-stream)]
+ (with-open [encrypted-stream (encryption/encrypt-stream input-stream {:secret-key secret})
+ decrypted-stream (encryption/maybe-decrypt-stream encrypted-stream {:secret-key secret})]
(is (= data (codecs/bytes->str (IOUtils/toByteArray decrypted-stream)))))))
(testing "Unencrypted streams come back as-is"
(let [input-stream (ByteArrayInputStream. (codecs/to-bytes "test string"))]
- (with-open [decrypted-stream (encryption/maybe-decrypt-stream secret input-stream)]
+ (with-open [decrypted-stream (encryption/maybe-decrypt-stream input-stream {:secret-key secret})]
(is (= "test string" (codecs/bytes->str (IOUtils/toByteArray decrypted-stream)))))))
(testing "Empty unencrypted streams come back as-is"
(let [input-stream (ByteArrayInputStream. (byte-array 0))]
- (with-open [decrypted-stream (encryption/maybe-decrypt-stream secret input-stream)]
+ (with-open [decrypted-stream (encryption/maybe-decrypt-stream input-stream {:secret-key secret})]
(is (= -1 (.read decrypted-stream))))))
(testing "Long unencrypted streams come back as-is"
(let [data (string/random-string 100000)
input-stream (ByteArrayInputStream. (codecs/to-bytes data))]
- (with-open [decrypted-stream (encryption/maybe-decrypt-stream secret input-stream)]
+ (with-open [decrypted-stream (encryption/maybe-decrypt-stream input-stream {:secret-key secret})]
(is (= data (codecs/bytes->str (IOUtils/toByteArray decrypted-stream))))))))
(deftest ^:parallel maybe-encrypt-for-stream-test
(testing "When secret is set, it encrypts the stream"
- (let [encrypted (encryption/maybe-encrypt-for-stream secret (codecs/to-bytes "test string"))]
+ (let [encrypted (encryption/maybe-encrypt-for-stream (codecs/to-bytes "test string") {:secret-key secret})]
(is (not= "test string" (codecs/bytes->str encrypted)))
- (is (= "test string" (slurp (encryption/maybe-decrypt-stream secret (ByteArrayInputStream. encrypted))))))
+ (is (= "test string" (slurp (encryption/maybe-decrypt-stream (ByteArrayInputStream. encrypted) {:secret-key secret})))))
(testing "When secret is not set, it does not encrypt the stream"
- (let [encrypted (encryption/maybe-encrypt-for-stream nil (codecs/to-bytes "test string"))]
+ (let [encrypted (encryption/maybe-encrypt-for-stream (codecs/to-bytes "test string") {:secret-key nil})]
(is (= "test string" (codecs/bytes->str encrypted)))))))
diff --git a/test_resources/error-migration.yaml b/test_resources/error-migration.yaml
index dd1c45ea346..120b2e25b3a 100644
--- a/test_resources/error-migration.yaml
+++ b/test_resources/error-migration.yaml
@@ -20,6 +20,9 @@ databaseChangeLog:
- column:
name: value
type: text
+ - column:
+ name: value_with_aad
+ type: text
- changeSet:
id: '1'