The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Store setting values encrypted under the setting's name (#81751)
Alexander Polyankina9a062731bec
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 792ece6acfb..c21d01d18a2 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -349,6 +349,7 @@:api #{metabase.app-db.cluster-lock ; TODO FIXMEmetabase.app-db.coremetabase.app-db.init+ metabase.app-db.settingmetabase.app-db.setupmetabase.app-db.transient-error}:uses #{auth-provider@@ -2235,8 +2236,7 @@settings{:team "DevEx"- :api #{metabase.settings.core- metabase.settings.init}+ :api #{metabase.settings.core metabase.settings.init}:uses #{apiapp-dbconfigdiff --git a/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.clj b/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.cljindex cbbb33cbb69..8297273dc8d 100644--- a/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.clj+++ b/enterprise/backend/src/metabase_enterprise/sso/integrations/token_utils.clj@@ -22,7 +22,7 @@expiration (t/instant (t/plus timestamp (t/seconds 300)))nonce (random-uuid)payload (str (.getEpochSecond timestamp) "." (.getEpochSecond expiration) "." nonce)- encrypted (encryption/encrypt (hashed-key) payload)]+ encrypted (encryption/encrypt payload {:secret-key (hashed-key)})](URLEncoder/encode encrypted "UTF-8")))(defn validate-token@@ -33,7 +33,7 @@(not-empty token)(try(let [decoded-token (URLDecoder/decode ^String token "UTF-8")- decrypted-payload (encryption/decrypt (hashed-key) decoded-token)+ decrypted-payload (encryption/decrypt decoded-token {:secret-key (hashed-key)})[_ expiration _] (str/split decrypted-payload #"\." 3)](t/< (t/instant) (Instant/ofEpochSecond (Long/parseLong expiration))))(catch Exception _diff --git a/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.clj b/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.cljindex 3efff0d4798..5ba3cdea521 100644--- a/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sso/integrations/token_utils_test.clj@@ -5,12 +5,15 @@[java-time.api :as t][metabase-enterprise.sso.integrations.token-utils :as token-utils][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][metabase.util.encryption :as encryption])(:import(java.net URLDecoder URLEncoder)))(set! *warn-on-reflection* true)+(use-fixtures :once (fixtures/initialize :db))+(deftest generate-token-test(testing "generate-token"(testing "should generate a URL-encoded string"@@ -24,7 +27,7 @@(mt/with-temporary-setting-values [sdk-encryption-validation-key "1FlZMdousOLX9d3SSL+KuWq2+l1gfKoFM7O4ZHqKjTgabo7QdqP8US2bNPN+PqisP1QOKvesxkxOigIrvvd5OQ=="](let [token (token-utils/generate-token)decoded-token (URLDecoder/decode token "UTF-8")- decrypted (encryption/decrypt (encryption/secret-key->hash "1FlZMdousOLX9d3SSL+KuWq2+l1gfKoFM7O4ZHqKjTgabo7QdqP8US2bNPN+PqisP1QOKvesxkxOigIrvvd5OQ==") decoded-token)+ decrypted (encryption/decrypt decoded-token {:secret-key (encryption/secret-key->hash "1FlZMdousOLX9d3SSL+KuWq2+l1gfKoFM7O4ZHqKjTgabo7QdqP8US2bNPN+PqisP1QOKvesxkxOigIrvvd5OQ==")})[ts exp nonce] (str/split decrypted #"\." 3)timestamp (Long/parseLong ts)expiration (Long/parseLong exp)]@@ -49,7 +52,7 @@expiration (t/instant (t/plus now (t/seconds 300)))nonce (random-uuid)payload (str (.getEpochSecond now) "." (.getEpochSecond expiration) "." nonce)- encrypted (encryption/encrypt encryption-key payload)+ encrypted (encryption/encrypt payload {:secret-key encryption-key})token (URLEncoder/encode encrypted "UTF-8")](is (true? (token-utils/validate-token token)))))(testing "returns false for expired token"@@ -57,7 +60,7 @@expiration (t/instant (t/minus now (t/seconds 10))) ;; 10 seconds in the pastnonce (random-uuid)payload (str (.getEpochSecond now) "." (.getEpochSecond expiration) "." nonce)- encrypted (encryption/encrypt encryption-key payload)+ encrypted (encryption/encrypt payload {:secret-key encryption-key})token (URLEncoder/encode encrypted "UTF-8")](is (false? (token-utils/validate-token token)))))(testing "returns false for nil token"@@ -71,6 +74,6 @@expiration (t/instant (t/plus now (t/seconds 300)))nonce (random-uuid)payload (str (.getEpochSecond now) "." (.getEpochSecond expiration) "." nonce)- encrypted (encryption/encrypt encryption-key payload)+ encrypted (encryption/encrypt payload {:secret-key encryption-key})token (URLEncoder/encode (str encrypted "tampered") "UTF-8")](is (false? (token-utils/validate-token token)))))))))diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex a6713d7df63..8e539237ba2 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -1,5 +1,4 @@(ns metabase.driver.bigquery-cloud-sdk- (:refer-clojure :exclude [mapv some empty? not-empty])(:require[clojure.core.async :as a][clojure.set :as set]@@ -29,7 +28,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [every? mapv some empty? not-empty]]+ [metabase.util.performance :as perf];; sync fix-ups read and write Database/Table/Field rows directly; the metadata provider is read-only^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2])@@ -98,7 +97,7 @@chosen by whoever supplied the credentials rather than a fixed Google endpoint."[service-account-json](or (when-not (str/blank? service-account-json)- (not-empty (get (json/decode service-account-json) "token_uri")))+ (perf/not-empty (get (json/decode service-account-json) "token_uri")))default-token-uri))(defmethod driver/connection-hosts :bigquery-cloud-sdk@@ -131,7 +130,7 @@(.setTransportOptions transport-options))](when universe-domain(.setUniverseDomain bq-bldr universe-domain))- (when-let [host (not-empty (:host details))]+ (when-let [host (perf/not-empty (:host details))](.setHost bq-bldr host))(.. bq-bldr build getService)))@@ -170,7 +169,7 @@[{:keys [dataset-filters-type dataset-filters-patterns]}](when (= "inclusion" dataset-filters-type)(let [segments (map str/trim (str/split (str dataset-filters-patterns) #","))]- (when (every? #(re-matches exact-dataset-id-re %) segments)+ (when (perf/every? #(re-matches exact-dataset-id-re %) segments);; a scan yields each dataset once however many segments matched it(distinct segments)))))@@ -434,7 +433,7 @@[dataset-id {data-type :data_type field-path-str :field_path table-name :table_name}](let [field-path (str/split field-path-str #"\.")[database-type base-type] (raw-type->database+base-type data-type)]- (when-let [nfc-path (not-empty (pop field-path))]+ (when-let [nfc-path (perf/not-empty (pop field-path))]{:name (peek field-path):table-name table-name:table-schema dataset-id@@ -748,9 +747,9 @@lifted by matching the names in `fields` to the names in the table schema."[^Table bq-table fields rff](let [^Schema schema (.. bq-table getDefinition getSchema)- field-idxs (mapv :database_position fields)+ field-idxs (perf/mapv :database_position fields)all-parsers (get-field-parsers schema)- parsers (mapv all-parsers field-idxs)+ parsers (perf/mapv all-parsers field-idxs)probe (list-sample-page bq-table (min (long initial-page-rows) table-rows-sample/max-sample-rows) nil)](transduce(comp (take table-rows-sample/max-sample-rows)@@ -996,7 +995,7 @@(dissoc :database-type :database-position)))cols {:cols columns}results (eduction (map (fn [^FieldValueList row]- (mapv parse-field-value row parsers)))+ (perf/mapv parse-field-value row parsers)))(reducible-bigquery-results page cancel-chan attempt-job-cancel-fn(adaptive-query-next-page job)))](respond cols results)))@@ -1204,7 +1203,7 @@(defmethod driver/normalize-db-details :bigquery-cloud-sdk[_driver database](let [details (driver.conn/default-details database)]- (when-not (empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))+ (when-not (perf/empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))(log/errorf (str "Database ID %d, which was migrated from the legacy :bigquery driver to :bigquery-cloud-sdk, has"" one or more OAuth style authentication scheme parameters saved to db-details, which cannot"" be automatically migrated to the newer driver (since it *requires* service-account-json instead);"@@ -1237,7 +1236,7 @@(defn- clustering-clause"Inline `CLUSTER BY col, ...` clause for a table's `indexes`, or nil when there's no clustering."[indexes]- (when-let [{:keys [columns]} (some #(when (= :clustering (:kind %)) %) indexes)]+ (when-let [{:keys [columns]} (perf/some #(when (= :clustering (:kind %)) %) indexes)](let [cols (str/join ", " (map #(sql.u/quote-name :bigquery-cloud-sdk :field (:name %)) columns))](format "CLUSTER BY %s" cols))))@@ -1253,7 +1252,7 @@(let [definition (.getDefinition bq-table)](when (instance? StandardTableDefinition definition)(when-let [^Clustering clustering (.getClustering ^StandardTableDefinition definition)]- (not-empty (vec (.getFields clustering))))))))))+ (perf/not-empty (vec (.getFields clustering))))))))))(defmethod driver/fetch-table-indexes :bigquery-cloud-sdk[_driver database schema table]@@ -1370,17 +1369,17 @@;; rather than waiting many minutes, we trade torward consistency by using SQL DML, whose table metadata;; is consistent, and we do not see cached non-existence and things like that causing trouble.(let [database (t2/select-one :model/Database db-id)- col-kws (mapv (comp keyword name :name) columns)+ col-kws (perf/mapv (comp keyword name :name) columns)num-cols (count col-kws);; bigquery allows 10k query parameters per requestmax-rows (max 1 (quot 10000 num-cols))chunk-size (min (or driver/*insert-chunk-rows* 1000) max-rows)](doseq [chunk (partition-all chunk-size data):let [lift #(if (coll? %) [:lift %] %)- lift-tuple #(mapv lift %)]]+ lift-tuple #(perf/mapv lift %)]](let [[sql & params] (sql.qp/format-honeysql driver {:insert-into table-name:columns col-kws- :values (mapv lift-tuple chunk)})]+ :values (perf/mapv lift-tuple chunk)})](driver/execute-raw-queries! driver database [[sql params]])))))(defmethod driver/execute-raw-queries! :bigquery-cloud-sdk@@ -1445,7 +1444,7 @@driver-api/databasedriver.conn/effective-detailslist-datasets- (some #{schema}))))+ (perf/some #{schema}))))(defmethod driver/table-name-length-limit :bigquery-cloud-sdk[_driver]diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yamlindex ded4a7c9ef3..dda4867225e 100644--- a/resources/migrations/058_update_migrations.yaml+++ b/resources/migrations/058_update_migrations.yaml@@ -1660,6 +1660,28 @@ databaseChangeLog:- customChange:class: metabase.app_db.custom_migrations.EncryptSetterNoneSettingsV58+ - changeSet:+ id: v58.2026-09-01T00:00:00+ author: ranquild+ comment: >-+ Add setting.value_with_aad, which stores a setting's value encrypted under additional authenticated data naming+ the setting (setting.<key>), so that a value moved between rows fails to decrypt: the bare value in+ setting.value, or its ciphertext, says nothing about which setting it belongs to, and a direct DB write could+ give one setting another's value. value is kept: this version writes both columns and reads only+ value_with_aad, so a version that predates the column keeps working through a rolling upgrade, and a rollback+ is just dropping value_with_aad again. The column is filled in from value after migrations, by+ metabase.app-db.setting/migrate-settings!, which also repairs rows such a version has written since.+ changes:+ - addColumn:+ tableName: setting+ columns:+ - column:+ name: value_with_aad+ type: ${text.type}+ remarks: "The setting's value, encrypted under additional authenticated data naming the setting"+ constraints:+ nullable: true+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex 6ed291214ce..24667337c76 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -73,6 +73,7 @@spec][metabase.app-db.querycompile+ current-timestamp-stringisajoinqualifydiff --git a/src/metabase/app_db/encryption.clj b/src/metabase/app_db/encryption.cljindex 46daa382f11..0dcc4045863 100644--- a/src/metabase/app_db/encryption.clj+++ b/src/metabase/app_db/encryption.clj@@ -1,8 +1,9 @@(ns metabase.app-db.encryption(:require+ [metabase.app-db.query :as mdb.query]+ [metabase.app-db.setting :as mdb.setting][metabase.util :as u][metabase.util.encryption :as encryption]- [metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [trs]][metabase.util.log :as log][metabase.util.malli :as mu]@@ -72,7 +73,10 @@:invalid - the sentinel exists but does not decrypt (wrong or unset key, corruption):absent - no sentinel: a `setting` table that does not exist yet (before migrations on a fresh database), norow, or the plaintext \"unencrypted\" marker (inserted by the v53 migration, and written back when- the database is decrypted) -- an explicit statement of the same thing a missing row means"+ the database is decrypted) -- an explicit statement of the same thing a missing row means++ Read raw, from the legacy `value` column: this runs before migrations, and `value` is the one column every version+ writes the sentinel to."[](let [raw (u/ignore-exceptions (t2/select-one-fn :value :setting :key encryption-check-key))](cond@@ -133,8 +137,8 @@This only runs in the one-shot \"no sentinel but content exists\" state, so it can afford to stream every columnfully (stopping at the first value that does not decrypt) rather than sampling; a partially encrypted column can- therefore never read as `:decryptable`. The `setting` table is not counted: whether a given setting is encrypted at- rest is decided per setting, so its values prove nothing about the database."+ therefore never read as `:decryptable`. The `setting` table is not counted: whether a setting's legacy `value` is+ encrypted at rest is decided per setting, and `value_with_aad` only decrypts under each row's own AAD."[](reduce (fn [acc [table column decryptable?]](case (column-content-status table column decryptable?)@@ -146,21 +150,31 @@(map #(conj % (comp encryption/decryptable-bytes? maybe-blob->bytes)) encrypted-bytes-columns))))(defn- replace-encryption-check!- "Replace the `encryption-check` sentinel on `conn`: with a fresh UUID encrypted by `encrypt-fn`, or with the- plaintext \"unencrypted\" marker when `encrypt-fn` is nil (the database is being decrypted)."- [conn encrypt-fn]- (t2/delete! :conn conn :setting :key encryption-check-key)- (t2/insert! :conn conn :setting {:key encryption-check-key- :value (if encrypt-fn- (encrypt-fn (str (random-uuid)))- "unencrypted")}))+ "Replace the `encryption-check` sentinel on `conn`: with a fresh UUID encrypted by `encrypt-fn` -- and, in+ `value_with_aad`, by `encrypt-setting-fn`, a function of a string and the setting key -- or with the plaintext+ \"unencrypted\" marker in both columns when they are nil (the database is being decrypted). Written to+ `value_with_aad` and to the legacy `value` both, so that a version predating `value_with_aad` reads the same answer+ from the same database."+ [conn encrypt-fn encrypt-setting-fn]+ (let [sentinel (if encrypt-fn (str (random-uuid)) "unencrypted")]+ (t2/delete! :conn conn :setting :key encryption-check-key)+ (t2/insert! :conn conn :setting {:key encryption-check-key+ :value (cond-> sentinel encrypt-fn encrypt-fn)+ :value_with_aad (cond-> sentinel encrypt-setting-fn (encrypt-setting-fn encryption-check-key))})))++(defn- encrypt-setting+ "A function of a string and a setting key that encrypts the string the way `setting.value_with_aad` holds it: under+ that setting's AAD, and under `secret-key` when given, the current MB_ENCRYPTION_SECRET_KEY otherwise."+ [secret-key]+ (fn [s setting-key]+ (encryption/maybe-encrypt s {:secret-key secret-key, :aad (mdb.setting/setting-aad setting-key)})))(defn- write-encryption-check!"Record that the database is encrypted under the current MB_ENCRYPTION_SECRET_KEY by replacing the `encryption-check`sentinel with a fresh UUID encrypted under it. Only ever writes the sentinel -- never touches any other row."[](t2/with-transaction [conn]- (replace-encryption-check! conn encryption/encrypt)))+ (replace-encryption-check! conn encryption/encrypt (encrypt-setting nil))))(def ^:private EncryptionState[:enum :encrypted :unencrypted :fresh :pre-sentinel :missing-key :wrong-key :not-decryptable])@@ -302,15 +316,24 @@(when (and (string? value)(not (encryption/decryptable-string? value)))(t2/update! :conn conn table {:id id} {column (encryption/encrypt value)})))- (t2/reducible-select [table :id [column :value]] {:where [:!= column nil]}))))))+ (t2/reducible-select [table :id [column :value]] {:where [:!= column nil]})))+ ;; `setting.value_with_aad` is bound to its row, so it is checked and encrypted under each row's own AAD+ (let [encrypt-setting-fn (encrypt-setting nil)]+ (run! (fn [{:keys [key value_with_aad]}]+ (when (and (string? value_with_aad)+ (not (encryption/decryptable-string? value_with_aad {:aad (mdb.setting/setting-aad key)})))+ (t2/update! :conn conn :setting {:key key} {:value_with_aad (encrypt-setting-fn value_with_aad key)})))+ (t2/reducible-select [:setting :key :value_with_aad] {:where [:!= :value_with_aad nil]}))))))(defn- do-encryption- "Encrypt or decrypts the db using the current `MB_ENCRYPTION_SECRET_KEY` to read data.-- The passed make-encrypt-fn is used to generate the encryption/decryption function to use by passing versions of encryption/maybe-encrypt to it."- [db-type data-source encrypting? make-encrypt-fn]- (let [encrypt-str-fn (make-encrypt-fn encryption/maybe-encrypt)- encrypt-bytes-fn (make-encrypt-fn encryption/maybe-encrypt-bytes)]+ "Encrypt or decrypt the db using the current `MB_ENCRYPTION_SECRET_KEY` to read data.++ When `encrypting?`, every value is re-encrypted under `to-key` (already hashed), or under the current+ MB_ENCRYPTION_SECRET_KEY when that is nil; otherwise every value is written back decrypted."+ [db-type data-source encrypting? to-key]+ (let [encrypt-str-fn (if encrypting? #(encryption/maybe-encrypt % {:secret-key to-key}) identity)+ encrypt-bytes-fn (if encrypting? #(encryption/maybe-encrypt-bytes % {:secret-key to-key}) identity)+ encrypt-setting-fn (if encrypting? (encrypt-setting to-key) (fn [s _setting-key] s))](t2/with-transaction [conn {:datasource data-source}](let [check-status (encryption-check-status)](when (= check-status :invalid)@@ -324,16 +347,23 @@;; a setting that is plaintext at rest while a key is configured (e.g. one newly designated encrypted but not yet;; re-encrypted) is exactly what this operation exists to fix, so we decrypt it leniently here rather than reject;; it. A value that looks encrypted but can't be decrypted with the current key still aborts.- (doseq [[key value] (t2/select-fn->fn :key :value :setting)]+ ;;+ ;; Both columns are re-encrypted: `value_with_aad` under each row's own AAD, and `value` because a version+ ;; predating that column reads it, so a rollback must not land on ciphertext under the old key.+ (doseq [{:keys [key value value_with_aad]} (t2/select :setting)](case key- "settings-last-updated" (let [current-timestamp-as-string-honeysql (h2x/cast (if (= db-type :mysql) :char :text)- (h2x/current-datetime-honeysql-form db-type))]- (t2/update! :conn conn :setting {:key key} {:value current-timestamp-as-string-honeysql}))+ "settings-last-updated" (let [now (mdb.query/current-timestamp-string db-type)]+ (t2/update! :conn conn :setting {:key key}+ {:value now+ :value_with_aad (encrypt-setting-fn now key)}))"encryption-check" nil- (t2/update! :conn conn :setting- {:key key}- {:value (encrypt-str-fn (encryption/maybe-decrypt-accepting-plaintext value))})))- (replace-encryption-check! conn (when encrypting? encrypt-str-fn))+ (let [aad-opts {:aad (mdb.setting/setting-aad key)}+ changes (cond-> {}+ (seq value) (assoc :value (encrypt-str-fn (encryption/maybe-decrypt-accepting-plaintext value)))+ (seq value_with_aad) (assoc :value_with_aad (encrypt-setting-fn (encryption/maybe-decrypt-accepting-plaintext value_with_aad aad-opts) key)))]+ (when (seq changes)+ (t2/update! :conn conn :setting {:key key} changes)))))+ (replace-encryption-check! conn (when encrypting? encrypt-str-fn) (when encrypting? encrypt-setting-fn))(doseq [[table column] encrypted-bytes-columns](reencrypt-encrypted-bytes-column! conn table column encrypt-bytes-fn))(t2/delete! :conn conn :model/QueryCache))))@@ -346,12 +376,9 @@(throw (ex-info "Cannot encrypt database with an empty key" {})))(when (and (nil? to-key) (not (encryption/default-encryption-enabled?)))(throw (ex-info "Cannot encrypt database: MB_ENCRYPTION_SECRET_KEY is not set" {})))- (do-encryption db-type data-source true (fn [maybe-encrypt-fn]- (if- (nil? to-key) maybe-encrypt-fn- (partial maybe-encrypt-fn (encryption/validate-and-hash-secret-key to-key))))))+ (do-encryption db-type data-source true (some-> to-key encryption/validate-and-hash-secret-key)))(defn decrypt-db"Decrypts the database using the current `MB_ENCRYPTION_SECRET_KEY` to read existing data"[db-type data-source]- (do-encryption db-type data-source false (constantly identity)))+ (do-encryption db-type data-source false nil))diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.cljindex 70c019adfe8..5580cdef999 100644--- a/src/metabase/app_db/query.clj+++ b/src/metabase/app_db/query.clj@@ -24,6 +24,7 @@[honey.sql :as sql][metabase.app-db.format :as app-db.format][metabase.util :as u]+ [metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][toucan2.core :as t2]@@ -116,6 +117,14 @@e))))]sql-args))+(defn current-timestamp-string+ "The application DB's own current timestamp, as a string. Read from the DB rather than from this machine's clock,+ for the `settings-last-updated` marker that instances compare against each other."+ ^String [db-type]+ ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)+ (let [cast-form (h2x/cast (if (= db-type :mysql) :char :text) (h2x/current-datetime-honeysql-form db-type))]+ (-> (t2/query-one {:select [[cast-form :timestamp]]}) :timestamp)))+;;; TODO -- we should mark this deprecated and tell people to use [[toucan2.core/query]] directly instead(defn query"Replacement for [[toucan.db/query]] -- uses Honey SQL 2 instead of Honey SQL 1, to ease the transition to thediff --git a/src/metabase/app_db/setting.clj b/src/metabase/app_db/setting.cljnew file mode 100644index 00000000000..13016abdf2c--- /dev/null+++ b/src/metabase/app_db/setting.clj@@ -0,0 +1,50 @@+(ns metabase.app-db.setting+ "How the application DB stores a setting's value: `setting.value_with_aad` holds it encrypted under additional+ authenticated data naming the setting, so that a value moved between rows -- whose ciphertext alone says nothing+ about which setting it belongs to -- fails to decrypt. The legacy `value` column holds it bare, for versions that+ predate the other. Lives here rather than with the Setting model because the encryption tooling here writes two rows+ the model never does: the `settings-last-updated` marker and the `encryption-check` sentinel."+ (:require+ [buddy.core.codecs :as codecs]+ [metabase.util :as u]+ [metabase.util.encryption :as encryption]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^{:arglists '(^bytes [setting-key])} setting-aad+ "The additional authenticated data a setting's value is encrypted under: `setting.<key>`."+ (memoize (fn ^bytes [setting-key] (codecs/to-bytes (str "setting." setting-key)))))++(defn migrate-settings!+ "Give every setting row that has no `value_with_aad` one, from the legacy `value` column beside it, stored the way+ the Setting model stores one. Runs from `metabase.app-db.setup/setup-db!` after migrations and before anything reads+ a setting, whenever the database's encryption state says every row can be read -- a row this cannot decrypt would+ otherwise be stored as if it were a value.++ Only an empty `value_with_aad` is filled in: a row a version predating the column wrote, on the first start after+ the upgrade or while such a version runs alongside this one. A row that has one is left exactly as it is, whatever+ `value` says beside it -- nothing in this version reads `value`, and a version that does reconciles it at its own+ startup.++ Reads and writes the table directly, never through the model: the model's read is the strict one this fills in+ for, and the cloud-migration guard on Toucan DML reads `read-only-mode` through it before every update -- a row that+ may itself be among those being filled in."+ []+ (let [filled (atom 0)]+ (t2/with-transaction [_conn]+ (doseq [{:keys [key value]} (t2/select :setting+ {:where [:and+ [:= :value_with_aad nil]+ [:not= :value nil]+ [:not= :value ""]]+ :for :update})+ :let [plain (u/ignore-exceptions (encryption/maybe-decrypt-accepting-plaintext value))]+ :when (some? plain)]+ (swap! filled inc)+ (t2/query {:update :setting+ :set {:value_with_aad (encryption/maybe-encrypt plain {:aad (setting-aad key)})}+ :where [:= :key key]})))+ (when (pos? @filled)+ (log/infof "Filled in the authenticated value of %d setting(s) from their value." @filled))))diff --git a/src/metabase/app_db/setup.clj b/src/metabase/app_db/setup.cljindex f517d80cc8a..219a17d3abb 100644--- a/src/metabase/app_db/setup.clj+++ b/src/metabase/app_db/setup.clj@@ -17,6 +17,7 @@[metabase.app-db.encryption :as mdb.encryption][metabase.app-db.jdbc-protocols :as mdb.jdbc-protocols][metabase.app-db.liquibase :as liquibase]+ [metabase.app-db.setting :as mdb.setting][metabase.config.core :as config][metabase.util :as u][metabase.util.honey-sql-2]@@ -241,6 +242,17 @@(migrate! data-source (if auto-migrate? :up :print))(log/info "Database Migrations Current ..." (u/emoji "✅")))+(defn- migrate-settings!+ "Run [[mdb.setting/migrate-settings!]] once the migrations have, in an encryption state where every row of the+ database can be read: a caller that skips [[mdb.encryption/check-encryption]] (`enable-encryption`, `copy!`) can get+ here with rows the key in hand cannot decrypt, and those are left for it to sort out. Here rather than at+ application startup because every entry point that migrates the app DB comes through [[setup-db!]], and several+ never reach `metabase.core.core/init!` -- `migrate up`, the serialization commands, `reset-password`, the+ encryption commands -- and a JVM that skipped the repair would read every setting as nil."+ [db-state]+ (when (#{:encrypted :unencrypted :fresh :pre-sentinel} db-state)+ (mdb.setting/migrate-settings!)))+;; TODO -- consider renaming to something like `verify-connection-and-migrate!`(mu/defn setup-db!"Connects to db and runs migrations. Don't use this directly, unless you know what you're doing;@@ -276,6 +288,7 @@(when manage-encryption-state?(mdb.encryption/check-encryption db-state))(run-schema-migrations! data-source auto-migrate?)+ (migrate-settings! db-state)(when manage-encryption-state?(mdb.encryption/record-encryption-state! db-state)))))):done))diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.cljindex 644f9d4261a..d0afedefc6e 100644--- a/src/metabase/core/core.clj+++ b/src/metabase/core/core.clj@@ -202,8 +202,6 @@;; and the test suite can take 2x longer. this is really unfortunate because it could lead to some false;; negatives, but for now there's not much we can do(mdb/setup-db! :create-sample-content? (not config/is-test?))- ;; runs before anything reads settings -- see its docstring- (setting/migrate-encrypted-settings!)(mdb/encrypt-plaintext-columns!);; In OSS, convert any Data Analysts group with members to a normal visible group(perms/sync-data-analyst-group-for-oss!)diff --git a/src/metabase/settings/core.clj b/src/metabase/settings/core.cljindex 629bb31b92b..1a2233c0936 100644--- a/src/metabase/settings/core.clj+++ b/src/metabase/settings/core.clj@@ -104,7 +104,6 @@log-deprecated-env-var-usage!get-value-of-typehas-advanced-setting-access?- migrate-encrypted-settings!obfuscate-valueobfuscated-value?read-settingdiff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.cljindex 47ceeeb0191..8d88301fd4c 100644--- a/src/metabase/settings/models/setting.clj+++ b/src/metabase/settings/models/setting.clj@@ -10,6 +10,7 @@[malli.core :as mc][medley.core :as m][metabase.api.common :as api]+ [metabase.app-db.setting :as mdb.setting][metabase.config.core :as config][metabase.events.core :as events][metabase.models.serialization :as serdes]@@ -263,6 +264,12 @@registered-settings(atom {}))+(defonce ^:private ^{:doc "Map of `:deprecated-name` (as a string) -> the name of the setting that declares it. A row+ stored under a name a setting used to have is still that setting's, and [[db-or-cache-value]] reads it as a+ fallback, so it has to resolve to the setting like the current name does."}+ settings-by-deprecated-name+ (atom {}))+(defprotocol ^:private Resolvable(resolve-setting [setting-definition-or-name]"Resolve the definition map for a Setting. `setting-definition-or-name` map be a map, keyword, or string."))@@ -297,12 +304,13 @@;; references for now(defmethod setting.cache/call-on-change :default[old new]- (let [rs @registered-settings- [d1 d2] (data/diff old new)]- (doseq [changed-setting (into (set (keys d1))- (set (keys d2)))]- (when-let [on-change (get-in rs [(keyword changed-setting) :on-change])]- (on-change (core/get old changed-setting) (core/get new changed-setting))))))+ (when (some? new)+ (let [rs @registered-settings+ [d1 d2] (data/diff old new)]+ (doseq [changed-setting (into (set (keys d1))+ (set (keys d2)))]+ (when-let [on-change (get-in rs [(keyword changed-setting) :on-change])]+ (on-change (core/get old changed-setting) (core/get new changed-setting)))))));;; +----------------------------------------------------------------------------------------------------------------+;;; | get |@@ -341,12 +349,14 @@(defn- maybe-resolve-setting"Like [[resolve-setting]] but returns nil for a setting with no code definition (e.g. one written straight to the DB- in a test) instead of throwing."+ in a test) instead of throwing. A name a setting used to go by resolves to that setting."[setting-or-name](try (resolve-setting setting-or-name)(catch clojure.lang.ExceptionInfo e(when-not (::unknown-setting-error (ex-data e))- (throw e)))))+ (throw e))+ (when-let [current-name (@settings-by-deprecated-name (name setting-or-name))]+ (@registered-settings current-name)))))(defn- encrypts? [setting-or-name](not= :no (:encryption (resolve-setting setting-or-name))))@@ -1185,6 +1195,8 @@(throw (ex-info (tru "Setting {0} uses :enabled-for-db?, but is not limited to only database-local values"setting-name){:setting setting})))+ (when-let [deprecated-name (:deprecated-name setting)]+ (swap! settings-by-deprecated-name assoc (name deprecated-name) setting-name))(swap! registered-settings assoc setting-name <>))));;; +----------------------------------------------------------------------------------------------------------------+@@ -1735,85 +1747,58 @@(name (:name invalid-setting))(ex-message (:parse-error invalid-setting))))))-(defn migrate-encrypted-settings!- "Reconcile the at-rest encryption of every registered setting's stored value with its declared `:encryption`, in- both directions: a `:encryption :no` setting whose row is encrypted is decrypted, and a setting that encrypts whose- row is plaintext is encrypted. Rows of unregistered settings are left alone, and whether a value is encrypted is- decided by [[encryption/decryptable-string?]] (actually decrypting), never by shape.-- Runs on every startup, before anything restores the settings cache. The encrypting half is what makes the strict- decrypting read survive a downgrade: a boot of an older version decrypts (in this very function, as it existed- there) or re-writes as plaintext every row its registry knew as `:encryption :no`, after the one-shot encryption- migrations have already run -- and such a row would otherwise fail the strict read and take the whole settings- cache down with it. Works around the standard getters/setters deliberately: values are rewritten byte-identical- modulo encryption. No-op when MB_ENCRYPTION_SECRET_KEY is not set.-- Runs with the settings cache disabled: any setting consulted while this runs (e.g. `read-only-mode`, which the- cloud-migration DML guard reads on every write this issues) is read directly from the DB. Restoring the cache- strictly decrypts every row -- including the very rows this function exists to repair -- so going through it here- would fail the repair on exactly the state it is repairing."- []- (when (encryption/default-encryption-enabled?)- (binding [config/*disable-setting-cache* true]- (let [{encrypting true, plaintext false} (group-by (comp boolean encrypts?) (vals @registered-settings))]- (t2/with-transaction [_conn]- (doseq [{v :value k :key}- (t2/select :setting {:for :update :where [:and- [:in :key (map setting-name plaintext)]- ;; these are *definitely* decrypted already, let's not bother looking- [:not [:in :value ["true" "false"]]]]})- :when (encryption/decryptable-string? v)]- (t2/update! :setting :key k {:value (encryption/decrypt v)}))- (doseq [{v :value k :key}- (t2/select :setting {:for :update :where [:and- [:in :key (map setting-name encrypting)]- [:!= :value nil]]})- :when (not (encryption/decryptable-string? v))]- (t2/update! :setting :key k {:value (encryption/encrypt v)})))))))--(defn- maybe-encrypt [setting-model]- ;; In tests, sometimes we need to insert/update settings that don't have definitions in the code and therefore can't- ;; be resolved. Fall back to maybe-encrypting these.- ;; Don't do any automatic handling of the "encryption-check" special setting used by mdb.encryption- (if (= "encryption-check" (:key setting-model))- setting-model- (let [resolved (maybe-resolve-setting (:key setting-model))]- (cond-> setting-model- (or (nil? resolved)- (encrypts? resolved))- (update :value encryption/maybe-encrypt)))))+(defn- write-setting-value+ "Store a Setting's `:value` in `:value_with_aad`, encrypted under additional authenticated data naming the setting+ (see [[mdb.setting/setting-aad]]) whenever MB_ENCRYPTION_SECRET_KEY is set -- every setting's, whatever its+ `:encryption` says. That flag describes the legacy `value` column, which is written here too, exactly as it was+ before `value_with_aad` existed: nothing in this version reads it, but a version that predates the column does, and+ keeping it current is what lets that version run alongside this one and what makes rolling back to it lossless.++ A key with no `defsetting` is refused: [[read-setting-value]] would read the row back as no value at all, so there is+ no way to write one that means anything."+ [setting]+ (let [setting-key (:key setting)+ value (:value setting)+ resolved (or (maybe-resolve-setting setting-key)+ (throw (ex-info (tru "Unknown setting: {0}" setting-key)+ {:setting-key setting-key})))]+ (assoc setting+ :value (cond-> value (encrypts? resolved) encryption/maybe-encrypt)+ :value_with_aad (some-> value (encryption/maybe-encrypt {:aad (mdb.setting/setting-aad setting-key)})))))++(defn- read-setting-value+ "Take a Setting's `:value` from the `:value_with_aad` it is stored in: decrypted under the additional authenticated+ data naming this setting, strictly, with [[encryption/maybe-decrypt]]. With MB_ENCRYPTION_SECRET_KEY set that column+ is ciphertext for every setting, so a plaintext value -- forged via a direct DB write, or left by a row that has never+ been through `enable-encryption` -- is rejected rather than trusted, and so is a ciphertext moved here from another+ setting's row, since it was authenticated under that setting's name.++ Two rows read as no value at all. One with no `value_with_aad` is a row only a version predating the column has ever+ written. One whose key has no `defsetting` -- a retired setting, one belonging to an edition this instance is not+ running, or a row written straight to the DB in a test -- is not read at all, not even decrypted: nothing can ask+ for such a setting by name, so there is no value to produce and no reason to touch what is stored there."+ [setting]+ (let [setting-key (:key setting)]+ (if (maybe-resolve-setting setting-key)+ (try+ (assoc setting :value (some-> (:value_with_aad setting)+ (encryption/maybe-decrypt {:aad (mdb.setting/setting-aad setting-key)})))+ (catch Throwable e+ (throw (ex-info (format "Error reading setting \"%s\": %s" setting-key (ex-message e))+ {:setting-key setting-key}+ e))))+ (assoc setting :value nil))))(t2/define-before-update :model/Setting[setting]- (maybe-encrypt setting))+ (write-setting-value setting))(t2/define-before-insert :model/Setting[setting]- (maybe-encrypt setting))--(defn- decrypt-setting-value-on-read- "Decrypt a Setting's `:value` on read. A setting whose `:encryption` is not `:no` is stored encrypted at rest, so it- is read strictly with [[encryption/maybe-decrypt]]: a plaintext value — forged via a direct DB write, or a legacy row- from before the setting became encrypted — is rejected rather than trusted. A `:no` setting (or one with no code- definition, e.g. in tests) is intentionally plaintext, so it is read leniently with- [[encryption/maybe-decrypt-accepting-plaintext]], which returns a plaintext value unchanged."- [setting]- (let [resolved (maybe-resolve-setting (:key setting))- decrypt (if (or (nil? resolved) (not (encrypts? resolved)))- encryption/maybe-decrypt-accepting-plaintext- encryption/maybe-decrypt)]- (try- (update setting :value decrypt)- (catch Throwable e- (throw (ex-info (format "Error decrypting setting \"%s\": %s" (:key setting) (ex-message e))- {:setting-key (:key setting)}- e))))))+ (write-setting-value setting))(t2/define-after-select :model/Setting[setting]- ;; Skip aggregate results (e.g. a `count` row) that carry no `:key` to resolve or `:value` to decrypt, and don't do- ;; any automatic handling of the "encryption-check" special setting used by mdb.encryption.- (if (or (nil? (:key setting))- (= "encryption-check" (:key setting)))- setting- (decrypt-setting-value-on-read setting)))+ ;; Skip aggregate results (e.g. a `count` row) that carry no `:key` to resolve the setting by.+ (cond-> setting+ (some? (:key setting)) read-setting-value))diff --git a/src/metabase/settings/models/setting/cache.clj b/src/metabase/settings/models/setting/cache.cljindex 778afbfa3ff..85fb9922dd9 100644--- a/src/metabase/settings/models/setting/cache.clj+++ b/src/metabase/settings/models/setting/cache.clj@@ -5,8 +5,9 @@[clojure.core :as core][clojure.string :as str][metabase.app-db.core :as mdb]+ [metabase.app-db.setting :as mdb.setting][metabase.util :as u]- [metabase.util.honey-sql-2 :as h2x]+ [metabase.util.encryption :as encryption][metabase.util.log :as log][toucan2.core :as t2])(:import@@ -72,18 +73,19 @@"Update the value of `settings-last-updated` in the DB; if the row does not exist, insert one."[](log/debug "Updating value of settings-last-updated in DB...")- ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)- (let [current-timestamp-as-string-honeysql (h2x/cast (if (= (mdb/db-type) :mysql) :char :text)- (h2x/current-datetime-honeysql-form (mdb/db-type)))]+ ;; Written raw, not through `:model/Setting`, so that `value` gets the plaintext timestamp a version predating+ ;; `value_with_aad` compares in SQL. `value_with_aad` is encrypted under the marker's AAD like any other setting's.+ (let [value (mdb/current-timestamp-string (mdb/db-type))+ value-with-aad (encryption/maybe-encrypt value {:aad (mdb.setting/setting-aad settings-last-updated-key)})];; attempt to UPDATE the existing row. If no row exists, `t2/update!` will return 0...- (or (pos? (t2/update! :setting {:key settings-last-updated-key} {:value current-timestamp-as-string-honeysql}))+ (or (pos? (t2/update! :setting {:key settings-last-updated-key} {:value value, :value_with_aad value-with-aad}));; ...at which point we will try to INSERT a new row. Note that it is entirely possible two instances can both;; try to INSERT it at the same time; one instance would fail because it would violate the PK constraint on;; `key`, and throw a SQLException. As long as one instance updates the value, we are fine, so we can go ahead;; and ignore that Exception if one is thrown.(try- ;; Use `simple-insert!` because we do *not* want to trigger pre-insert behavior, such as encrypting `:value`- (t2/insert! (t2/table-name (t2/resolve-model :model/Setting)) :key settings-last-updated-key, :value current-timestamp-as-string-honeysql)+ (t2/insert! (t2/table-name (t2/resolve-model :model/Setting))+ :key settings-last-updated-key, :value value, :value_with_aad value-with-aad)(catch java.sql.SQLException e;; go ahead and log the whole SQLException message chain anyway on the off chance that it *wasn't* just a;; race condition issue@@ -118,10 +120,10 @@(when-let [last-known-update (cache-last-updated-at)];; compare it to the value in the DB. This is done be seeing whether a row exists;; WHERE value > <local-value>- (u/prog1 (t2/select-one-fn :value :model/Setting- {:where [:and- [:= :key settings-last-updated-key]- [:> :value last-known-update]]})+ ;; compared here rather than in SQL: what is stored is ciphertext, so only the decrypted timestamps can be ordered+ (u/prog1 (when-let [db-value (t2/select-one-fn :value :model/Setting :key settings-last-updated-key)]+ (when (pos? (compare db-value last-known-update))+ db-value))(log/trace "last known Settings update: " (pr-str last-known-update))(log/trace "actual last Settings update:" (pr-str <>))(when <>diff --git a/src/metabase/settings/settings.clj b/src/metabase/settings/settings.cljindex 4aa09bbfc84..46a9a769f6e 100644--- a/src/metabase/settings/settings.clj+++ b/src/metabase/settings/settings.clj@@ -1,8 +1,24 @@(ns metabase.settings.settings(:require[metabase.config.core :as config]+ [metabase.settings.models.setting :refer [defsetting]][metabase.util.malli :as mu]))+(defsetting settings-last-updated+ "When any Setting was last changed, as a timestamp string the application DB produced. Instances compare it against+ their own copy to notice that another instance has changed a Setting and their cache is stale.++ Written by [[metabase.settings.models.setting.cache/update-settings-last-updated!]] rather than through this+ Setting, which is why it has no setter: its value has to come from the application DB's clock, not from any one+ instance's. It is declared here so that it is a registered Setting like any other, and so its row is read like any+ other."+ :visibility :internal+ :type :string+ :encryption :no+ :setter :none+ :audit :never+ :export? false)+(mu/defn application-name-for-setting-descriptions"Returns the value of the [[application-name]] setting so setting docstrings can be generated during the compilationstage. Use this instead of `application-name` in descriptions, otherwise the `application-name` setting'sdiff --git a/src/metabase/util/encryption.clj b/src/metabase/util/encryption.cljindex 42eaab52bac..27979295c30 100644--- a/src/metabase/util/encryption.clj+++ b/src/metabase/util/encryption.clj@@ -74,55 +74,68 @@"\n""For more information, see https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html"))+(defn- opts->secret-key+ "The 64-byte key `opts` name, or the hashed MB_ENCRYPTION_SECRET_KEY when they name none."+ ^bytes [{:keys [^bytes secret-key]}]+ (or secret-key default-secret-key))++(defn- cipher-opts+ "Buddy's options for the AEAD scheme, with the `:aad` from `opts`, if any. Additional authenticated data is covered+ by the ciphertext's MAC without being encrypted: a value encrypted under some AAD only decrypts under the same AAD,+ so binding a value to the row it belongs to is a matter of naming the row here."+ [{:keys [^bytes aad]}]+ (cond-> {:algorithm :aes256-cbc-hmac-sha512}+ aad (assoc :aad aad)))++;;; Every function below takes an optional trailing `opts` map, with `:secret-key` (a 64-byte hashed key, as+;;; [[secret-key->hash]] makes one; by default the hashed MB_ENCRYPTION_SECRET_KEY) and, for the string and byte-array+;;; functions, `:aad` (bytes the ciphertext is bound to -- see [[cipher-opts]]). `:secret-key` is what lets tests pass+;;; a key directly instead of using `with-redefs`, and what key rotation uses to write under the key it rotates to.+(defn encrypt-bytes- "Encrypt bytes `b` using a `secret-key` (a 64-byte byte array), by default is the hashed value of- `MB_ENCRYPTION_SECRET_KEY`."+ "Encrypt bytes `b`."{:added "0.41.0"}- (^String [^bytes b]- (encrypt-bytes default-secret-key b))- (^String [^String secret-key, ^bytes b]+ (^bytes [^bytes b]+ (encrypt-bytes b nil))+ (^bytes [^bytes b opts](let [initialization-vector (nonce/random-bytes 16)]- (->> (crypto/encrypt b- secret-key- initialization-vector- {:algorithm :aes256-cbc-hmac-sha512})+ (->> (crypto/encrypt b (opts->secret-key opts) initialization-vector (cipher-opts opts))(concat initialization-vector)byte-array))))(defn encrypt- "Encrypt string `s` as hex bytes using a `secret-key` (a 64-byte byte array), which by default is the hashed value of- `MB_ENCRYPTION_SECRET_KEY`."+ "Encrypt string `s`, returning the ciphertext as base64."(^String [^String s]- (encrypt default-secret-key s))- (^String [^String secret-key, ^String s]- (->> (codecs/to-bytes s)- (encrypt-bytes secret-key)- codec/base64-encode)))+ (encrypt s nil))+ (^String [^String s opts]+ (-> (codecs/to-bytes s)+ (encrypt-bytes opts)+ codec/base64-encode)))(defn decrypt-bytes- "Decrypt bytes `b` using a `secret-key` (a 64-byte byte array), which by default is the hashed value of- `MB_ENCRYPTION_SECRET_KEY`."+ "Decrypt bytes `b`."{:added "0.41.0"}- (^String [^bytes b]- (decrypt-bytes default-secret-key b))- (^String [secret-key, ^bytes b]+ (^bytes [^bytes b]+ (decrypt-bytes b nil))+ (^bytes [^bytes b opts](let [[initialization-vector message] (split-at 16 b)](crypto/decrypt (byte-array message)- secret-key+ (opts->secret-key opts)(byte-array initialization-vector)- {:algorithm :aes256-cbc-hmac-sha512}))))+ (cipher-opts opts)))))(defn encrypt-stream"Wraps a plaintext input stream into an input stream that encrypts it using AES256 CBC.- The encryption format is slightly different for streams vs. fixed length data"+ The encryption format is slightly different for streams vs. fixed length data, and carries no AAD."{:added "0.53.0"}(^InputStream [^InputStream input-stream]- (encrypt-stream default-secret-key input-stream))- (^InputStream [secret-key ^InputStream input-stream]- (let [spec aes-streaming-spec+ (encrypt-stream input-stream nil))+ (^InputStream [^InputStream input-stream opts]+ (let [secret-key (opts->secret-key opts)+ spec aes-streaming-specspec-header (codecs/to-bytes (format "%-32s" spec))- cipher (Cipher/getInstance spec)- iv (nonce/random-bytes 16)]+ cipher (Cipher/getInstance spec)+ iv (nonce/random-bytes 16)](.init cipher Cipher/ENCRYPT_MODE (SecretKeySpec. (bytes/slice secret-key 32 64) "AES") (IvParameterSpec. iv))(SequenceInputStream. (ByteArrayInputStream. (bytes/concat spec-header iv)) (CipherInputStream. input-stream cipher)))))@@ -130,28 +143,29 @@"Encrypts a byte-array in a way that can be used to read it with decrypt-stream instead of decrypt."{:added "0.53.0"}(^bytes [^bytes input]- (encrypt-for-stream default-secret-key input))- (^bytes [secret-key ^bytes input]- (with-open [encrypted (encrypt-stream secret-key (ByteArrayInputStream. input))]+ (encrypt-for-stream input nil))+ (^bytes [^bytes input opts]+ (with-open [encrypted (encrypt-stream (ByteArrayInputStream. input) opts)](.readAllBytes encrypted))))(defn maybe-decrypt-stream"Wraps a possibly-encrypted input stream into a new input stream that decrypts it if necessary."{:added "0.53.0"}(^InputStream [^InputStream input-stream]- (maybe-decrypt-stream default-secret-key input-stream))- (^InputStream [secret-key ^InputStream input-stream]- (let [spec-array (byte-array 32)+ (maybe-decrypt-stream input-stream nil))+ (^InputStream [^InputStream input-stream opts]+ (let [secret-key (opts->secret-key opts)+ spec-array (byte-array 32)spec-array-length (.read input-stream spec-array)- spec (str/trim (codecs/bytes->str spec-array))]+ spec (str/trim (codecs/bytes->str spec-array))](cond(= spec-array-length -1)input-stream(and (= spec-array-length 32) (= spec aes-streaming-spec))(let [cipher (Cipher/getInstance spec)- iv (byte-array 16)- _ (.read input-stream iv)]+ iv (byte-array 16)+ _ (.read input-stream iv)](.init cipher Cipher/DECRYPT_MODE (SecretKeySpec. (bytes/slice secret-key 32 64) "AES") (IvParameterSpec. iv))(CipherInputStream. input-stream cipher))@@ -161,42 +175,42 @@input-stream)))))(defn decrypt- "Decrypt string `s` using a `secret-key` (a 64-byte byte array), by default the hashed value of- `MB_ENCRYPTION_SECRET_KEY`."+ "Decrypt base64 ciphertext `s` back to the string it was made from."(^String [^String s]- (decrypt default-secret-key s))- (^String [secret-key, ^String s]- (codecs/bytes->str (decrypt-bytes secret-key (codec/base64-decode s)))))+ (decrypt s nil))+ (^String [^String s opts]+ (codecs/bytes->str (decrypt-bytes (codec/base64-decode s) opts))))(defn maybe-encrypt- "If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of `s`; otherwise return `s` as-is."+ "If there is a key (MB_ENCRYPTION_SECRET_KEY, or `:secret-key` in `opts`), return an encrypted version of `s`;+ otherwise return `s` as-is."(^String [^String s]- (maybe-encrypt default-secret-key s))- (^String [secret-key, ^String s]- (if secret-key+ (maybe-encrypt s nil))+ (^String [^String s opts]+ (if (opts->secret-key opts)(when (seq s)- (encrypt secret-key s))+ (encrypt s opts))s)))(defn maybe-encrypt-bytes- "If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of the given bytes `b`; otherwise return `b`- as-is."+ "If there is a key, return an encrypted version of the given bytes `b`; otherwise return `b` as-is."{:added "0.41.0"}(^bytes [^bytes b]- (maybe-encrypt-bytes default-secret-key b))- (^bytes [secret-key, ^bytes b]- (if secret-key+ (maybe-encrypt-bytes b nil))+ (^bytes [^bytes b opts]+ (if (opts->secret-key opts)(when (seq b)- (encrypt-bytes secret-key b))+ (encrypt-bytes b opts))b)))(defn maybe-encrypt-for-stream- "If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of `s` that can be used to stream the data; otherwise return `s` as-is."+ "If there is a key, return an encrypted version of `s` that can be used to stream the data; otherwise return `s`+ as-is."(^bytes [^bytes s]- (maybe-encrypt-for-stream default-secret-key s))- (^bytes [secret-key, ^bytes s]- (if secret-key- (encrypt-for-stream secret-key s)+ (maybe-encrypt-for-stream s nil))+ (^bytes [^bytes s opts]+ (if (opts->secret-key opts)+ (encrypt-for-stream s opts)s)))(def ^:private ^:const aes256-tag-length 32)@@ -238,81 +252,76 @@(possibly-encrypted-bytes? b)))))(defn decryptable-bytes?- "Whether `b` is encrypted with `secret-key`: it has the shape of ciphertext *and* decrypts. The ciphertext is- authenticated, so a wrong key, corruption, and plaintext that merely looks like ciphertext all yield false. Always- false when no key is set. Use this, not [[possibly-encrypted-bytes?]], wherever the answer decides whether a value is- encrypted, decrypted, or left alone."- ([^bytes b] (decryptable-bytes? default-secret-key b))- ([secret-key ^bytes b]- (boolean (and secret-key+ "Whether `b` is encrypted with the key (and under the `:aad`, if any): it has the shape of ciphertext *and* decrypts.+ The ciphertext is authenticated, so a wrong key, a wrong AAD, corruption, and plaintext that merely looks like+ ciphertext all yield false. Always false when there is no key. Use this, not [[possibly-encrypted-bytes?]], wherever+ the answer decides whether a value is encrypted, decrypted, or left alone."+ ([^bytes b] (decryptable-bytes? b nil))+ ([^bytes b opts]+ (boolean (and (opts->secret-key opts)(possibly-encrypted-bytes? b)- (u/ignore-exceptions (decrypt-bytes secret-key b) true)))))+ (u/ignore-exceptions (decrypt-bytes b opts) true)))))(defn decryptable-string?- "Whether `s` is encrypted with `secret-key`: it has the shape of ciphertext *and* decrypts. See [[decryptable-bytes?]]."- ([^String s] (decryptable-string? default-secret-key s))- ([secret-key ^String s]- (boolean (and secret-key+ "Whether `s` is encrypted with the key (and under the `:aad`, if any): it has the shape of ciphertext *and* decrypts.+ See [[decryptable-bytes?]]."+ ([^String s] (decryptable-string? s nil))+ ([^String s opts]+ (boolean (and (opts->secret-key opts)(possibly-encrypted-string? s)- (u/ignore-exceptions (decrypt secret-key s) true)))))+ (u/ignore-exceptions (decrypt s opts) true)))))(defn maybe-decrypt-accepting-plaintext- "Plaintext-tolerant decrypt of a String `s`. If `MB_ENCRYPTION_SECRET_KEY` is set and `s` is encrypted, decrypt it;- a value that is stored as plaintext is returned as-is. A value that *looks* encrypted but cannot be decrypted with- the current key (wrong key, tampering, corruption) throws — it must not be silently returned, or a re-encrypting- caller would double-encrypt it into something permanently unrecoverable. This differs from the strict- [[maybe-decrypt]] only in tolerating genuine plaintext; use it for values that may legitimately be plaintext at rest- (rows written before encryption was enabled, key rotation, settings, and migration reads).-- `secret-key` is accepted as an argument so tests can pass it directly instead of using `with-redefs` to run in- parallel."- (^String [^String s] (maybe-decrypt-accepting-plaintext default-secret-key s))- (^String [secret-key ^String s]+ "Plaintext-tolerant decrypt of a String `s`. If there is a key and `s` is encrypted, decrypt it; a value that is+ stored as plaintext is returned as-is. A value that *looks* encrypted but cannot be decrypted with the key (wrong+ key, wrong AAD, tampering, corruption) throws -- it must not be silently returned, or a re-encrypting caller would+ double-encrypt it into something permanently unrecoverable. This differs from the strict [[maybe-decrypt]] only in+ tolerating genuine plaintext; use it for values that may legitimately be plaintext at rest (rows written before+ encryption was enabled, key rotation, settings, and migration reads)."+ (^String [^String s] (maybe-decrypt-accepting-plaintext s nil))+ (^String [^String s opts](cond- (nil? secret-key) s- (possibly-encrypted-string? s) (decrypt secret-key s)+ (nil? (opts->secret-key opts)) s+ (possibly-encrypted-string? s) (decrypt s opts):else s)))(defn maybe-decrypt-bytes-accepting-plaintext"Plaintext-tolerant counterpart to [[maybe-decrypt-accepting-plaintext]] for a byte array `b` (e.g. secret values):an encrypted value is decrypted, a plaintext value is returned as-is, and a value that looks encrypted but cannot be- decrypted with the current key throws rather than being returned (returning it would let a re-encrypting caller+ decrypted with the key throws rather than being returned (returning it would let a re-encrypting callerdouble-encrypt it into something permanently unrecoverable)."- (^bytes [^bytes b] (maybe-decrypt-bytes-accepting-plaintext default-secret-key b))- (^bytes [secret-key ^bytes b]+ (^bytes [^bytes b] (maybe-decrypt-bytes-accepting-plaintext b nil))+ (^bytes [^bytes b opts](cond- (nil? secret-key) b- (possibly-encrypted-bytes? b) (decrypt-bytes secret-key b)- :else b)))+ (nil? (opts->secret-key opts)) b+ (possibly-encrypted-bytes? b) (decrypt-bytes b opts)+ :else b)))(defn maybe-decrypt- "Strict decrypt of a String `s`. When `MB_ENCRYPTION_SECRET_KEY` is set, `s` must be an encrypted value that decrypts- with the current key: a value that is not encrypted throws (it was written outside the encrypting path — a plaintext- value cannot stand in for an encrypted one), and a value that is encrypted but cannot be decrypted with the current- key (wrong key, tampering, or corruption) also throws rather than being trusted. When no key is set, `s` is returned- as-is (there is no key to decrypt with). For values that may legitimately be plaintext at rest, use- [[maybe-decrypt-accepting-plaintext]].-- `secret-key` is accepted as an argument so tests can pass it directly instead of using `with-redefs` to run in- parallel."- (^String [^String s] (maybe-decrypt default-secret-key s))- (^String [secret-key ^String s]+ "Strict decrypt of a String `s`. When there is a key, `s` must be an encrypted value that decrypts with it (and+ under the `:aad`, if any): a value that is not encrypted throws (it was written outside the encrypting path -- a+ plaintext value cannot stand in for an encrypted one), and a value that is encrypted but cannot be decrypted with+ the key (wrong key, wrong AAD, tampering, or corruption) also throws rather than being trusted. When there is no+ key, `s` is returned as-is (there is no key to decrypt with). For values that may legitimately be plaintext at rest,+ use [[maybe-decrypt-accepting-plaintext]]."+ (^String [^String s] (maybe-decrypt s nil))+ (^String [^String s opts](cond- (nil? secret-key) s+ (nil? (opts->secret-key opts)) s(nil? s) s- (possibly-encrypted-string? s) (decrypt secret-key s)+ (possibly-encrypted-string? s) (decrypt s opts):else (throw (ex-info "Expected an encrypted value but the stored value is not encrypted."{:type ::not-encrypted})))))(defn maybe-decrypt-bytes"Strict counterpart to [[maybe-decrypt-bytes-accepting-plaintext]] for a byte array `b`: a value that is not encrypted,- or that cannot be decrypted with the current key, throws rather than being trusted. When no key is set, `b` is returned+ or that cannot be decrypted with the key, throws rather than being trusted. When there is no key, `b` is returnedas-is."- (^bytes [^bytes b] (maybe-decrypt-bytes default-secret-key b))- (^bytes [secret-key ^bytes b]+ (^bytes [^bytes b] (maybe-decrypt-bytes b nil))+ (^bytes [^bytes b opts](cond- (nil? secret-key) b- (nil? b) b- (possibly-encrypted-bytes? b) (decrypt-bytes secret-key b)- :else (throw (ex-info "Expected an encrypted value but the stored value is not encrypted."- {:type ::not-encrypted})))))+ (nil? (opts->secret-key opts)) b+ (nil? b) b+ (possibly-encrypted-bytes? b) (decrypt-bytes b opts)+ :else (throw (ex-info "Expected an encrypted value but the stored value is not encrypted."+ {:type ::not-encrypted})))))diff --git a/test/metabase/app_db/query_test.clj b/test/metabase/app_db/query_test.cljindex e5195552079..b72b44a317f 100644--- a/test/metabase/app_db/query_test.clj+++ b/test/metabase/app_db/query_test.clj@@ -8,6 +8,7 @@[metabase.app-db.query :as mdb.query][metabase.query-processor.compile :as qp.compile][metabase.query-processor.test :as qp]+ [metabase.settings.models.setting :as setting][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u]@@ -19,6 +20,25 @@(use-fixtures :once (fixtures/initialize :db))+(defn- random-setting-name!+ "Register a Setting under a random name and return it. The Setting model refuses to write a key it cannot resolve to+ a definition, and these tests need a fresh key per racing thread -- more than one `defsetting` could supply."+ []+ (let [setting-name (keyword (str "query-test-setting-" (random-uuid)))]+ (setting/register-setting! {:name setting-name+ :namespace (ns-name *ns*)+ :type :string+ :encryption :no+ :visibility :internal})+ (name setting-name)))++(defn- random-value!+ "A fresh value to store in `column`: the `key` column holds setting names, the `value` column anything."+ [column]+ (if (= column :key)+ (random-setting-name!)+ (str (random-uuid))))+(defn- verify-same-query"Ensure that the formatted native query derived from an mbql query produce the same results."[q]@@ -85,7 +105,7 @@(deftest select-or-insert!-test;; We test both a case where the database protects against duplicates, and where it does not.- ;; Using Setting is perfect because it has only two required fields - (the primary) key & value (with no constraint).+ ;; Using the `setting` table is perfect because it has only two required fields - (the primary) key & value (with;;;; In the `:key` case using the `idempotent-insert!` rather than an `or` prevents the from application throwing an;; exception when there are race conditions. For `:value` it prevents us silently inserting duplicates.@@ -95,7 +115,7 @@(doseq [search-col columns](testing (format "When the search column %s a uniqueness constraint in the db"(if (= :key search-col) "has" "does not have"))- (let [search-value (str (random-uuid))+ (let [search-value (random-value! search-col)other-col (first (remove #{search-col} columns))](try;; ensure there is no database detritus to trip us up@@ -108,7 +128,7 @@;; Make sure all the threads are in the mutating path(.countDown latch)(.await latch)- {other-col (str (random-uuid))})))+ {other-col (random-value! other-col)})))results (set (mt/repeat-concurrently threads thunk))n (count results)latest (t2/select-one :model/Setting search-col search-value)]@@ -137,15 +157,15 @@(deftest updated-or-insert!-test;; We test both a case where the database protects against duplicates, and where it does not.- ;; Using Setting is perfect because it has only two required fields - (the primary) key & value (with no constraint).+ ;; Using the `setting` table is perfect because it has only two required fields - (the primary) key & value (with(let [columns [:key :value]](doseq [search-col columns](testing (format "When the search column %s a uniqueness constraint in the db"(if (= :key search-col) "has" "does not have"))(doseq [already-exists? [true false]]- (let [search-value (str (random-uuid))+ (let [search-value (random-value! search-col)other-col (first (remove #{search-col} columns))- other-value (str (random-uuid))]+ other-value (random-value! other-col)](try;; ensure there is no database detritus to trip us up(t2/delete! :model/Setting search-col search-value)@@ -154,7 +174,7 @@(let [threads 5latch (CountDownLatch. threads)thunk (fn []- (u/prog1 (str (random-uuid))+ (u/prog1 (random-value! other-col)(mdb.query/update-or-insert! :model/Setting {search-col search-value}(fn [_];; Make sure all the threads are in the mutating pathdiff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.cljindex de6f3b6f9f1..6e83d794280 100644--- a/test/metabase/app_db/schema_migrations_test.clj+++ b/test/metabase/app_db/schema_migrations_test.clj@@ -2559,88 +2559,88 @@(deftest ^:mb/old-migrations-test populate-enabled-embedding-settings-works(testing "Check that embedding settings are nil when enable-embedding is nil"(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]- (t2/delete! :model/Setting :key "enable-embedding")+ (t2/delete! :setting :key "enable-embedding")(migrate!)- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-interactive")))- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-static")))- (is (= nil (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk")))))+ (is (= nil (t2/select-one :setting :key "enable-embedding-interactive")))+ (is (= nil (t2/select-one :setting :key "enable-embedding-static")))+ (is (= nil (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk"))))))(testing "Check that embedding settings are true when enable-embedding is true"(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]- (t2/delete! :model/Setting :key "enable-embedding")- (t2/insert! :model/Setting {:key "enable-embedding" :value "true"})+ (t2/delete! :setting :key "enable-embedding")+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "true")})(migrate!)- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk")))))+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk"))))))(testing "Check that embedding settings are false when enable-embedding is false"(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]- (t2/delete! :model/Setting :key "enable-embedding")- (t2/insert! :model/Setting {:key "enable-embedding" :value "false"})+ (t2/delete! :setting :key "enable-embedding")+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "false")})(migrate!)- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk"))))))+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk")))))))(deftest ^:mb/old-migrations-test populate-enabled-embedding-settings-encrypted-works(testing "With encryption turned on > "(mt/with-temp-env-var-value! [MB_ENCRYPTION_SECRET_KEY "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"](testing "Check that embedding settings are nil when enable-embedding is nil"(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]- (t2/delete! :model/Setting :key "enable-embedding")+ (t2/delete! :setting :key "enable-embedding")(migrate!)- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-interactive")))- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-static")))- (is (= nil (t2/select-one :model/Setting :key "enable-embedding-sdk")))))+ (is (= nil (t2/select-one :setting :key "enable-embedding-interactive")))+ (is (= nil (t2/select-one :setting :key "enable-embedding-static")))+ (is (= nil (t2/select-one :setting :key "enable-embedding-sdk")))))(testing "Check that embedding settings are true when enable-embedding is true"(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]- (t2/delete! :model/Setting :key "enable-embedding")- (t2/insert! :model/Setting {:key "enable-embedding" :value "true"})+ (t2/delete! :setting :key "enable-embedding")+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "true")})(migrate!)- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))- (is (= "true" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk")))))+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))+ (is (= "true" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk"))))))(testing "Check that embedding settings are false when enable-embedding is false"(impl/test-migrations ["v51.2024-09-26T03:01:00" "v51.2024-09-26T03:03:00"] [migrate!]- (t2/delete! :model/Setting :key "enable-embedding")- (t2/insert! :model/Setting {:key "enable-embedding" :value "false"})+ (t2/delete! :setting :key "enable-embedding")+ (t2/insert! :setting {:key "enable-embedding" :value (encryption/maybe-encrypt "false")})(migrate!)- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-interactive")))- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-static")))- (is (= "false" (t2/select-one-fn :value :model/Setting :key "enable-embedding-sdk"))))))))+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-interactive"))))+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-static"))))+ (is (= "false" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "enable-embedding-sdk")))))))))(deftest ^:mb/old-migrations-test populate-embedding-origin-settings-works(testing "Check that embedding-origins are unset when embedding-app-origin is unset"(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]- (t2/delete! :model/Setting :key "embedding-app-origin")+ (t2/delete! :setting :key "embedding-app-origin")(migrate!)- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-interactive")))- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-sdk"))))))+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-interactive")))+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-sdk"))))))(deftest ^:mb/old-migrations-test populate-embedding-origin-settings-works-2(testing "Check that embedding-origins settings are propigated when embedding-app-origin is set to some value"(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]- (t2/delete! :model/Setting :key "embedding-app-origin")- (t2/insert! :model/Setting {:key "embedding-app-origin" :value "1.2.3.4:5555"})- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origin")))+ (t2/delete! :setting :key "embedding-app-origin")+ (t2/insert! :setting {:key "embedding-app-origin" :value (encryption/maybe-encrypt "1.2.3.4:5555")})+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origin"))))(migrate!)- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origins-interactive"))))))+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origins-interactive")))))))(deftest ^:mb/old-migrations-test populate-embedding-origin-settings-encrypted-works(testing "With encryption turned on > "(mt/with-temp-env-var-value! [MB_ENCRYPTION_SECRET_KEY "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"](testing "Check that embedding-origins are unset when embedding-app-origin is unset"(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]- (t2/delete! :model/Setting :key "embedding-app-origin")+ (t2/delete! :setting :key "embedding-app-origin")(migrate!)- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-interactive")))- (is (= nil (t2/select-one :model/Setting :key "embedding-app-origins-sdk")))))+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-interactive")))+ (is (= nil (t2/select-one :setting :key "embedding-app-origins-sdk")))))(testing "Check that embedding-origins settings are propigated when embedding-app-origin is set to some value"(impl/test-migrations "v51.2024-09-26T03:04:00" [migrate!]- (t2/delete! :model/Setting :key "embedding-app-origin")- (t2/insert! :model/Setting {:key "embedding-app-origin" :value "1.2.3.4:5555"})- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origin")))+ (t2/delete! :setting :key "embedding-app-origin")+ (t2/insert! :setting {:key "embedding-app-origin" :value (encryption/maybe-encrypt "1.2.3.4:5555")})+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origin"))))(migrate!)- (is (= "1.2.3.4:5555" (t2/select-one-fn :value :model/Setting :key "embedding-app-origins-interactive"))))))))+ (is (= "1.2.3.4:5555" (encryption/maybe-decrypt-accepting-plaintext (t2/select-one-fn :value :setting :key "embedding-app-origins-interactive")))))))));;;;;; 53+ tests should go below this line please <3diff --git a/test/metabase/app_db/setup_test.clj b/test/metabase/app_db/setup_test.cljindex badd77ec74e..48999f9bff1 100644--- a/test/metabase/app_db/setup_test.clj+++ b/test/metabase/app_db/setup_test.clj@@ -4,12 +4,16 @@[clojure.test :refer :all][metabase.app-db.connection :as mdb.connection][metabase.app-db.connection-pool-setup :as mdb.connection-pool-setup]+ [metabase.app-db.core :as mdb][metabase.app-db.data-source :as mdb.data-source][metabase.app-db.liquibase :as liquibase]+ [metabase.app-db.setting :as mdb.setting][metabase.app-db.setup :as mdb.setup][metabase.app-db.test-util :as mdb.test-util][metabase.driver :as driver][metabase.test :as mt]+ [metabase.util.encryption :as encryption]+ [metabase.util.encryption-test :as encryption-test][toucan2.core :as t2])(:import(liquibase.changelog ChangeSet)))@@ -226,3 +230,35 @@{:delete [:field]:from [[:metabase_field :field]]:where [:= :field.id 0]}))))))++(deftest setup-db-leaves-settings-alone-without-their-key-test+ (testing "setup-db! fills in setting.details, except in an encryption state where the rows cannot be read"+ (mt/with-temp-empty-app-db [_conn :h2]+ (mdb/setup-db! :create-sample-content? false)+ (let [ciphertext (encryption-test/with-secret-key "ABCDEFGH12345678"+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)+ (t2/insert! :model/Setting {:key "site-name", :value "Sad Can"})+ (t2/select-one-fn :value_with_aad :setting :key "site-name"))]+ (is (encryption/possibly-encrypted-string? ciphertext))+ (testing "with no key the state is :missing-key, and a repair would only wrap the ciphertext as a value"+ (reset! (:status mdb.connection/*application-db*) ::not-set-up)+ (mdb/setup-db! :create-sample-content? false :manage-encryption-state? false)+ (is (= ciphertext (t2/select-one-fn :value_with_aad :setting :key "site-name"))))))))++(deftest encryption-check-status-falls-back-to-value-test+ (testing "the sentinel is read from `value`, whatever `value_with_aad` holds -- an older version rewrites only `value`"+ (mt/with-temp-empty-app-db [_conn :h2]+ (mdb/setup-db! :create-sample-content? false)+ (encryption-test/with-secret-key "ABCDEFGH12345678"+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)+ (is (= :valid (mdb/encryption-check-status)))+ (testing "an authenticated value left under a key a rotation on an older version has since replaced"+ (let [old-key (encryption/secret-key->hash "12345678ABCDEFGH")]+ (t2/update! :setting :key "encryption-check"+ {:value_with_aad (encryption/encrypt (str (random-uuid))+ {:secret-key old-key+ :aad (mdb.setting/setting-aad "encryption-check")})})+ (is (= :valid (mdb/encryption-check-status)))))+ (testing "the unencrypted marker an older version's remove-encryption writes to value alone"+ (t2/update! :setting :key "encryption-check" {:value "unencrypted"})+ (is (= :absent (mdb/encryption-check-status))))))))diff --git a/test/metabase/cmd/dump_to_h2_test.clj b/test/metabase/cmd/dump_to_h2_test.cljindex 79aa7d930b8..bc460106e19 100644--- a/test/metabase/cmd/dump_to_h2_test.clj+++ b/test/metabase/cmd/dump_to_h2_test.clj@@ -15,6 +15,7 @@[metabase.cmd.test-util :as cmd.test-util][metabase.config.core :as config][metabase.driver :as driver]+ [metabase.settings.core :refer [defsetting]][metabase.test :as mt][metabase.test.data.interface :as tx][metabase.util.encryption-test :as encryption-test]@@ -23,6 +24,11 @@(set! *warn-on-reflection* true)+(defsetting dump-to-h2-test-setting+ "Test setting -- a setting row to check is dumped decrypted or encrypted as asked."+ :visibility :internal+ :encryption :when-encryption-key-set)+(deftest dump-deletes-target-db-files-tests;; test fails when the application db is anything but H2 presently;; TODO: make this test work with postgres / mysql / mariadb@@ -79,7 +85,7 @@;; does) so no encrypted column is left plaintext for the strict model reads;; the update and dump below trigger(mdb.encryption/encrypt-db driver/*driver* (:data-source mdb.connection/*application-db*) nil)- (t2/insert! :model/Setting {:key "my-site-admin", :value "baz"})+ (t2/insert! :model/Setting {:key "dump-to-h2-test-setting", :value "baz"})(t2/update! :model/Database 1 {:details {:db "/tmp/test.db"}})(dump-to-h2/dump-to-h2! h2-file-plaintext {:dump-plaintext? true})(dump-to-h2/dump-to-h2! h2-file-enc {:dump-plaintext? false})@@ -87,7 +93,7 @@(testing "decodes settings and dashboard.details"(with-open [target-conn (.getConnection (copy.h2/h2-data-source h2-file-plaintext))](is (= "baz" (:value (first (jdbc/query {:connection target-conn}- "select \"VALUE\" from SETTING where \"KEY\"='my-site-admin';")))))+ "select \"VALUE\" from SETTING where \"KEY\"='dump-to-h2-test-setting';")))))(is (= "{\"db\":\"/tmp/test.db\"}"(:details (first (jdbc/query {:connection target-conn}"select details from metabase_database where id=1;")))))))@@ -95,7 +101,7 @@(with-open [target-conn (.getConnection (copy.h2/h2-data-source h2-file-enc))](is (not (= "baz"(:value (first (jdbc/query {:connection target-conn}- "select \"VALUE\" from SETTING where \"KEY\"='my-site-admin';"))))))+ "select \"VALUE\" from SETTING where \"KEY\"='dump-to-h2-test-setting';"))))))(is (not (= "{\"db\":\"/tmp/test.db\"}"(:details (first (jdbc/query {:connection target-conn}"select details from metabase_database where id=1;"))))))))@@ -103,7 +109,7 @@(with-open [target-conn (.getConnection (copy.h2/h2-data-source h2-file-default-enc))](is (not (= "baz"(:value (first (jdbc/query {:connection target-conn}- "select \"VALUE\" from SETTING where \"KEY\"='my-site-admin';"))))))+ "select \"VALUE\" from SETTING where \"KEY\"='dump-to-h2-test-setting';"))))))(is (not (= "{\"db\":\"/tmp/test.db\"}"(:details (first (jdbc/query {:connection target-conn}"select details from metabase_database where id=1;")))))))))))))))diff --git a/test/metabase/cmd/enable_encryption_test.clj b/test/metabase/cmd/enable_encryption_test.cljindex 65d4912a23b..1c6089c5663 100644--- a/test/metabase/cmd/enable_encryption_test.clj+++ b/test/metabase/cmd/enable_encryption_test.clj@@ -5,6 +5,7 @@[metabase.app-db.core :as mdb][metabase.cmd.core :as cmd][metabase.cmd.enable-encryption :refer [enable-encryption!]]+ [metabase.settings.core :refer [defsetting]][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util.encryption :as encryption]@@ -15,6 +16,11 @@(use-fixtures :once (fixtures/initialize :db))+(defsetting enable-encryption-test-setting+ "Test setting -- a plaintext setting row for `enable-encryption` to encrypt."+ :visibility :internal+ :encryption :no)+(defn- raw-value [k](t2/select-one-fn :value :setting :key k))@@ -32,7 +38,7 @@(encryption-test/with-secret-key nil(mt/with-temp-empty-app-db [_conn :h2](mdb/setup-db! :create-sample-content? true)- (t2/insert! :model/Setting {:key "test-setting", :value "plain value"})+ (t2/insert! :model/Setting {:key "enable-encryption-test-setting", :value "plain value"});; the v53 migration's legacy plaintext marker; new code never writes it and reads it as "no sentinel"(is (= "unencrypted" (raw-value "encryption-check")))(encryption-test/with-secret-key "key1"@@ -42,8 +48,8 @@(testing "the command encrypts everything and writes the sentinel"(enable-encryption!)(is (encryption/decryptable-string? (raw-value "encryption-check")))- (is (encryption/decryptable-string? (raw-value "test-setting")))- (is (= "plain value" (t2/select-one-fn :value :model/Setting :key "test-setting")))+ (is (encryption/decryptable-string? (raw-value "enable-encryption-test-setting")))+ (is (= "plain value" (t2/select-one-fn :value :model/Setting :key "enable-encryption-test-setting")))(is (encryption/decryptable-string? (t2/select-one-fn :details :metabase_database)))(is (map? (t2/select-one-fn :details :model/Database))))(testing "startup now succeeds"diff --git a/test/metabase/cmd/remove_encryption_test.clj b/test/metabase/cmd/remove_encryption_test.cljindex bd20bfd48f8..aee9f91b175 100644--- a/test/metabase/cmd/remove_encryption_test.clj+++ b/test/metabase/cmd/remove_encryption_test.clj@@ -5,6 +5,7 @@[metabase.app-db.core :as mdb][metabase.cmd.core :as cmd][metabase.cmd.remove-encryption :refer [remove-encryption!]]+ [metabase.settings.core :refer [defsetting]][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util.encryption :as encryption]@@ -15,6 +16,11 @@(use-fixtures :once (fixtures/initialize :db))+(defsetting remove-encryption-test-setting+ "Test setting -- an encrypted setting row for `remove-encryption` to decrypt."+ :visibility :internal+ :encryption :when-encryption-key-set)+(defn- raw-value [data-source keyy](-> (jdbc/query {:connection data-source}["select \"VALUE\" from setting where setting.\"KEY\"=?;" keyy])@@ -28,12 +34,12 @@(deftest remove-encryption!-test(testing "removing encryption"- (encryption-test/with-secret-key "key1"- (mt/with-temp-empty-app-db [_conn :h2]+ (mt/with-temp-empty-app-db [_conn :h2]+ (encryption-test/with-secret-key "key1"(mdb/setup-db! :create-sample-content? true)- (t2/insert! :model/Setting {:key "test-setting", :value "unencrypted value"})+ (t2/insert! :model/Setting {:key "remove-encryption-test-setting", :value "unencrypted value"})(is (encryption/decryptable-string? (raw-value _conn "encryption-check")))- (is (encryption/decryptable-string? (raw-value _conn "test-setting")))+ (is (encryption/decryptable-string? (raw-value _conn "remove-encryption-test-setting")))(remove-encryption!)(is (= "unencrypted" (raw-value _conn "encryption-check")))- (is (not (encryption/possibly-encrypted-string? (raw-value _conn "test-setting"))))))))+ (is (not (encryption/possibly-encrypted-string? (raw-value _conn "remove-encryption-test-setting"))))))))diff --git a/test/metabase/cmd/rotate_encryption_key_test.clj b/test/metabase/cmd/rotate_encryption_key_test.cljindex 4f1d3eaaa48..ce90229c547 100644--- a/test/metabase/cmd/rotate_encryption_key_test.clj+++ b/test/metabase/cmd/rotate_encryption_key_test.clj@@ -14,6 +14,7 @@[metabase.config.core :as config][metabase.driver :as driver][metabase.models.interface :as mi]+ [metabase.settings.core :refer [defsetting]][metabase.test :as mt][metabase.test.data.interface :as tx][metabase.test.fixtures :as fixtures]@@ -29,6 +30,16 @@(use-fixtures :once (fixtures/initialize :db))+(defsetting rotate-test-plaintext-setting+ "Test setting -- a setting that is not encrypted at rest."+ :visibility :internal+ :encryption :no)++(defsetting rotate-test-encrypted-setting+ "Test setting -- a setting that is encrypted at rest."+ :visibility :internal+ :encryption :when-encryption-key-set)+(defn- raw-value [keyy](:value (first (jdbc/query {:datasource (mdb/data-source)}[(if (= driver/*driver* :h2)@@ -83,7 +94,7 @@(tx/create-db! driver/*driver* {:database-name db-name}))(binding [copy/*copy-h2-database-details* true](load-from-h2/load-from-h2! h2-fixture-db-file))- (t2/insert! :model/Setting {:key "nocrypt", :value "unencrypted value"})+ (t2/insert! :model/Setting {:key "rotate-test-plaintext-setting", :value "unencrypted value"})(t2/insert! :model/Setting {:key "settings-last-updated", :value original-timestamp})(let [u (first (t2/insert-returning-instances! :model/User {:email "[email protected]":first_name "No"@@ -98,7 +109,7 @@:creator_id @user-id}))](reset! secret-id-unenc (u/the-id secret)))(encryption-test/with-secret-key k1- (t2/insert! :model/Setting {:key "k1crypted", :value "encrypted with k1"})+ (t2/insert! :model/Setting {:key "rotate-test-encrypted-setting", :value "encrypted with k1"})(mdb.encryption/encrypt-db driver/*driver* data-source nil)(t2/update! :model/Database 1 {:details {:db "/tmp/test.db"}});; other encrypted-json columns that must also be re-encrypted on rotation@@ -114,13 +125,13 @@(rotate-encryption-key! k1);; plain->newkey(testing "for unencrypted values"- (is (not= "unencrypted value" (raw-value "nocrypt")))- (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "nocrypt")))+ (is (not= "unencrypted value" (raw-value "rotate-test-plaintext-setting")))+ (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "rotate-test-plaintext-setting")))(is (mt/secret-value-equals? secret-val (t2/select-one-fn :value :model/Secret :id @secret-id-unenc))));; samekey->samekey(testing "for values encrypted with the same key"- (is (not= "encrypted with k1" (raw-value "k1crypted")))- (is (= "encrypted with k1" (t2/select-one-fn :value :model/Setting :key "k1crypted")))+ (is (not= "encrypted with k1" (raw-value "rotate-test-encrypted-setting")))+ (is (= "encrypted with k1" (t2/select-one-fn :value :model/Setting :key "rotate-test-encrypted-setting")))(is (mt/secret-value-equals? secret-val (t2/select-one-fn :value :model/Secret :id @secret-id-enc))))))(testing "settings-last-updated is updated AND plaintext"(is (not= original-timestamp (raw-value "settings-last-updated")))@@ -129,7 +140,7 @@(encryption-test/with-secret-key k1 (rotate-encryption-key! k2))(testing "with new key"(encryption-test/with-secret-key k2- (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "nocrypt")))+ (is (= "unencrypted value" (t2/select-one-fn :value :model/Setting :key "rotate-test-plaintext-setting")))(is (= {:db "/tmp/test.db"} (t2/select-one-fn :details :model/Database :id 1)))(is (= {:database-enable-actions true} (t2/select-one-fn :settings :model/Database :id 1)))(is (= {:locale "en"} (t2/select-one-fn :settings :model/User :id @user-id)))@@ -138,7 +149,7 @@(encryption-test/with-secret-key k1;; reading a value that looks encrypted but can't be decrypted with the current key throws rather;; than returning the raw ciphertext, so it can never be mistaken for a real plaintext value- (is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :value :model/Setting :key "nocrypt")))+ (is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :value :model/Setting :key "rotate-test-plaintext-setting")))(is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :details :model/Database :id 1)))(is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :settings :model/Database :id 1)))(is (thrown? clojure.lang.ExceptionInfo (t2/select-one-fn :settings :model/User :id @user-id)))@@ -165,7 +176,7 @@(t2/update! :model/Database {:name "k3"} {:details {:db "/tmp/test.db"}})(encryption-test/with-secret-key k2 ; with the last key that we rotated to in the test(rotate-encryption-key! nil))- (is (= "unencrypted value" (raw-value "nocrypt")))+ (is (= "unencrypted value" (raw-value "rotate-test-plaintext-setting")));; at this point, both the originally encrypted, and the originally unencrypted secret instances;; should be decrypted(is (mt/secret-value-equals? secret-val (t2/select-one-fn :value :model/Secret :id @secret-id-unenc)))@@ -213,7 +224,7 @@(encryption-test/with-secret-key k1(set-encryption-check-raw! (encryption/encrypt (str (random-uuid))))(let [good-id (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"en\"}"))- bad-id (insert-user-with-raw-settings! (encryption/encrypt k2-hashed "{\"locale\":\"fr\"}"))]+ bad-id (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"fr\"}" {:secret-key k2-hashed}))](mdb/decrypt-db :h2 (mdb/data-source))(is (= "{\"locale\":\"en\"}" (t2/select-one-fn :settings :core_user :id good-id)))(is (= "{}" (t2/select-one-fn :settings :core_user :id bad-id)))@@ -234,7 +245,7 @@(mt/with-empty-h2-app-db!(encryption-test/with-secret-key k1(t2/delete! :setting :key "encryption-check")- (insert-user-with-raw-settings! (encryption/encrypt k2-hashed "{\"locale\":\"fr\"}"))+ (insert-user-with-raw-settings! (encryption/encrypt "{\"locale\":\"fr\"}" {:secret-key k2-hashed}))(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Can't decrypt app db with MB_ENCRYPTION_SECRET_KEY"@@ -243,7 +254,7 @@(mt/with-empty-h2-app-db!(encryption-test/with-secret-key k1(set-encryption-check-raw! (encryption/encrypt (str (random-uuid))))- (insert-channel-with-raw-details! (encryption/encrypt k2-hashed "{\"url\":\"http://bad\"}"))+ (insert-channel-with-raw-details! (encryption/encrypt "{\"url\":\"http://bad\"}" {:secret-key k2-hashed}))(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Can't decrypt app db with MB_ENCRYPTION_SECRET_KEY"diff --git a/test/metabase/models/interface_test.clj b/test/metabase/models/interface_test.cljindex 2a2bca20069..ce6a6a2bb47 100644--- a/test/metabase/models/interface_test.clj+++ b/test/metabase/models/interface_test.clj@@ -123,11 +123,11 @@(is (= {:a 1}(encryption-test/with-secret-key "qwe"(mi/encrypted-json-out- (encryption/encrypt (encryption/secret-key->hash "qwe") "{\"a\": 1}"))))))+ (encryption/encrypt "{\"a\": 1}" {:secret-key (encryption/secret-key->hash "qwe")}))))))(testing "Logs an error message when incoming data looks encrypted"(mt/with-log-messages-for-level [messages :error](mi/encrypted-json-out- (encryption/encrypt (encryption/secret-key->hash "qwe") "{\"a\": 1}"))+ (encryption/encrypt "{\"a\": 1}" {:secret-key (encryption/secret-key->hash "qwe")}))(is (=? [{:level :error:e nil:message "Could not decrypt encrypted field! Have you forgot to set MB_ENCRYPTION_SECRET_KEY?"}]@@ -140,7 +140,7 @@(mt/with-log-messages-for-level [messages :error](encryption-test/with-secret-key "qwe"(mi/encrypted-json-out- (encryption/encrypt (encryption/secret-key->hash "qwe") "{\"a\": 1")))+ (encryption/encrypt "{\"a\": 1" {:secret-key (encryption/secret-key->hash "qwe")})))(is (=? [{:level :error, :e nil, :message #"(?s)^Error parsing JSON: .*"}](messages)))))))diff --git a/test/metabase/models/serialization/resolve/db_test.clj b/test/metabase/models/serialization/resolve/db_test.cljindex 331e1b2c4d4..44bb38170cc 100644--- a/test/metabase/models/serialization/resolve/db_test.clj+++ b/test/metabase/models/serialization/resolve/db_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase.models.serialization :as serdes]+ [metabase.settings.models.setting.cache :as setting.cache][metabase.test :as mt][toucan2.core :as t2]))@@ -98,6 +99,7 @@(deftest cached-export-database-fk-test(mt/with-temp [:model/Database {db-id :id} {:name "test-db"}]+ (setting.cache/restore-cache!)(serdes/with-cache(t2/with-call-count [call-count](is (= "test-db" (serdes/*export-database-fk* db-id)))diff --git a/test/metabase/settings/models/setting/cache_test.clj b/test/metabase/settings/models/setting/cache_test.cljindex af29e773904..81520ae411e 100644--- a/test/metabase/settings/models/setting/cache_test.clj+++ b/test/metabase/settings/models/setting/cache_test.clj@@ -2,11 +2,13 @@(:require[clojure.test :refer :all][metabase.app-db.core :as mdb]+ [metabase.app-db.setting :as mdb.setting][metabase.settings.models.setting-test :as setting-test][metabase.settings.models.setting.cache :as setting.cache][metabase.system.core :as system][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util.encryption :as encryption][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -23,15 +25,25 @@(defn- update-settings-last-updated-value-in-db!"Simulate a different instance updating the value of `settings-last-updated` in the DB by updating its value without- updating our locally cached value.."+ updating our locally cached value.++ Written raw, and to both columns, exactly as+ [[metabase.settings.models.setting.cache/update-settings-last-updated!]] writes them -- through `:model/Setting` the+ raw SQL below would end up inside the JSON envelope rather than being evaluated."[]- (t2/update! :model/Setting {:key setting.cache/settings-last-updated-key}- {:value ^:allow-raw-sql [:raw (case (mdb/db-type)- ;; make it one second in the future so we don't end up getting an exact match when we try to test- ;; to see if things update below- :h2 "cast(dateadd('second', 1, current_timestamp) AS text)"- :mysql "cast((current_timestamp + interval 1 second) AS char)"- :postgres "cast((current_timestamp + interval '1 second') AS text)")]}))+ (let [ts (-> (t2/query-one+ {:select [[^:allow-raw-sql+ [:raw (case (mdb/db-type)+ ;; make it one second in the future so we don't end up getting an exact match when we try to test+ ;; to see if things update below+ :h2 "cast(dateadd('second', 1, current_timestamp) AS text)"+ :mysql "cast((current_timestamp + interval 1 second) AS char)"+ :postgres "cast((current_timestamp + interval '1 second') AS text)")]+ :timestamp]]})+ :timestamp)]+ (t2/update! :setting {:key setting.cache/settings-last-updated-key}+ {:value ts+ :value_with_aad (encryption/maybe-encrypt ts {:aad (mdb.setting/setting-aad setting.cache/settings-last-updated-key)})})))(defn- simulate-another-instance-updating-setting! [setting-name new-value](if new-valuediff --git a/test/metabase/settings/models/setting_test.clj b/test/metabase/settings/models/setting_test.cljindex 9e75ed7ae78..f8e40d1c1cb 100644--- a/test/metabase/settings/models/setting_test.clj+++ b/test/metabase/settings/models/setting_test.clj@@ -7,6 +7,7 @@[medley.core :as m][metabase.app-db.connection :as mdb.connection][metabase.app-db.core :as mdb]+ [metabase.app-db.setting :as mdb.setting][metabase.cloud-migration.models.cloud-migration :as cloud-migration][metabase.config.core :as config][metabase.settings.models.setting :as setting :refer [defsetting]]@@ -26,7 +27,7 @@(set! *warn-on-reflection* true);; side-effect require: registers the DML build guard exercised by-;; [[migrate-encrypted-settings!-does-not-depend-on-settings-cache-test]]+;; [[migrate-settings!-repairs-past-the-read-only-mode-guard-test]](comment cloud-migration/keep-me)(use-fixtures :once (fixtures/initialize :db))@@ -617,52 +618,104 @@;;; ----------------------------------------------- Encrypted Settings ------------------------------------------------(defn- actual-value-in-db [setting-key]- (-> (mdb/query {:select [:value]+(defn- raw-setting-value-with-aad+ "The setting's `value_with_aad` exactly as it sits in the DB, bypassing the model's decrypting read."+ [setting-key]+ (-> (mdb/query {:select [:value_with_aad]:from [:setting]:where [:= :key (name setting-key)]})- first :value))+ first :value_with_aad))++(defn- aad-opts+ "Encryption opts binding a value to `setting-key`, as the Setting model stores one."+ [setting-key & {:as more}]+ (merge {:aad (mdb.setting/setting-aad (name setting-key))} more))++(defn- decrypt-setting-value+ "Decrypt `raw`, a `value_with_aad`, under the AAD of `setting-key`."+ [setting-key raw]+ (encryption/decrypt raw (aad-opts setting-key)))(deftest encrypted-settings-test(testing "If encryption is *enabled*, make sure Settings get saved as encrypted!";; Setting an encryption key without running encrypt-db leaves the other encrypted settings in the shared app DB;; stored plaintext, and restoring the whole-table settings cache strictly decrypts every one of them. Use an- ;; isolated app DB so this test's key only meets settings it wrote itself.+ ;; isolated app DB so this test's key only meets settings it wrote itself -- and write without a key first: every+ ;; write under a key leaves ciphertext behind (the last-updated marker included) that nothing can read once the+ ;; key is gone.(mt/with-temp-empty-app-db [_conn :h2](mdb/setup-db! :create-sample-content? false)+ (testing "If encryption is not enabled, of course Settings shouldn't get saved as encrypted."+ (encryption-test/with-secret-key nil+ (toucan-name! "Sad Can")+ (is (= "Sad Can"+ (raw-setting-value-with-aad :toucan-name)))))(encryption-test/with-secret-key "ABCDEFGH12345678"+ ;; every row so far is plaintext, which a strict read under a key rejects -- so do what `enable-encryption`+ ;; does before the first write restores the settings cache over them+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)(toucan-name! "Sad Can")- (is (u/base64-string? (actual-value-in-db :toucan-name)))- (testing "make sure it can be decrypted as well..."+ (is (u/base64-string? (raw-setting-value-with-aad :toucan-name)))+ (testing "the ciphertext is bound to the setting it belongs to: it decrypts under that name and no other"(is (= "Sad Can"- (toucan-name)))))- (testing "But if encryption is not enabled, of course Settings shouldn't get saved as encrypted."- (encryption-test/with-secret-key nil- (toucan-name! "Sad Can")+ (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name))))+ (is (thrown? Throwable (encryption/decrypt (raw-setting-value-with-aad :toucan-name))))+ (is (thrown? Throwable (decrypt-setting-value :site-name (raw-setting-value-with-aad :toucan-name)))))+ (testing "make sure it can be decrypted as well..."(is (= "Sad Can"- (actual-value-in-db :toucan-name))))))))+ (toucan-name))))))))(deftest decrypt-error-names-setting-test(testing "a Setting row that fails the decrypting read names the setting in the message (and never the value)"(encryption-test/with-secret-key "0123456789abcdef"(let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"Error decrypting setting \"toucan-name\": Expected an encrypted value"- (#'setting/decrypt-setting-value-on-read {:key "toucan-name" :value "plaintext-sekret"})))]+ #"Error reading setting \"toucan-name\": Expected an encrypted value"+ (#'setting/read-setting-value {:key "toucan-name" :value_with_aad "plaintext-sekret"})))](is (not (re-find #"sekret" (ex-message e))))(is (= "toucan-name" (:setting-key (ex-data e))))))))+(deftest setting-value-aad-test+ (let [read #'setting/read-setting-value]+ (encryption-test/with-secret-key "ABCDEFGH12345678"+ (testing "a value round trips through the AAD naming its setting"+ (is (= "Sad Can"+ (:value (read {:key "test-setting-1"+ :value_with_aad (encryption/encrypt "Sad Can" (aad-opts :test-setting-1))})))))+ (testing "a value moved to another setting's row is rejected, and the message never carries the value"+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Error reading setting \"test-setting-2\""+ (read {:key "test-setting-2"+ :value_with_aad (encryption/encrypt "Sad Can" (aad-opts :test-setting-1))})))]+ (is (not (re-find #"Sad Can" (ex-message e))))))+ (testing "a plaintext value is rejected while a key is set"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Expected an encrypted value"+ (read {:key "test-setting-1" :value_with_aad "Sad Can"}))))+ (testing "a row written only by a version predating `value_with_aad` reads as no value at all"+ (is (= {:key "test-setting-1" :value nil :value_with_aad nil}+ (read {:key "test-setting-1" :value "Sad Can" :value_with_aad nil}))))+ (testing "a key with no `defsetting` reads as no value at all, without so much as a decrypt"+ (is (= {:key "no-such-setting" :value nil :value_with_aad "not even ciphertext"}+ (read {:key "no-such-setting" :value_with_aad "not even ciphertext"})))))+ (testing "a key with no `defsetting` cannot be written: how it is stored is the setting's to decide"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unknown setting: no-such-setting"+ (#'setting/write-setting-value {:key "no-such-setting" :value "Sad Can"}))))))+(deftest previously-encrypted-settings-test(testing "Make sure settings that were encrypted don't cause `user-facing-info` to blow up if encyrption key changed"(mt/with-temp-empty-app-db [_conn :h2](mdb/setup-db! :create-sample-content? false)- (mt/discard-setting-changes [test-json-setting]- (encryption-test/with-secret-key "0B9cD6++AME+A7/oR7Y2xvPRHX3cHA2z7w+LbObd/9Y="- (test-json-setting! {:abc 123})- (is (not= "{\"abc\":123}"- (actual-value-in-db :test-json-setting))))- (testing (str "If fetching the Setting fails (e.g. because key changed) `user-facing-info` should return `nil` "- "rather than failing entirely")- (encryption-test/with-secret-key nil+ ;; No `discard-setting-changes`: restoring the setting means writing it back without the key it was written+ ;; under, and a write restores the whole settings cache over the row it cannot read. The temp app DB is thrown+ ;; away either way.+ (encryption-test/with-secret-key "0B9cD6++AME+A7/oR7Y2xvPRHX3cHA2z7w+LbObd/9Y="+ (test-json-setting! {:abc 123})+ (is (not= "{\"abc\":123}"+ (raw-setting-value-with-aad :test-json-setting))))+ (testing (str "If fetching the Setting fails (e.g. because key changed) `user-facing-info` should return `nil` "+ "rather than failing entirely")+ (encryption-test/with-secret-key nil+ (binding [config/*disable-setting-cache* true](is (= {:key :test-json-setting:value nil:is_env_setting false@@ -708,7 +761,7 @@(testing "make sure uncached setting still saves to the DB"(uncached-setting! "ABCDEF")(is (= "ABCDEF"- (actual-value-in-db "uncached-setting"))))+ (raw-setting-value-with-aad "uncached-setting"))))(testing "make sure that fetching the Setting always fetches the latest value from the DB"(uncached-setting! "ABCDEF")(t2/update! :model/Setting {:key "uncached-setting"}@@ -1765,72 +1818,46 @@(testing (format "We have defined a setting for the %s validation tests" format)(is (var? (resolve (ns-validation-setting-symbol format)))))))))-(deftest migrate-encrypted-settings!-works- ;; Isolated app DB: with a secret key active this mutates the at-rest encryption of every registered setting row,- ;; which would poison the shared test DB for later tests running with a different (or no) key.- (mt/with-temp-empty-app-db [_conn :h2]- (mdb/setup-db! :create-sample-content? false)- (testing "It works when a secret key is set"- (encryption-test/with-secret-key "ABCDEFGH12345678"- (t2/insert! :setting {:key "test-never-encrypted-setting" :value (encryption/maybe-encrypt "foobar")})- ;; Sanity check: the value is encrypted- (is (not= "foobar" (actual-value-in-db :test-never-encrypted-setting)))- (setting/migrate-encrypted-settings!)- (is (= "foobar" (actual-value-in-db :test-never-encrypted-setting)))- (setting/migrate-encrypted-settings!)- (is (= "foobar" (actual-value-in-db :test-never-encrypted-setting)))))- (testing "It doesn't do anything when the secret key is not set"+(deftest migrate-settings!-test+ (testing "rows a version predating `value_with_aad` wrote get it filled in from `value` on startup"+ (mt/with-temp-empty-app-db [_conn :h2]+ (mdb/setup-db! :create-sample-content? false)(encryption-test/with-secret-key "ABCDEFGH12345678"+ (t2/delete! :setting :key "toucan-name")(t2/delete! :setting :key "test-never-encrypted-setting")- (t2/insert! :setting {:key "test-never-encrypted-setting" :value (encryption/maybe-encrypt "foobar")}))- (encryption-test/with-secret-key nil- (is (not= "foobar" (actual-value-in-db :test-never-encrypted-setting)))- (setting/migrate-encrypted-settings!)- (is (not= "foobar" (actual-value-in-db :test-never-encrypted-setting)))))))--(deftest migrate-encrypted-settings!-does-not-depend-on-settings-cache-test- ;; The cloud-migration read-only-mode guard (a `t2.pipeline/build :before` method registered when- ;; `metabase.cloud-migration.models.cloud-migration` loads -- required above so this holds in a targeted test run- ;; too) runs on every DML statement and reads a setting. On a fresh JVM that read triggers a full strict- ;; settings-cache restore, which fails on the very plaintext row this function exists to repair -- so the repair- ;; itself must never go through the cache. Regression test for the chicken-and-egg startup crash.+ ;; `value` only, as that version writes it: encrypted for a setting that encrypts, plaintext for one that does not+ (t2/insert! :setting [{:key "toucan-name", :value (encryption/encrypt "Lenny")}+ {:key "test-never-encrypted-setting", :value "foobar"}])+ (mdb.setting/migrate-settings!)+ (testing "an encrypted row's value is stored under its AAD"+ (is (= "Lenny" (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name)))))+ (testing "a plaintext row's value is too -- every setting's is, whatever its :encryption says"+ (is (= "foobar" (decrypt-setting-value :test-never-encrypted-setting+ (raw-setting-value-with-aad :test-never-encrypted-setting)))))+ (testing "and the values are readable again"+ (setting.cache/restore-cache!)+ (is (= "Lenny" (toucan-name))))+ (testing "a row that already has a stored value is left byte-identical, whatever `value` says beside it"+ (toucan-name! "Sad Can")+ (let [before (raw-setting-value-with-aad :toucan-name)]+ (t2/update! :setting :key "toucan-name" {:value (encryption/encrypt "Bird Can")})+ (mdb.setting/migrate-settings!)+ (is (= before (raw-setting-value-with-aad :toucan-name)))+ (is (= "Sad Can" (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name))))))))))++(deftest migrate-settings!-writes-past-the-read-only-mode-guard-test+ ;; The cloud-migration guard on Toucan DML (registered when `metabase.cloud-migration.models.cloud-migration`+ ;; loads -- required above so this holds in a targeted run too) reads `read-only-mode` through the model before+ ;; every update, and the model's read is strict. The backfill's writes must not go through it: here that row holds a+ ;; plaintext value under a key, which the strict read rejects, and the backfill still has to get its other rows done.(mt/with-temp-empty-app-db [_conn :h2](mdb/setup-db! :create-sample-content? false)(encryption-test/with-secret-key "ABCDEFGH12345678"- (t2/insert! :setting {:key "toucan-name" :value "Lenny"})- (binding [config/*disable-setting-cache* false]- ;; Simulate a fresh JVM. `setting.cache/cache*` is the atom holding this app DB's in-memory settings map; nil- ;; means never populated, so the next cached read must do the full (strictly decrypting) restore. And- ;; `last-update-check` is the AtomicLong nanotime of the last staleness check: zeroing it defeats the- ;; one-minute throttle that otherwise skips the check entirely in a warm test JVM.- (reset! (#'setting.cache/cache*) nil)- (.set ^java.util.concurrent.atomic.AtomicLong @#'setting.cache/last-update-check 0)- (setting/migrate-encrypted-settings!))- (is (encryption/decryptable-string? (actual-value-in-db :toucan-name)))- (is (= "Lenny" (encryption/decrypt (actual-value-in-db :toucan-name)))))))--(deftest migrate-encrypted-settings!-encrypts-strict-settings- ;; raw :setting (not :model/Setting) throughout: the model's before-insert would encrypt the value, and these tests- ;; need genuinely plaintext rows at rest. Isolated app DB for the same reason as [[migrate-encrypted-settings!-works]].- (mt/with-temp-empty-app-db [_conn :h2]- (mdb/setup-db! :create-sample-content? false)- (testing "a plaintext row of a setting that encrypts is encrypted at rest on startup (e.g. after a downgraded boot decrypted it)"- (encryption-test/with-secret-key "ABCDEFGH12345678"- (t2/insert! :setting {:key "toucan-name" :value "Lenny"})- (is (not (encryption/decryptable-string? (actual-value-in-db :toucan-name))))- (setting/migrate-encrypted-settings!)- (is (encryption/decryptable-string? (actual-value-in-db :toucan-name)))- (is (= "Lenny" (encryption/decrypt (actual-value-in-db :toucan-name))))- (testing "already-encrypted rows are left byte-identical"- (let [before (actual-value-in-db :toucan-name)]- (setting/migrate-encrypted-settings!)- (is (= before (actual-value-in-db :toucan-name)))))))- (testing "without an encryption key nothing happens"- (encryption-test/with-secret-key nil- (t2/delete! :setting :key "toucan-name")- (t2/insert! :setting {:key "toucan-name" :value "Lenny"})- (setting/migrate-encrypted-settings!)- (is (= "Lenny" (actual-value-in-db :toucan-name)))))))+ (mdb/encrypt-db (mdb/db-type) (mdb/data-source) nil)+ (t2/insert! :setting [{:key "read-only-mode", :value "false", :value_with_aad "false"}+ {:key "toucan-name", :value (encryption/encrypt "Lenny")}])+ (mdb.setting/migrate-settings!)+ (is (= "Lenny" (decrypt-setting-value :toucan-name (raw-setting-value-with-aad :toucan-name)))))))(deftest setter-none-does-not-imply-encryption-test(testing "`:setter :none` does not imply encryption -- it is decided by type or stated explicitly, like any setting"diff --git a/test/metabase/setup/core_test.clj b/test/metabase/setup/core_test.cljindex 36c0b1647f1..25f2c77e1e8 100644--- a/test/metabase/setup/core_test.clj+++ b/test/metabase/setup/core_test.clj@@ -5,12 +5,14 @@[metabase.app-db.connection :as mdb.connection][metabase.app-db.core :as mdb][metabase.app-db.encryption :as mdb.encryption]+ [metabase.app-db.setting :as mdb.setting][metabase.appearance.core :as appearance][metabase.config.core :as config][metabase.driver :as driver][metabase.models.interface :as mi][metabase.query-processor.middleware.cache-backend.interface :as i][metabase.settings.models.setting :as setting]+ [metabase.settings.models.setting.cache :as setting.cache][metabase.setup.core :as setup][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -35,6 +37,7 @@(testing "The has-user-setup getter should cache truthy results since it can never become falsey";; make sure some test users are created.(mt/initialize-if-needed! :test-users)+ (setting.cache/restore-cache!)(t2/with-call-count [call-count];; call has-user-setup several times.(dotimes [_ 5]@@ -56,6 +59,7 @@(is (<= (call-count)10)))))) ;; in dev/test we check settings for an override(testing "Switch back to the 'normal' app DB; value should still be cached for it"+ (setting.cache/restore-cache!)(t2/with-call-count [call-count](is (true?(setup/has-user-setup)))@@ -155,8 +159,8 @@(reset! (:status mdb.connection/*application-db*) ::setup-finished)(is (= :done (mdb/setup-db! :create-sample-content? false))))))))))(testing "Database created with encryption configured is encrypted"- (encryption-test/with-secret-key "key2"- (mt/with-temp-empty-app-db [_conn driver/*driver*]+ (mt/with-temp-empty-app-db [_conn driver/*driver*]+ (encryption-test/with-secret-key "key2"(mdb/setup-db! :create-sample-content? true)(is (encryption/decryptable-string? (t2/select-one-fn :value "setting" :key "encryption-check")))(is (encryption/decryptable-string? (t2/select-one-fn :details "metabase_database")))@@ -194,8 +198,8 @@(deftest sentinel-state-never-triggers-encryption-test(testing "a restart must never encrypt a pre-existing plaintext row, whatever state the sentinel is in"- (encryption-test/with-secret-key "sentinel-state-key-1"- (mt/with-temp-empty-app-db [_conn :h2]+ (mt/with-temp-empty-app-db [_conn :h2]+ (encryption-test/with-secret-key "sentinel-state-key-1"(mdb/setup-db! :create-sample-content? true)(let [db-id (t2/select-one-fn :id :metabase_database)plaintext "{\"host\":\"example.com\"}"@@ -226,8 +230,8 @@(deftest fresh-install-with-encryption-key-test(testing "a database created with MB_ENCRYPTION_SECRET_KEY set stores every encrypted-at-rest value encrypted"- (encryption-test/with-secret-key "fresh-install-test-key-1234"- (mt/with-temp-empty-app-db [_conn :h2]+ (mt/with-temp-empty-app-db [_conn :h2]+ (encryption-test/with-secret-key "fresh-install-test-key-1234"(mdb/setup-db! :create-sample-content? true)(testing "encrypted-at-rest columns (read raw, so no model transform can hide a plaintext value)"(doseq [[table column] @#'mdb.encryption/encrypted-string-columns@@ -244,4 +248,8 @@:let [definition (get @setting/registered-settings (keyword k))]:when (and definition (not= :no (:encryption definition)))](testing k- (is (encryption/decryptable-string? v)))))))))+ (is (encryption/decryptable-string? v)))))+ (testing "every setting's value_with_aad, under its own setting's AAD"+ (doseq [{k :key v :value_with_aad} (t2/select :setting {:where [:!= :value_with_aad nil]})]+ (testing k+ (is (encryption/decryptable-string? v {:aad (mdb.setting/setting-aad k)})))))))))diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex c7386294922..260ec4c454c 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -18,6 +18,7 @@[mb.hawk.parallel][metabase.analytics.prometheus :as prometheus][metabase.app-db.core :as mdb]+ [metabase.app-db.setting :as mdb.setting][metabase.app-db.transient-error :as transient-error][metabase.audit-app.core :as audit][metabase.classloader.core :as classloader]@@ -45,6 +46,7 @@[metabase.test.util.log][metabase.timeline.models.timeline-event :as timeline-event][metabase.util :as u]+ [metabase.util.encryption :as encryption][metabase.util.files :as u.files][metabase.util.json :as json][metabase.util.random :as u.random]@@ -567,21 +569,31 @@(list `with-temp-env-var-value! '[a])(list `with-temp-env-var-value! '[a b c]))))+(defn- raw-setting+ "The `setting` row for `setting-k` as it sits in the table, or nil."+ [setting-k]+ (t2/select-one [:setting :value :value_with_aad] :key setting-k))+(defn- upsert-raw-setting!- [original-value setting-k value]+ "Write `value` for `setting-k` straight into the table, bypassing the model and so any setter: `value` bare, and+ `value_with_aad` the way the model stores it, so the app reads the value back. A nil `value` removes the row."+ [original setting-k value](if (some? value)- (if original-value- (t2/update! :model/Setting setting-k {:value value})- (t2/insert! :model/Setting :key setting-k :value value))- (when original-value- (t2/delete! :model/Setting :key setting-k)))+ (let [row {:value value+ :value_with_aad (encryption/maybe-encrypt value {:aad (mdb.setting/setting-aad setting-k)})}]+ (if original+ (t2/update! :setting :key setting-k row)+ (t2/insert! :setting (assoc row :key setting-k))))+ (when original+ (t2/delete! :setting :key setting-k)))(setting.cache/restore-cache!))(defn- restore-raw-setting!- [original-value setting-k]- (if original-value- (t2/update! :model/Setting setting-k {:value original-value})- (t2/delete! :model/Setting :key setting-k))+ "Put back the row [[raw-setting]] found, byte for byte, or remove the one written over nothing."+ [original setting-k]+ (if original+ (t2/update! :setting :key setting-k original)+ (t2/delete! :setting :key setting-k))(setting.cache/restore-cache!))(defn do-with-temporary-setting-value!@@ -610,7 +622,7 @@(if (and (not raw-setting?) (setting/env-var-value setting-k))(do-with-temp-env-var-value! (setting/setting-env-map-name setting-k) value thunk)(let [original-value (if raw-setting?- (t2/select-one-fn :value :model/Setting :key setting-k)+ (raw-setting setting-k)(if skip-init?(setting/read-setting setting-k)(setting/get setting-k)))]diff --git a/test/metabase/util/encryption_test.clj b/test/metabase/util/encryption_test.cljindex ccaf4d84a8b..6b90d5110a3 100644--- a/test/metabase/util/encryption_test.clj+++ b/test/metabase/util/encryption_test.clj@@ -13,17 +13,25 @@(set! *warn-on-reflection* true)+(defn- clear-setting-cache!+ "Empty the Setting cache so values have to be fetched from the DB again. Emptied rather than restored: restoring+ reads and decrypts every setting row there and then, which fails on rows written under a key other than the one in+ effect at the boundary -- exactly what changing keys leaves behind. The once-a-minute throttle on the check that+ repopulates an empty cache is zeroed too, so the next read restores it whole instead of going to the DB row by row."+ []+ (reset! (#'setting.cache/cache*) nil)+ (.set ^java.util.concurrent.atomic.AtomicLong @#'setting.cache/last-update-check 0))+(defn do-with-secret-key! [^String secret-key thunk]- ;; flush the Setting cache so unencrypted values have to be fetched from the DB again(initialize/initialize-if-needed! :db)- (setting.cache/restore-cache!)+ (clear-setting-cache!)(try(with-redefs [encryption/default-secret-key (when (seq secret-key)(encryption/secret-key->hash secret-key))](thunk))(finally- ;; reset the cache again so nothing that happened during the test is persisted.- (setting.cache/restore-cache!))))+ ;; clear the cache again so nothing that happened during the test is persisted.+ (clear-setting-cache!))))(defmacro with-secret-key"Run `body` with the encryption secret key temporarily bound to `secret-key`. Useful for testing how functions behave@@ -50,49 +58,49 @@(deftest ^:parallel hash-pattern-test(is (re= #"^[0-9A-Za-z/+]+=*$"- (encryption/encrypt secret "Hello!"))))+ (encryption/encrypt "Hello!" {:secret-key secret}))))(deftest ^:parallel hashing-isnt-idempotent-test(testing "test that encrypting something twice gives you two different ciphertexts"- (is (not= (encryption/encrypt secret "Hello!")- (encryption/encrypt secret "Hello!")))))+ (is (not= (encryption/encrypt "Hello!" {:secret-key secret})+ (encryption/encrypt "Hello!" {:secret-key secret})))))(deftest ^:parallel decrypt-test(testing "test that we can decrypt something"(is (= "Hello!"- (encryption/decrypt secret (encryption/encrypt secret "Hello!"))))))+ (encryption/decrypt (encryption/encrypt "Hello!" {:secret-key secret}) {:secret-key secret})))))(deftest ^:parallel decrypt-bytes-test(testing "test that we can decrypt binary data"(let [data (byte-array (range 0 100))](is (= (seq data)- (seq (encryption/decrypt-bytes secret (encryption/encrypt-bytes secret data))))))))+ (seq (encryption/decrypt-bytes (encryption/encrypt-bytes data {:secret-key secret}) {:secret-key secret})))))))(deftest ^:parallel exception-with-wrong-decryption-key-test(testing "trying to decrypt something with the wrong key with `decrypt` should throw an Exception"(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Message seems corrupt or manipulated"- (encryption/decrypt secret-2 (encryption/encrypt secret "WOW"))))))+ (encryption/decrypt (encryption/encrypt "WOW" {:secret-key secret}) {:secret-key secret-2})))))(deftest ^:parallel maybe-decrypt-not-encrypted-test(testing "trying to `maybe-decrypt-accepting-plaintext` something that's not encrypted should return it as-is"(is (= "{\"a\":100}"- (encryption/maybe-decrypt-accepting-plaintext secret "{\"a\":100}")))+ (encryption/maybe-decrypt-accepting-plaintext "{\"a\":100}" {:secret-key secret})))(is (= "abc"- (encryption/maybe-decrypt-accepting-plaintext secret "abc")))))+ (encryption/maybe-decrypt-accepting-plaintext "abc" {:secret-key secret})))))(deftest ^:parallel maybe-decrypt-with-wrong-key-test(testing (str "decrypting something encrypted with a different key using `maybe-decrypt-accepting-plaintext` throws ""rather than returning the ciphertext — returning it would let a re-encrypting caller double-encrypt it")- (let [original-ciphertext (encryption/encrypt secret "WOW")]+ (let [original-ciphertext (encryption/encrypt "WOW" {:secret-key secret})](is (thrown? Throwable- (encryption/maybe-decrypt-accepting-plaintext secret-2 original-ciphertext))))))+ (encryption/maybe-decrypt-accepting-plaintext original-ciphertext {:secret-key secret-2}))))))(deftest ^:parallel no-errors-for-unencrypted-test(testing "Something obviously not encrypted should avoiding trying to decrypt it (and thus not log an error)"(mt/with-log-messages-for-level [messages :warn]- (encryption/maybe-decrypt-accepting-plaintext secret "abc")+ (encryption/maybe-decrypt-accepting-plaintext "abc" {:secret-key secret})(is (empty? (messages))))))(def ^:private fake-ciphertext@@ -104,39 +112,39 @@(deftest ^:parallel possibly-encrypted-test(testing "a value shaped like ciphertext but that cannot be decrypted with the current key throws rather than being returned as-is"(is (thrown? Throwable- (encryption/maybe-decrypt-accepting-plaintext secret fake-ciphertext)))))+ (encryption/maybe-decrypt-accepting-plaintext fake-ciphertext {:secret-key secret})))))(deftest ^:parallel decryptable-string-test(testing "true only for ciphertext that decrypts with the given key"- (let [ciphertext (encryption/encrypt secret "WOW")]- (is (true? (encryption/decryptable-string? secret ciphertext)))+ (let [ciphertext (encryption/encrypt "WOW" {:secret-key secret})]+ (is (true? (encryption/decryptable-string? ciphertext {:secret-key secret})))(testing "false for ciphertext under another key"- (is (false? (encryption/decryptable-string? secret-2 ciphertext))))+ (is (false? (encryption/decryptable-string? ciphertext {:secret-key secret-2}))))(testing "false with no key, even for genuine ciphertext"- (is (false? (encryption/decryptable-string? nil ciphertext))))))+ (is (false? (encryption/decryptable-string? ciphertext {:secret-key nil}))))))(testing "false for anything that is not ciphertext"- (is (false? (encryption/decryptable-string? secret "WOW")))- (is (false? (encryption/decryptable-string? secret "")))- (is (false? (encryption/decryptable-string? secret nil)))+ (is (false? (encryption/decryptable-string? "WOW" {:secret-key secret})))+ (is (false? (encryption/decryptable-string? "" {:secret-key secret})))+ (is (false? (encryption/decryptable-string? nil {:secret-key secret})))(testing "including plaintext shaped like ciphertext, which `possibly-encrypted-string?` cannot tell apart"(is (true? (encryption/possibly-encrypted-string? fake-ciphertext)))- (is (false? (encryption/decryptable-string? secret fake-ciphertext))))))+ (is (false? (encryption/decryptable-string? fake-ciphertext {:secret-key secret}))))))(deftest ^:parallel decryptable-bytes-test(let [plaintext (codecs/to-bytes "WOW")- ciphertext (encryption/encrypt-bytes secret plaintext)+ ciphertext (encryption/encrypt-bytes plaintext {:secret-key secret})fake-bytes (byte-array 64)](testing "true only for ciphertext that decrypts with the given key"- (is (true? (encryption/decryptable-bytes? secret ciphertext)))- (is (false? (encryption/decryptable-bytes? secret-2 ciphertext)))- (is (false? (encryption/decryptable-bytes? nil ciphertext))))+ (is (true? (encryption/decryptable-bytes? ciphertext {:secret-key secret})))+ (is (false? (encryption/decryptable-bytes? ciphertext {:secret-key secret-2})))+ (is (false? (encryption/decryptable-bytes? ciphertext {:secret-key nil}))))(testing "false for anything that is not ciphertext"- (is (false? (encryption/decryptable-bytes? secret plaintext)))- (is (false? (encryption/decryptable-bytes? secret (byte-array 0))))- (is (false? (encryption/decryptable-bytes? secret nil)))+ (is (false? (encryption/decryptable-bytes? plaintext {:secret-key secret})))+ (is (false? (encryption/decryptable-bytes? (byte-array 0) {:secret-key secret})))+ (is (false? (encryption/decryptable-bytes? nil {:secret-key secret})))(testing "including bytes shaped like ciphertext"(is (true? (encryption/possibly-encrypted-bytes? fake-bytes)))- (is (false? (encryption/decryptable-bytes? secret fake-bytes)))))))+ (is (false? (encryption/decryptable-bytes? fake-bytes {:secret-key secret})))))))(deftest ^:parallel possibly-encrypted-returns-booleans-test(testing "the shape checks never return nil"@@ -145,63 +153,63 @@(is (false? (encryption/possibly-encrypted-string? "not base64!")))(is (false? (encryption/possibly-encrypted-bytes? nil)))(is (false? (encryption/possibly-encrypted-bytes? (byte-array 3))))- (is (true? (encryption/possibly-encrypted-string? (encryption/encrypt secret "WOW"))))))+ (is (true? (encryption/possibly-encrypted-string? (encryption/encrypt "WOW" {:secret-key secret}))))))(deftest ^:parallel maybe-decrypt-strict-test(testing "strict `maybe-decrypt`"(testing "decrypts a genuinely encrypted value"- (is (= "WOW" (encryption/maybe-decrypt secret (encryption/encrypt secret "WOW")))))+ (is (= "WOW" (encryption/maybe-decrypt (encryption/encrypt "WOW" {:secret-key secret}) {:secret-key secret}))))(testing "throws on a value that is not encrypted"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"not encrypted"- (encryption/maybe-decrypt secret "{\"a\":100}")))+ (encryption/maybe-decrypt "{\"a\":100}" {:secret-key secret})))(is (thrown-with-msg? clojure.lang.ExceptionInfo #"not encrypted"- (encryption/maybe-decrypt secret "abc"))))+ (encryption/maybe-decrypt "abc" {:secret-key secret}))))(testing "throws on a value that looks encrypted but fails to decrypt (wrong key or corrupt)"- (is (thrown? Throwable (encryption/maybe-decrypt secret fake-ciphertext)))- (is (thrown? Throwable (encryption/maybe-decrypt secret-2 (encryption/encrypt secret "WOW")))))+ (is (thrown? Throwable (encryption/maybe-decrypt fake-ciphertext {:secret-key secret})))+ (is (thrown? Throwable (encryption/maybe-decrypt (encryption/encrypt "WOW" {:secret-key secret}) {:secret-key secret-2}))))(testing "decrypts an encrypted byte array"- (is (= "WOW" (String. ^bytes (encryption/maybe-decrypt-bytes secret (encryption/encrypt-bytes secret (.getBytes "WOW")))))))+ (is (= "WOW" (String. ^bytes (encryption/maybe-decrypt-bytes (encryption/encrypt-bytes (.getBytes "WOW") {:secret-key secret}) {:secret-key secret})))))(testing "passes nil through but rejects a blank (non-encrypted) string"(is (nil? (encryption/maybe-decrypt nil)))(is (thrown-with-msg? clojure.lang.ExceptionInfo #"not encrypted"- (encryption/maybe-decrypt secret ""))))))+ (encryption/maybe-decrypt "" {:secret-key secret}))))))(deftest ^:parallel stream-encryption-test(testing "Can encrypt stream"(let [input-stream (ByteArrayInputStream. (.getBytes "test string"))- encrypted-stream (encryption/encrypt-stream secret input-stream)+ encrypted-stream (encryption/encrypt-stream input-stream {:secret-key secret})output-string (slurp encrypted-stream)](is (not= "test string" output-string))))(testing "Can encrypt and decrypt streams"(let [input-stream (ByteArrayInputStream. (.getBytes "test string"))]- (with-open [encrypted-stream (encryption/encrypt-stream secret input-stream)- decrypted-stream (encryption/maybe-decrypt-stream secret encrypted-stream)]+ (with-open [encrypted-stream (encryption/encrypt-stream input-stream {:secret-key secret})+ decrypted-stream (encryption/maybe-decrypt-stream encrypted-stream {:secret-key secret})](is (= "test string" (slurp decrypted-stream))))))(testing "Can encrypt and decrypt a large stream"(let [data (string/random-string 100000)input-stream (ByteArrayInputStream. (codecs/to-bytes data))]- (with-open [encrypted-stream (encryption/encrypt-stream secret input-stream)- decrypted-stream (encryption/maybe-decrypt-stream secret encrypted-stream)]+ (with-open [encrypted-stream (encryption/encrypt-stream input-stream {:secret-key secret})+ decrypted-stream (encryption/maybe-decrypt-stream encrypted-stream {:secret-key secret})](is (= data (codecs/bytes->str (IOUtils/toByteArray decrypted-stream)))))))(testing "Unencrypted streams come back as-is"(let [input-stream (ByteArrayInputStream. (codecs/to-bytes "test string"))]- (with-open [decrypted-stream (encryption/maybe-decrypt-stream secret input-stream)]+ (with-open [decrypted-stream (encryption/maybe-decrypt-stream input-stream {:secret-key secret})](is (= "test string" (codecs/bytes->str (IOUtils/toByteArray decrypted-stream)))))))(testing "Empty unencrypted streams come back as-is"(let [input-stream (ByteArrayInputStream. (byte-array 0))]- (with-open [decrypted-stream (encryption/maybe-decrypt-stream secret input-stream)]+ (with-open [decrypted-stream (encryption/maybe-decrypt-stream input-stream {:secret-key secret})](is (= -1 (.read decrypted-stream))))))(testing "Long unencrypted streams come back as-is"(let [data (string/random-string 100000)input-stream (ByteArrayInputStream. (codecs/to-bytes data))]- (with-open [decrypted-stream (encryption/maybe-decrypt-stream secret input-stream)]+ (with-open [decrypted-stream (encryption/maybe-decrypt-stream input-stream {:secret-key secret})](is (= data (codecs/bytes->str (IOUtils/toByteArray decrypted-stream))))))))(deftest ^:parallel maybe-encrypt-for-stream-test(testing "When secret is set, it encrypts the stream"- (let [encrypted (encryption/maybe-encrypt-for-stream secret (codecs/to-bytes "test string"))]+ (let [encrypted (encryption/maybe-encrypt-for-stream (codecs/to-bytes "test string") {:secret-key secret})](is (not= "test string" (codecs/bytes->str encrypted)))- (is (= "test string" (slurp (encryption/maybe-decrypt-stream secret (ByteArrayInputStream. encrypted))))))+ (is (= "test string" (slurp (encryption/maybe-decrypt-stream (ByteArrayInputStream. encrypted) {:secret-key secret})))))(testing "When secret is not set, it does not encrypt the stream"- (let [encrypted (encryption/maybe-encrypt-for-stream nil (codecs/to-bytes "test string"))]+ (let [encrypted (encryption/maybe-encrypt-for-stream (codecs/to-bytes "test string") {:secret-key nil})](is (= "test string" (codecs/bytes->str encrypted)))))))diff --git a/test_resources/error-migration.yaml b/test_resources/error-migration.yamlindex dd1c45ea346..120b2e25b3a 100644--- a/test_resources/error-migration.yaml+++ b/test_resources/error-migration.yaml@@ -20,6 +20,9 @@ databaseChangeLog:- column:name: valuetype: text+ - column:+ name: value_with_aad+ type: text- changeSet:id: '1'