Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GHY-4378] Add data_sensitivity col to field (#81744)

Tyler Pludee315710292e6
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index e3b45702827..3a98d822ba0 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1134,24 +1134,34 @@
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
- :model/Field {field-id :id} {:name "Some Field" :table_id no-schema-id}
+ :model/Field {field-id :id} {:name "Some Field"
+ :table_id no-schema-id
+ :data_sensitivity :PII}
+ :model/Field {plain-id :id} {:name "Plain Field" :table_id no-schema-id}
:model/FieldUserSettings {description :description}
- {:field_id field-id
- :description "Some custom Description"}]
+ {:field_id field-id
+ :description "Some custom Description"
+ :data_sensitivity :PII}]
(testing "field values"
(let [ser (serdes/extract-one "FieldUserSettings" {} (t2/select-one :model/FieldUserSettings :field_id field-id))]
- (is (=? {:serdes/meta [{:model "Database" :id "My Database"}
- {:model "Table" :id "Schemaless Table"}
- {:model "Field" :id "Some Field"}
- {:model "FieldUserSettings" :id "1"}] ; Always 1.
- :created_at string?
- :description description}
+ (is (=? {:serdes/meta [{:model "Database" :id "My Database"}
+ {:model "Table" :id "Schemaless Table"}
+ {:model "Field" :id "Some Field"}
+ {:model "FieldUserSettings" :id "1"}] ; Always 1.
+ :created_at string?
+ :description description
+ :data_sensitivity :PII}
ser))
(is (not (contains? ser :field_id))
":field_id is dropped; its implied by the path")
(testing "depend only on the Database; the parent Field is synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
(set (serdes/deserialization-dependencies ser)))))))
+ (testing "data_sensitivity on the Field itself"
+ (is (= :PII (:data_sensitivity (ts/extract-one "Field" field-id)))
+ "a labeled field exports the keyword as-is")
+ (is (not (contains? (ts/extract-one "Field" plain-id) :data_sensitivity))
+ "an unlabeled field exports no key, so nil never reaches the YAML"))
(testing "extract-metabase behavior"
(let [models (->> {} (extract/extract) (map (comp :model last :serdes/meta)))]
(is (= 1
@@ -1178,6 +1188,12 @@
(is (= #{["FieldUserSettings" f2-id]}
(set (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc))))))
(t2/delete! :model/FieldUserSettings :field_id f2-id))
+ (testing "with user-edits-only and a FieldUserSettings row holding only data_sensitivity: that field appears as FieldUserSettings"
+ (t2/insert! :model/FieldUserSettings {:field_id f1-id :data_sensitivity :PII})
+ (let [desc (serdes/descendants "Table" table-id {:user-edits-only true})]
+ (is (= #{["FieldUserSettings" f1-id]}
+ (set (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc))))))
+ (t2/delete! :model/FieldUserSettings :field_id f1-id))
(testing "with user-edits-only and all fields edited: all appear as FieldUserSettings, not Field"
(t2/insert! :model/FieldUserSettings {:field_id f1-id})
(t2/insert! :model/FieldUserSettings {:field_id f2-id})
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index 4c3a513d86e..6b964bb4050 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -2113,6 +2113,44 @@
(is (= "desc"
(t2/select-one-fn :description :model/Field (:id f3)))))))
+(deftest field-data-sensitivity-round-trip-test
+ (testing "data_sensitivity travels through export and import on both Field and FieldUserSettings"
+ (let [serialized (atom nil)
+ in-db? (fn [entity] (-> entity :serdes/meta first :id (= "labeled-db")))
+ field-ser (fn [entities field-name]
+ (u/seek #(and (in-db? %)
+ (-> % :serdes/meta last :model (= "Field"))
+ (= field-name (:name %)))
+ entities))]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [db (ts/create! :model/Database :name "labeled-db")
+ table (ts/create! :model/Table :name "CONTACTS" :db_id (:id db))
+ email (ts/create! :model/Field :name "CONTACT_EMAIL" :table_id (:id table) :data_sensitivity :PII)
+ _ (ts/create! :model/Field :name "CONTACT_ROW" :table_id (:id table))
+ _ (ts/create! :model/FieldUserSettings :field_id (:id email) :data_sensitivity :PII)]
+ (reset! serialized (into [] (serdes.extract/extract {})))
+ (testing "the labeled Field and its FieldUserSettings both export the label"
+ (is (= :PII (:data_sensitivity (field-ser @serialized "CONTACT_EMAIL"))))
+ (is (= :PII (:data_sensitivity (u/seek #(and (in-db? %)
+ (-> % :serdes/meta last :model (= "FieldUserSettings")))
+ @serialized)))))
+ (testing "the unlabeled Field exports no data_sensitivity key"
+ (is (not (contains? (field-ser @serialized "CONTACT_ROW") :data_sensitivity))))))
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (let [db-id (t2/select-one-fn :id :model/Database :name "labeled-db")
+ table-id (t2/select-one-fn :id :model/Table :name "CONTACTS" :db_id db-id)
+ email-id (t2/select-one-fn :id :model/Field :name "CONTACT_EMAIL" :table_id table-id)
+ row-id (t2/select-one-fn :id :model/Field :name "CONTACT_ROW" :table_id table-id)]
+ (testing "the label imports back to the keyword on the Field"
+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id email-id))))
+ (testing "the label imports back to the keyword on the FieldUserSettings mirror"
+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id email-id))))
+ (testing "the unlabeled field stays nil with no mirror row"
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id row-id)))
+ (is (not (t2/exists? :model/FieldUserSettings :field_id row-id))))))))))
+
(deftest blank-eid-creates-new-entity-test
(mt/with-empty-h2-app-db!
(let [coll (ts/create! :model/Collection :name "mycoll")
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index 5c6c72db672..5020d1ecd77 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -145,6 +145,25 @@
(update :visibility_type keyword)
(update :base_type keyword))))))))))
+(deftest field-data-sensitivity-yaml-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/Database db {:name "My Company Data"}
+ :model/Table table {:name "Customers" :db_id (:id db)}
+ :model/Field email {:name "Email"
+ :table_id (:id table)
+ :data_sensitivity :PII}
+ :model/Field _ {:name "Id" :table_id (:id table)}
+ :model/FieldUserSettings _ {:field_id (:id email) :data_sensitivity :PII}]
+ (storage/store! (into [] (extract/extract {})) (storage.files/file-writer dump-dir))
+ (let [fields-dir (io/file dump-dir "databases" "my_company_data" "tables" "customers" "fields")
+ read-yaml (fn [file-name] (yaml/from-file (io/file fields-dir file-name)))]
+ (testing "the label is written as the bare enum string on both files"
+ (is (= "PII" (:data_sensitivity (read-yaml "email.yaml"))))
+ (is (= "PII" (:data_sensitivity (read-yaml "email___fieldusersettings.yaml")))))
+ (testing "an unlabeled field's file has no data_sensitivity line"
+ (is (not (contains? (read-yaml "id.yaml") :data_sensitivity)))))))))
+
(deftest entity-counts-report-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-empty-h2-app-db!
diff --git a/resources/migrations/064/20260901_field_data_sensitivity.yaml b/resources/migrations/064/20260901_field_data_sensitivity.yaml
new file mode 100644
index 00000000000..a6e792c6f63
--- /dev/null
+++ b/resources/migrations/064/20260901_field_data_sensitivity.yaml
@@ -0,0 +1,32 @@
+## Add a nullable data_sensitivity classification label to metabase_field and metabase_field_user_settings
+
+databaseChangeLog:
+ - changeSet:
+ id: v64.2026-09-01T00:00:00
+ author: tplude
+ comment: Add nullable data_sensitivity classification column to metabase_field
+ changes:
+ - addColumn:
+ tableName: metabase_field
+ columns:
+ - column:
+ name: data_sensitivity
+ type: varchar(32)
+ remarks: "Data sensitivity classification of the field. One of PII, SEC_KEY, PHI, PCI_FIN, BIO_GEN, SENS_PERS, CORP_IP, SYS_TELEMETRY, BIZ_CONF, PUBLIC."
+ constraints:
+ nullable: true
+
+ - changeSet:
+ id: v64.2026-09-01T00:00:01
+ author: tplude
+ comment: Add nullable data_sensitivity column to metabase_field_user_settings
+ changes:
+ - addColumn:
+ tableName: metabase_field_user_settings
+ columns:
+ - column:
+ name: data_sensitivity
+ type: varchar(32)
+ remarks: "User-set data_sensitivity for the Field. A non-NULL value here overrides any value an automated writer puts in metabase_field.data_sensitivity."
+ constraints:
+ nullable: true
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index e51591a1020..dfc8d4f8f25 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -308,6 +308,24 @@
(mr/def ::column.visibility-type
(into [:enum {:decode/normalize keyword}] column-visibility-types))
+(def column-data-sensitivity-types
+ "Possible values for column `:data-sensitivity`, most severe first. The order is the precedence an automated
+ classifier uses when several categories match a single column. `nil` means the column has never been classified;
+ `:PUBLIC` means it was reviewed and found not sensitive."
+ [:SEC_KEY ; Security Credentials & Secrets
+ :SYS_TELEMETRY ; Infrastructure & System Secrets
+ :PHI ; Protected Health Information
+ :BIO_GEN ; Biometric & Genetic Data
+ :PCI_FIN ; Financial & Payment Card Data
+ :SENS_PERS ; Special Category Personal Traits
+ :PII ; Personally Identifiable Information
+ :CORP_IP ; Intellectual Property & Code
+ :BIZ_CONF ; Confidential Business Data
+ :PUBLIC]) ; Non-Sensitive / Public Information
+
+(mr/def ::column.data-sensitivity
+ (into [:enum {:decode/normalize keyword}] column-data-sensitivity-types))
+
(mr/def ::column.legacy-source
"Possible values for `column.source` -- this is added by [[metabase.lib.metadata.result-metadata]] for historical
reasons (it is used in a few places in the FE). DO NOT use this in the backend for any purpose, use `:lib/source`
@@ -353,6 +371,7 @@
[:database-type {:optional true} [:maybe :string]]
[:active {:optional true} :boolean]
[:visibility-type {:optional true} [:maybe ::column.visibility-type]]
+ [:data-sensitivity {:optional true} [:maybe ::column.data-sensitivity]]
;; if this is a field from another table (implicit join), this is the field in the current table that should be
;; used to perform the implicit join. e.g. if current table is `VENUES` and this field is `CATEGORIES.ID`, then the
;; `fk_field_id` would be `VENUES.CATEGORY_ID`. In a `:field` reference this is saved in the options map as
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index 0a6ece26e98..e768843d30c 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -153,6 +153,7 @@
{:select [:field/active
:field/base_type
:field/coercion_strategy
+ :field/data_sensitivity
:field/database_partitioned
:field/database_type
:field/description
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 38f4d2a0f11..a811cbb31ca 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -32,6 +32,11 @@
#_{:clj-kondo/ignore [:missing-docstring]} ; false positive
(p/import-def metabase.lib.schema.metadata/column-visibility-types visibility-types)
+(def data-sensitivity-types
+ "Possible values for a Field's `:data_sensitivity`, most severe first. See
+ [[metabase.lib.schema.metadata/column-data-sensitivity-types]]."
+ metabase.lib.schema.metadata/column-data-sensitivity-types)
+
(methodical/defmethod t2/table-name :model/Field [_model] :metabase_field)
(methodical/defmethod t2/model-for-automagic-hydration [:default :destination] [_model _k] :model/Field)
@@ -108,6 +113,7 @@
:semantic_type transform-field-semantic-type
:visibility_type mi/transform-keyword
:has_field_values mi/transform-keyword
+ :data_sensitivity mi/transform-keyword
:fingerprint transform-json-fingerprints
:settings mi/transform-json
:nfc_path mi/transform-json})
@@ -159,7 +165,7 @@
(def field-user-settings
"Set of user-settable values for a Field"
#{:semantic_type :description :display_name :visibility_type :has_field_values :effective_type :coercion_strategy :fk_target_field_id
- :caveats :points_of_interest :nfc_path :json_unfolding :settings})
+ :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity})
(defn- ensure-field-user-settings-exist-for-fk-target-field [field]
(warehouse-schema.db/insert-field-user-settings!
@@ -470,7 +476,7 @@
#{[db-path]}))
(defmethod serdes/make-spec "Field" [_model-name opts]
- {:copy [:active :base_type :caveats :coercion_strategy :custom_position :database_default :database_indexed
+ {:copy [:active :base_type :caveats :coercion_strategy :custom_position :data_sensitivity :database_default :database_indexed
:database_is_auto_increment :database_is_generated :database_is_nullable :database_is_pk
:database_partitioned :database_position :database_required :database_type
:description :display_name :effective_type :has_field_values :is_defective_duplicate
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index 14ecd4aa091..8b5f6fa2582 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -18,6 +18,7 @@
:semantic_type field/transform-field-semantic-type
:visibility_type mi/transform-keyword
:has_field_values mi/transform-keyword
+ :data_sensitivity mi/transform-keyword
:settings mi/transform-json
:nfc_path mi/transform-json})
@@ -60,7 +61,7 @@
(defmethod serdes/make-spec "FieldUserSettings" [_model-name _opts]
{:copy [:semantic_type :description :display_name :visibility_type
:has_field_values :effective_type :coercion_strategy :caveats
- :points_of_interest :nfc_path :json_unfolding :settings]
+ :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity]
:transform {:created_at (serdes/date)
:fk_target_field_id (serdes/fk :model/Field)
:field_id {::serdes/fk true
diff --git a/src/metabase/warehouse_schema_rest/api/field.clj b/src/metabase/warehouse_schema_rest/api/field.clj
index 4ade23ac14a..8868c9e17f2 100644
--- a/src/metabase/warehouse_schema_rest/api/field.clj
+++ b/src/metabase/warehouse_schema_rest/api/field.clj
@@ -36,6 +36,10 @@
"Schema for a valid `Field` visibility type."
(into [:enum] (map name field/visibility-types)))
+(def ^:private FieldDataSensitivity
+ "Schema for a valid `Field` data sensitivity classification."
+ (into [:enum] (map name field/data-sensitivity-types)))
+
(def ^:private max-field-ids-for-table-id-lookup
1000)
@@ -152,6 +156,7 @@
[:semantic_type {:optional true} [:maybe ms/FieldSemanticOrRelationTypeKeywordOrString]]
[:coercion_strategy {:optional true} [:maybe ms/CoercionStrategyKeywordOrString]]
[:visibility_type {:optional true} [:maybe FieldVisibilityType]]
+ [:data_sensitivity {:optional true} [:maybe FieldDataSensitivity]]
[:has_field_values {:optional true} [:maybe ::lib.schema.metadata/column.has-field-values]]
[:settings {:optional true} [:maybe ms/Map]]
[:nfc_path {:optional true} [:maybe [:sequential ms/NonBlankString]]]
@@ -198,7 +203,8 @@
id
(u/select-keys-when body
{:present #{:caveats :description :fk_target_field_id :points_of_interest :semantic_type
- :coercion_strategy :effective_type :has_field_values :nfc_path :json_unfolding}
+ :coercion_strategy :effective_type :has_field_values :nfc_path :json_unfolding
+ :data_sensitivity}
:non-nil #{:display_name :visibility_type :settings}})))))
(when (some? json-unfolding)
(update-nested-fields-on-json-unfolding-change! field json-unfolding))
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index 6e83d794280..613d973db5d 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -3313,3 +3313,36 @@
"conversations without a blob are untouched")
(is (thrown? Exception (t2/query "SELECT state FROM metabot_conversation"))
"metabot_conversation.state is gone")))))
+
+(deftest add-field-data-sensitivity-test
+ (testing "v64.2026-09-01: data_sensitivity is added to metabase_field and metabase_field_user_settings as nullable columns"
+ (impl/test-migrations ["v64.2026-09-01T00:00:00" "v64.2026-09-01T00:00:01"] [migrate!]
+ (let [db-id (t2/insert-returning-pk! :metabase_database {:name "Sensitivity Test DB"
+ :engine "h2"
+ :created_at :%now
+ :updated_at :%now
+ :details "{}"})
+ table-id (t2/insert-returning-pk! :metabase_table {:active true
+ :db_id db-id
+ :name "a table"
+ :created_at :%now
+ :updated_at :%now})
+ field-id (t2/insert-returning-pk! :metabase_field {:table_id table-id
+ :name "email"
+ :active true
+ :base_type "type/Text"
+ :database_type "TEXT"
+ :created_at :%now
+ :updated_at :%now})]
+ (migrate!)
+ (testing "an existing field reads NULL"
+ (is (nil? (t2/select-one-fn :data_sensitivity :metabase_field :id field-id))))
+ (testing "a value writes and reads back on metabase_field"
+ (t2/update! :metabase_field field-id {:data_sensitivity "PII"})
+ (is (= "PII" (t2/select-one-fn :data_sensitivity :metabase_field :id field-id))))
+ (testing "a value writes and reads back on the user-settings mirror"
+ (t2/insert! :metabase_field_user_settings {:field_id field-id :data_sensitivity "SYS_TELEMETRY"})
+ (is (= "SYS_TELEMETRY" (t2/select-one-fn :data_sensitivity :metabase_field_user_settings :field_id field-id))))
+ (testing "the mirror column is nullable"
+ (t2/update! :metabase_field_user_settings :field_id field-id {:data_sensitivity nil})
+ (is (nil? (t2/select-one-fn :data_sensitivity :metabase_field_user_settings :field_id field-id))))))))
diff --git a/test/metabase/lib/schema/metadata_test.cljc b/test/metabase/lib/schema/metadata_test.cljc
index 60df6620a1c..9e13be9892d 100644
--- a/test/metabase/lib/schema/metadata_test.cljc
+++ b/test/metabase/lib/schema/metadata_test.cljc
@@ -170,3 +170,21 @@
:metabase.lib.field/original-effective-type :type/Text
:metabase.lib.field/simple-display-name "Category: Name"
:metabase.lib.query/transformation-added-base-type true)))))
+
+(deftest ^:parallel column-data-sensitivity-test
+ (testing "every member of the severity-ordered vector validates and the vector has no duplicates"
+ (is (= 10 (count lib.schema.metadata/column-data-sensitivity-types)))
+ (is (apply distinct? lib.schema.metadata/column-data-sensitivity-types))
+ (doseq [member lib.schema.metadata/column-data-sensitivity-types]
+ (is (mr/validate ::lib.schema.metadata/column.data-sensitivity member))))
+ (testing "the most severe member comes first and PUBLIC comes last"
+ (is (= :SEC_KEY (first lib.schema.metadata/column-data-sensitivity-types)))
+ (is (= :PUBLIC (last lib.schema.metadata/column-data-sensitivity-types))))
+ (testing "normalization keywordizes a string member"
+ (is (= :PII (lib/normalize ::lib.schema.metadata/column.data-sensitivity "PII"))))
+ (testing "values outside the enum are rejected"
+ (are [value] (not (mr/validate ::lib.schema.metadata/column.data-sensitivity value))
+ :pii
+ :NOT_A_MEMBER
+ "PII"
+ nil)))
diff --git a/test/metabase/lib_be/metadata/jvm_test.clj b/test/metabase/lib_be/metadata/jvm_test.clj
index 2d43c26311e..50621e216ea 100644
--- a/test/metabase/lib_be/metadata/jvm_test.clj
+++ b/test/metabase/lib_be/metadata/jvm_test.clj
@@ -270,6 +270,22 @@
(is (=? {:database-partitioned true}
(lib.metadata/field mp (:id buyer-id)))))))
+(deftest ^:parallel data-sensitivity-metadata-test
+ (mt/with-temp [:model/Database db {:engine :h2}
+ :model/Table table {:name "CONTACTS" :db_id (:id db)}
+ :model/Field labeled {:name "EMAIL" :table_id (:id table) :base_type :type/Text :data_sensitivity :PII}
+ :model/Field plain {:name "ID" :table_id (:id table) :base_type :type/Integer}]
+ (let [mp (lib.metadata.jvm/application-database-metadata-provider (:id db))]
+ (testing "a labeled field carries the label as a keyword under the kebab-case key"
+ (let [col (lib.metadata/field mp (:id labeled))]
+ (is (=? {:data-sensitivity :PII} col))
+ (is (not (me/humanize (mr/explain ::lib.schema.metadata/column col))))))
+ (testing "an unlabeled field carries nil"
+ (let [col (lib.metadata/field mp (:id plain))]
+ (is (contains? col :data-sensitivity))
+ (is (nil? (:data-sensitivity col)))
+ (is (not (me/humanize (mr/explain ::lib.schema.metadata/column col)))))))))
+
(deftest ^:parallel instance->metadata-normalize-column-test
(testing "instance->metadata should normalize column metadata"
(let [legacy-col {:active true
diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.clj
index f1895b86504..58abb3132a7 100644
--- a/test/metabase/query_processor/middleware/results_metadata_test.clj
+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj
@@ -116,6 +116,25 @@
(is (= (:updated_at card)
(t2/select-one-fn :updated_at :model/Card :id (u/the-id card))))))))
+(deftest data-sensitivity-in-result-metadata-test
+ (testing "a Field's data_sensitivity label reaches query results through the Lib column metadata"
+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :name) {:data_sensitivity :PII}
+ (let [query (mt/mbql-query venues {:fields [$id $name]})]
+ (mt/with-temp [:model/Card card {:dataset_query query}]
+ (let [result (qp/process-query
+ (qp/userland-query
+ query
+ {:card-id (u/the-id card)
+ :query-hash (qp.util/query-hash {})}))]
+ (when-not (= :completed (:status result))
+ (throw (ex-info "Query failed." result)))
+ (testing "the label reaches the live query response through the Lib column metadata"
+ (is (= [nil :PII]
+ (map :data_sensitivity (get-in result [:data :cols])))))
+ (testing "the persisted Card result_metadata stores the label as its string form, and only on the labeled column"
+ (is (= [nil "PII"]
+ (map :data_sensitivity (card-metadata card)))))))))))
+
(defn- test-card-1 []
(let [eid (u/generate-nano-id)]
{:dataset_query (mt/native-query {:query "SELECT * FROM VENUES"})
diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
index 766e56a61f2..f552780045c 100644
--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
@@ -486,3 +486,47 @@
:coercion-strategy nil}))]
(is (some #(contains? (last %) :fingerprint_version) updates)
"base_type-change reset still includes fingerprint reset")))))
+
+(deftest base-type-change-leaves-data-sensitivity-test
+ (testing "the override write on a base type change does not include data_sensitivity"
+ (mt/with-temp [:model/Field {field-id :id} {:data_sensitivity :PII}]
+ (let [updates (updates-that-will-be-performed!
+ (merge default-metadata
+ {:id field-id
+ :base-type :type/Text
+ :effective-type :type/Text})
+ (merge default-metadata
+ {:id field-id
+ :base-type :type/Integer
+ :effective-type :type/Integer
+ :data-sensitivity :PII}))]
+ (is (= #{"FieldUserSettings" "Field"} (into #{} (map first) updates)))
+ (is (not-any? (fn [[_ _ changes]] (contains? changes :data_sensitivity)) updates)))))
+ (testing "a user-set label survives a column being dropped and re-created with a different type"
+ (mt/with-temp-test-data [["table"
+ [{:field-name "field"
+ :base-type :type/Text}]
+ (try
+ (sync/sync-table! (t2/select-one :model/Table (mt/id :table)))
+ (let [field-id (mt/id :table :field)]
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PII"})
+ (sql-jdbc.execute/do-with-connection-with-options
+ :h2
+ (mt/db)
+ {}
+ (fn [conn]
+ (doseq [sql ["ALTER TABLE \"TABLE\" DROP COLUMN \"FIELD\";"
+ "ALTER TABLE \"TABLE\" ADD COLUMN \"FIELD\" INTEGER;"
+ "INSERT INTO \"TABLE\"(field) VALUES(1);"]]
+ (next.jdbc/execute! conn [sql]))))
+ (sync/sync-table! (t2/select-one :model/Table (mt/id :table)))
+ (is (=? {:base_type :type/Integer
+ :effective_type :type/Integer
+ :data_sensitivity :PII}
+ (t2/select-one :model/Field :id field-id)))
+ (is (=? {:effective_type :type/Integer
+ :data_sensitivity :PII}
+ (t2/select-one :model/FieldUserSettings :field_id field-id))))
+ (finally
+ (t2/delete! :model/Database (mt/id)))))))
diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.clj
index 86e71eecb2c..3251391bbe2 100644
--- a/test/metabase/sync/sync_metadata/fields_test.clj
+++ b/test/metabase/sync/sync_metadata/fields_test.clj
@@ -591,3 +591,48 @@
{a "A", b "B"} (u/index-by :name (t2/select :model/Field :table_id (:id table)))]
(is (true? (:database_is_nullable a)))
(is (false? (:database_is_nullable b)))))))
+
+(deftest data-sensitivity-survives-sync-test
+ (testing "a user-set data_sensitivity is untouched by a full sync and by bare Field updates"
+ (mt/with-temp-test-data [["sens_table"
+ [{:field-name "email", :base-type :type/Text}]
+ (let [db (mt/db)
+ field-id (mt/id :sens_table :email)
+ field #(t2/select-one-fn :data_sensitivity :model/Field :id field-id)
+ mirror #(t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)]
+ (testing "a freshly synced field is unclassified and has no user-settings row"
+ (is (nil? (field)))
+ (is (not (t2/exists? :model/FieldUserSettings :field_id field-id))))
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PII"})
+ (testing "after a full sync"
+ (sync/sync-database! db)
+ (is (= :PII (field)))
+ (is (= :PII (mirror))))
+ (testing "after a bare update to the Field with a different label"
+ (t2/update! :model/Field field-id {:data_sensitivity :SEC_KEY})
+ (is (= :PII (field)))
+ (is (= :PII (mirror))))))))
+
+(deftest data-sensitivity-survives-column-drop-and-readd-test
+ (testing "a user-set data_sensitivity survives the warehouse column being dropped and added back"
+ (mt/with-temp-test-data [["readd_table"
+ [{:field-name "email", :base-type :type/Text}]
+ (try
+ (let [db (mt/db)
+ db-spec (sql-jdbc.conn/db->pooled-connection-spec db)
+ field-id (mt/id :readd_table :email)
+ field #(t2/select-one [:model/Field :active :data_sensitivity] :id field-id)]
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PHI"})
+ (jdbc/execute! db-spec ["ALTER TABLE \"READD_TABLE\" DROP COLUMN \"EMAIL\";"])
+ (sync/sync-database! db)
+ (testing "the field is inactive while the column is gone"
+ (is (=? {:active false, :data_sensitivity :PHI} (field))))
+ (jdbc/execute! db-spec ["ALTER TABLE \"READD_TABLE\" ADD COLUMN \"EMAIL\" VARCHAR;"])
+ (sync/sync-database! db)
+ (testing "the same Field row is reactivated and keeps its label"
+ (is (=? {:active true, :data_sensitivity :PHI} (field)))
+ (is (= :PHI (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))
+ (finally
+ (t2/delete! :model/Database (mt/id)))))))
diff --git a/test/metabase/warehouse_schema/models/field_test.clj b/test/metabase/warehouse_schema/models/field_test.clj
index fae02d25f76..a72faad0c93 100644
--- a/test/metabase/warehouse_schema/models/field_test.clj
+++ b/test/metabase/warehouse_schema/models/field_test.clj
@@ -309,3 +309,38 @@
:model/Table {table-id :id} {:db_id db-id}]
(let [field-id (op table-id)]
(assert-coercion-effective-type-invariant! field-id label)))))))
+
+(deftest data-sensitivity-transform-round-trip-test
+ (testing "data_sensitivity is stored as the enum string and read back as a keyword"
+ (mt/with-temp [:model/Field {field-id :id} {:data_sensitivity :PII}]
+ (is (= "PII" (t2/select-one-fn :data_sensitivity :metabase_field :id field-id)))
+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))
+ (testing "a field with no label reads nil and has no user-settings row"
+ (mt/with-temp [:model/Field {field-id :id}]
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (not (t2/exists? :model/FieldUserSettings :field_id field-id))))))
+
+(deftest data-sensitivity-user-setting-overlay-test
+ (testing "a user-set label wins over a bare update to the Field"
+ (mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]
+ (field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})
+ (t2/update! :model/Field field-id {:data_sensitivity :PII})
+ (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))
+ (testing "a user-set label also overlays an update that does not mention data_sensitivity"
+ (mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]
+ (field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})
+ (t2/update! :model/Field field-id {:description "touched"})
+ (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))
+ (testing "a nil user setting does not block a bare update to the Field"
+ (mt/with-temp [:model/Field {field-id :id :as field}]
+ (field-user-settings/upsert-user-settings field {:data_sensitivity nil})
+ (is (t2/exists? :model/FieldUserSettings :field_id field-id))
+ (t2/update! :model/Field field-id {:data_sensitivity :PII})
+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))
+ (testing "upsert-user-settings with a nil value clears a previously set label on the mirror"
+ (mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]
+ (field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})
+ (field-user-settings/upsert-user-settings field {:data_sensitivity nil})
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))))
diff --git a/test/metabase/warehouse_schema_rest/api/field_test.clj b/test/metabase/warehouse_schema_rest/api/field_test.clj
index da0b237235e..5620c2ecace 100644
--- a/test/metabase/warehouse_schema_rest/api/field_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/field_test.clj
@@ -117,7 +117,8 @@
:semantic_type
:json_unfolding
:fk_target_field_id
- :nfc_path]))
+ :nfc_path
+ :data_sensitivity]))
(mt/defdataset integer-coerceable
[["t" [{:field-name "f"
@@ -137,7 +138,8 @@
:visibility_type :normal
:json_unfolding false
:fk_target_field_id nil
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
original-val)))
(let [;; set it
response (mt/user-http-request :crowberto :put 200
@@ -148,7 +150,8 @@
:semantic_type :type/Name
:json_unfolding true
:visibility_type :sensitive
- :nfc_path ["bob" "dobbs"]})
+ :nfc_path ["bob" "dobbs"]
+ :data_sensitivity "PII"})
updated-val (simple-field-details (t2/select-one :model/Field :id field-id))]
(testing "response body should be the updated field"
(is (= {:name "Field Test"
@@ -158,7 +161,8 @@
:visibility_type "sensitive"
:json_unfolding true
:fk_target_field_id nil
- :nfc_path ["bob" "dobbs"]}
+ :nfc_path ["bob" "dobbs"]
+ :data_sensitivity "PII"}
(simple-field-details response))))
(testing "updated value"
(is (= {:name "Field Test"
@@ -168,12 +172,14 @@
:visibility_type :sensitive
:json_unfolding true
:fk_target_field_id nil
- :nfc_path ["bob" "dobbs"]}
+ :nfc_path ["bob" "dobbs"]
+ :data_sensitivity :PII}
updated-val)))
;; unset it
- (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description nil
- :semantic_type nil
- :nfc_path nil})
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description nil
+ :semantic_type nil
+ :nfc_path nil
+ :data_sensitivity nil})
(testing "response"
(is (= {:name "Field Test"
:display_name "yay"
@@ -182,9 +188,39 @@
:visibility_type :sensitive
:json_unfolding true
:fk_target_field_id nil
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(simple-field-details (t2/select-one :model/Field :id field-id)))))))))))
+(deftest update-field-data-sensitivity-test
+ (testing "PUT /api/field/:id"
+ (mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]
+ (testing "setting data_sensitivity writes the same value to the Field and to the user-settings mirror"
+ (is (= "SEC_KEY"
+ (:data_sensitivity (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)
+ {:data_sensitivity "SEC_KEY"}))))
+ (is (= :SEC_KEY (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (= :SEC_KEY (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))
+ (testing "a null data_sensitivity clears both tables"
+ (is (nil? (:data_sensitivity (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)
+ {:data_sensitivity nil}))))
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))
+ (testing "an update that omits data_sensitivity leaves it unchanged"
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PHI"})
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description "unrelated"})
+ (is (= :PHI (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))))
+
+(deftest update-field-data-sensitivity-validation-test
+ (testing "PUT /api/field/:id rejects data_sensitivity values outside the enum"
+ (mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]
+ (doseq [bad-value ["pii" "NOT_A_MEMBER" 123 ["PII"]]]
+ (testing (pr-str bad-value)
+ (is (=? {:errors {:data_sensitivity some?}}
+ (mt/user-http-request :crowberto :put 400 (format "field/%d" field-id)
+ {:data_sensitivity bad-value})))
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id))))))))
+
(deftest update-field-test-2
(testing "PUT /api/field/:id"
(testing "updating coercion strategies"
@@ -238,7 +274,9 @@
(testing "PUT /api/field/:id"
(testing "A field can only be updated by a superuser"
(mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]
- (mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:name "Field Test 2"})))))
+ (mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:name "Field Test 2"})
+ (mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:data_sensitivity "PII"})
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))))
(deftest ^:parallel update-field-hydrated-target-test
(testing "PUT /api/field/:id"
@@ -640,7 +678,8 @@
:semantic_type :type/FK
:fk_target_field_id true
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type nil})
(testing "after change"
@@ -651,7 +690,8 @@
:semantic_type nil
:fk_target_field_id false
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2)))))))))
(deftest update-fk-target-field-id-test-2
@@ -670,7 +710,8 @@
:semantic_type :type/FK
:fk_target_field_id true
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps before-change))))
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-3) {:fk_target_field_id field-id-2})
(testing "after change"
@@ -682,7 +723,8 @@
:semantic_type :type/FK
:fk_target_field_id true
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps after-change)))
(is (not= (:fk_target_field_id before-change)
(:fk_target_field_id after-change)))))))))
@@ -699,7 +741,8 @@
:semantic_type nil
:fk_target_field_id false
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type :type/FK
:fk_target_field_id field-id-1})
@@ -711,7 +754,8 @@
:semantic_type :type/FK
:fk_target_field_id true
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2)))))))))
(deftest fk-target-field-id-shouldnt-change-test
@@ -729,7 +773,8 @@
:semantic_type :type/FK
:fk_target_field_id true
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:description "foo"})
(testing "after change"
@@ -740,7 +785,8 @@
:semantic_type :type/FK
:fk_target_field_id true
:json_unfolding false
- :nfc_path nil}
+ :nfc_path nil
+ :data_sensitivity nil}
(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))))))
(deftest update-field-type-dimension-test
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index a1b513b8ed7..c82e2c40554 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -393,6 +393,18 @@
(mt/user-http-request :rasta :get 200 (format "table/%d/query_metadata" (mt/id :users))))
"Make sure that getting the User table does *not* include password info")))))
+(deftest query-metadata-data-sensitivity-test
+ (testing "GET api/table/:id/query_metadata returns each field's data_sensitivity, nil when unclassified"
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db)}
+ :model/Field _ {:table_id (:id table) :name "email" :data_sensitivity :PII}
+ :model/Field _ {:table_id (:id table) :name "id"}]
+ (is (= {"email" "PII"
+ "id" nil}
+ (->> (mt/user-http-request :crowberto :get 200 (format "table/%d/query_metadata" (:id table)))
+ :fields
+ (into {} (map (juxt :name :data_sensitivity)))))))))
+
(deftest query-metadata-transform-hydration-test
(testing "GET /api/table/:id/query_metadata hydrates the source :transform based on transforms availability (GDGT-2523)"
(mt/with-temp [:model/Transform transform {:name "My transform"