The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GHY-4378] Add data_sensitivity col to field (#81744)
Tyler Pludee315710292e6
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex e3b45702827..3a98d822ba0 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -1134,24 +1134,34 @@(mt/with-empty-h2-app-db!(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}- :model/Field {field-id :id} {:name "Some Field" :table_id no-schema-id}+ :model/Field {field-id :id} {:name "Some Field"+ :table_id no-schema-id+ :data_sensitivity :PII}+ :model/Field {plain-id :id} {:name "Plain Field" :table_id no-schema-id}:model/FieldUserSettings {description :description}- {:field_id field-id- :description "Some custom Description"}]+ {:field_id field-id+ :description "Some custom Description"+ :data_sensitivity :PII}](testing "field values"(let [ser (serdes/extract-one "FieldUserSettings" {} (t2/select-one :model/FieldUserSettings :field_id field-id))]- (is (=? {:serdes/meta [{:model "Database" :id "My Database"}- {:model "Table" :id "Schemaless Table"}- {:model "Field" :id "Some Field"}- {:model "FieldUserSettings" :id "1"}] ; Always 1.- :created_at string?- :description description}+ (is (=? {:serdes/meta [{:model "Database" :id "My Database"}+ {:model "Table" :id "Schemaless Table"}+ {:model "Field" :id "Some Field"}+ {:model "FieldUserSettings" :id "1"}] ; Always 1.+ :created_at string?+ :description description+ :data_sensitivity :PII}ser))(is (not (contains? ser :field_id))":field_id is dropped; its implied by the path")(testing "depend only on the Database; the parent Field is synthesized on import if missing"(is (= #{[{:model "Database" :id "My Database"}]}(set (serdes/deserialization-dependencies ser)))))))+ (testing "data_sensitivity on the Field itself"+ (is (= :PII (:data_sensitivity (ts/extract-one "Field" field-id)))+ "a labeled field exports the keyword as-is")+ (is (not (contains? (ts/extract-one "Field" plain-id) :data_sensitivity))+ "an unlabeled field exports no key, so nil never reaches the YAML"))(testing "extract-metabase behavior"(let [models (->> {} (extract/extract) (map (comp :model last :serdes/meta)))](is (= 1@@ -1178,6 +1188,12 @@(is (= #{["FieldUserSettings" f2-id]}(set (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc))))))(t2/delete! :model/FieldUserSettings :field_id f2-id))+ (testing "with user-edits-only and a FieldUserSettings row holding only data_sensitivity: that field appears as FieldUserSettings"+ (t2/insert! :model/FieldUserSettings {:field_id f1-id :data_sensitivity :PII})+ (let [desc (serdes/descendants "Table" table-id {:user-edits-only true})]+ (is (= #{["FieldUserSettings" f1-id]}+ (set (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc))))))+ (t2/delete! :model/FieldUserSettings :field_id f1-id))(testing "with user-edits-only and all fields edited: all appear as FieldUserSettings, not Field"(t2/insert! :model/FieldUserSettings {:field_id f1-id})(t2/insert! :model/FieldUserSettings {:field_id f2-id})diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex 4c3a513d86e..6b964bb4050 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -2113,6 +2113,44 @@(is (= "desc"(t2/select-one-fn :description :model/Field (:id f3)))))))+(deftest field-data-sensitivity-round-trip-test+ (testing "data_sensitivity travels through export and import on both Field and FieldUserSettings"+ (let [serialized (atom nil)+ in-db? (fn [entity] (-> entity :serdes/meta first :id (= "labeled-db")))+ field-ser (fn [entities field-name]+ (u/seek #(and (in-db? %)+ (-> % :serdes/meta last :model (= "Field"))+ (= field-name (:name %)))+ entities))]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [db (ts/create! :model/Database :name "labeled-db")+ table (ts/create! :model/Table :name "CONTACTS" :db_id (:id db))+ email (ts/create! :model/Field :name "CONTACT_EMAIL" :table_id (:id table) :data_sensitivity :PII)+ _ (ts/create! :model/Field :name "CONTACT_ROW" :table_id (:id table))+ _ (ts/create! :model/FieldUserSettings :field_id (:id email) :data_sensitivity :PII)]+ (reset! serialized (into [] (serdes.extract/extract {})))+ (testing "the labeled Field and its FieldUserSettings both export the label"+ (is (= :PII (:data_sensitivity (field-ser @serialized "CONTACT_EMAIL"))))+ (is (= :PII (:data_sensitivity (u/seek #(and (in-db? %)+ (-> % :serdes/meta last :model (= "FieldUserSettings")))+ @serialized)))))+ (testing "the unlabeled Field exports no data_sensitivity key"+ (is (not (contains? (field-ser @serialized "CONTACT_ROW") :data_sensitivity))))))+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (let [db-id (t2/select-one-fn :id :model/Database :name "labeled-db")+ table-id (t2/select-one-fn :id :model/Table :name "CONTACTS" :db_id db-id)+ email-id (t2/select-one-fn :id :model/Field :name "CONTACT_EMAIL" :table_id table-id)+ row-id (t2/select-one-fn :id :model/Field :name "CONTACT_ROW" :table_id table-id)]+ (testing "the label imports back to the keyword on the Field"+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id email-id))))+ (testing "the label imports back to the keyword on the FieldUserSettings mirror"+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id email-id))))+ (testing "the unlabeled field stays nil with no mirror row"+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id row-id)))+ (is (not (t2/exists? :model/FieldUserSettings :field_id row-id))))))))))+(deftest blank-eid-creates-new-entity-test(mt/with-empty-h2-app-db!(let [coll (ts/create! :model/Collection :name "mycoll")diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.cljindex 5c6c72db672..5020d1ecd77 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj@@ -145,6 +145,25 @@(update :visibility_type keyword)(update :base_type keyword))))))))))+(deftest field-data-sensitivity-yaml-test+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [:model/Database db {:name "My Company Data"}+ :model/Table table {:name "Customers" :db_id (:id db)}+ :model/Field email {:name "Email"+ :table_id (:id table)+ :data_sensitivity :PII}+ :model/Field _ {:name "Id" :table_id (:id table)}+ :model/FieldUserSettings _ {:field_id (:id email) :data_sensitivity :PII}]+ (storage/store! (into [] (extract/extract {})) (storage.files/file-writer dump-dir))+ (let [fields-dir (io/file dump-dir "databases" "my_company_data" "tables" "customers" "fields")+ read-yaml (fn [file-name] (yaml/from-file (io/file fields-dir file-name)))]+ (testing "the label is written as the bare enum string on both files"+ (is (= "PII" (:data_sensitivity (read-yaml "email.yaml"))))+ (is (= "PII" (:data_sensitivity (read-yaml "email___fieldusersettings.yaml")))))+ (testing "an unlabeled field's file has no data_sensitivity line"+ (is (not (contains? (read-yaml "id.yaml") :data_sensitivity)))))))))+(deftest entity-counts-report-test(ts/with-random-dump-dir [dump-dir "serdesv2-"](mt/with-empty-h2-app-db!diff --git a/resources/migrations/064/20260901_field_data_sensitivity.yaml b/resources/migrations/064/20260901_field_data_sensitivity.yamlnew file mode 100644index 00000000000..a6e792c6f63--- /dev/null+++ b/resources/migrations/064/20260901_field_data_sensitivity.yaml@@ -0,0 +1,32 @@+## Add a nullable data_sensitivity classification label to metabase_field and metabase_field_user_settings++databaseChangeLog:+ - changeSet:+ id: v64.2026-09-01T00:00:00+ author: tplude+ comment: Add nullable data_sensitivity classification column to metabase_field+ changes:+ - addColumn:+ tableName: metabase_field+ columns:+ - column:+ name: data_sensitivity+ type: varchar(32)+ remarks: "Data sensitivity classification of the field. One of PII, SEC_KEY, PHI, PCI_FIN, BIO_GEN, SENS_PERS, CORP_IP, SYS_TELEMETRY, BIZ_CONF, PUBLIC."+ constraints:+ nullable: true++ - changeSet:+ id: v64.2026-09-01T00:00:01+ author: tplude+ comment: Add nullable data_sensitivity column to metabase_field_user_settings+ changes:+ - addColumn:+ tableName: metabase_field_user_settings+ columns:+ - column:+ name: data_sensitivity+ type: varchar(32)+ remarks: "User-set data_sensitivity for the Field. A non-NULL value here overrides any value an automated writer puts in metabase_field.data_sensitivity."+ constraints:+ nullable: truediff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex e51591a1020..dfc8d4f8f25 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -308,6 +308,24 @@(mr/def ::column.visibility-type(into [:enum {:decode/normalize keyword}] column-visibility-types))+(def column-data-sensitivity-types+ "Possible values for column `:data-sensitivity`, most severe first. The order is the precedence an automated+ classifier uses when several categories match a single column. `nil` means the column has never been classified;+ `:PUBLIC` means it was reviewed and found not sensitive."+ [:SEC_KEY ; Security Credentials & Secrets+ :SYS_TELEMETRY ; Infrastructure & System Secrets+ :PHI ; Protected Health Information+ :BIO_GEN ; Biometric & Genetic Data+ :PCI_FIN ; Financial & Payment Card Data+ :SENS_PERS ; Special Category Personal Traits+ :PII ; Personally Identifiable Information+ :CORP_IP ; Intellectual Property & Code+ :BIZ_CONF ; Confidential Business Data+ :PUBLIC]) ; Non-Sensitive / Public Information++(mr/def ::column.data-sensitivity+ (into [:enum {:decode/normalize keyword}] column-data-sensitivity-types))+(mr/def ::column.legacy-source"Possible values for `column.source` -- this is added by [[metabase.lib.metadata.result-metadata]] for historicalreasons (it is used in a few places in the FE). DO NOT use this in the backend for any purpose, use `:lib/source`@@ -353,6 +371,7 @@[:database-type {:optional true} [:maybe :string]][:active {:optional true} :boolean][:visibility-type {:optional true} [:maybe ::column.visibility-type]]+ [:data-sensitivity {:optional true} [:maybe ::column.data-sensitivity]];; if this is a field from another table (implicit join), this is the field in the current table that should be;; used to perform the implicit join. e.g. if current table is `VENUES` and this field is `CATEGORIES.ID`, then the;; `fk_field_id` would be `VENUES.CATEGORY_ID`. In a `:field` reference this is saved in the options map asdiff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex 0a6ece26e98..e768843d30c 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -153,6 +153,7 @@{:select [:field/active:field/base_type:field/coercion_strategy+ :field/data_sensitivity:field/database_partitioned:field/database_type:field/descriptiondiff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 38f4d2a0f11..a811cbb31ca 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -32,6 +32,11 @@#_{:clj-kondo/ignore [:missing-docstring]} ; false positive(p/import-def metabase.lib.schema.metadata/column-visibility-types visibility-types)+(def data-sensitivity-types+ "Possible values for a Field's `:data_sensitivity`, most severe first. See+ [[metabase.lib.schema.metadata/column-data-sensitivity-types]]."+ metabase.lib.schema.metadata/column-data-sensitivity-types)+(methodical/defmethod t2/table-name :model/Field [_model] :metabase_field)(methodical/defmethod t2/model-for-automagic-hydration [:default :destination] [_model _k] :model/Field)@@ -108,6 +113,7 @@:semantic_type transform-field-semantic-type:visibility_type mi/transform-keyword:has_field_values mi/transform-keyword+ :data_sensitivity mi/transform-keyword:fingerprint transform-json-fingerprints:settings mi/transform-json:nfc_path mi/transform-json})@@ -159,7 +165,7 @@(def field-user-settings"Set of user-settable values for a Field"#{:semantic_type :description :display_name :visibility_type :has_field_values :effective_type :coercion_strategy :fk_target_field_id- :caveats :points_of_interest :nfc_path :json_unfolding :settings})+ :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity})(defn- ensure-field-user-settings-exist-for-fk-target-field [field](warehouse-schema.db/insert-field-user-settings!@@ -470,7 +476,7 @@#{[db-path]}))(defmethod serdes/make-spec "Field" [_model-name opts]- {:copy [:active :base_type :caveats :coercion_strategy :custom_position :database_default :database_indexed+ {:copy [:active :base_type :caveats :coercion_strategy :custom_position :data_sensitivity :database_default :database_indexed:database_is_auto_increment :database_is_generated :database_is_nullable :database_is_pk:database_partitioned :database_position :database_required :database_type:description :display_name :effective_type :has_field_values :is_defective_duplicatediff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.cljindex 14ecd4aa091..8b5f6fa2582 100644--- a/src/metabase/warehouse_schema/models/field_user_settings.clj+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj@@ -18,6 +18,7 @@:semantic_type field/transform-field-semantic-type:visibility_type mi/transform-keyword:has_field_values mi/transform-keyword+ :data_sensitivity mi/transform-keyword:settings mi/transform-json:nfc_path mi/transform-json})@@ -60,7 +61,7 @@(defmethod serdes/make-spec "FieldUserSettings" [_model-name _opts]{:copy [:semantic_type :description :display_name :visibility_type:has_field_values :effective_type :coercion_strategy :caveats- :points_of_interest :nfc_path :json_unfolding :settings]+ :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity]:transform {:created_at (serdes/date):fk_target_field_id (serdes/fk :model/Field):field_id {::serdes/fk truediff --git a/src/metabase/warehouse_schema_rest/api/field.clj b/src/metabase/warehouse_schema_rest/api/field.cljindex 4ade23ac14a..8868c9e17f2 100644--- a/src/metabase/warehouse_schema_rest/api/field.clj+++ b/src/metabase/warehouse_schema_rest/api/field.clj@@ -36,6 +36,10 @@"Schema for a valid `Field` visibility type."(into [:enum] (map name field/visibility-types)))+(def ^:private FieldDataSensitivity+ "Schema for a valid `Field` data sensitivity classification."+ (into [:enum] (map name field/data-sensitivity-types)))+(def ^:private max-field-ids-for-table-id-lookup1000)@@ -152,6 +156,7 @@[:semantic_type {:optional true} [:maybe ms/FieldSemanticOrRelationTypeKeywordOrString]][:coercion_strategy {:optional true} [:maybe ms/CoercionStrategyKeywordOrString]][:visibility_type {:optional true} [:maybe FieldVisibilityType]]+ [:data_sensitivity {:optional true} [:maybe FieldDataSensitivity]][:has_field_values {:optional true} [:maybe ::lib.schema.metadata/column.has-field-values]][:settings {:optional true} [:maybe ms/Map]][:nfc_path {:optional true} [:maybe [:sequential ms/NonBlankString]]]@@ -198,7 +203,8 @@id(u/select-keys-when body{:present #{:caveats :description :fk_target_field_id :points_of_interest :semantic_type- :coercion_strategy :effective_type :has_field_values :nfc_path :json_unfolding}+ :coercion_strategy :effective_type :has_field_values :nfc_path :json_unfolding+ :data_sensitivity}:non-nil #{:display_name :visibility_type :settings}})))))(when (some? json-unfolding)(update-nested-fields-on-json-unfolding-change! field json-unfolding))diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.cljindex 6e83d794280..613d973db5d 100644--- a/test/metabase/app_db/schema_migrations_test.clj+++ b/test/metabase/app_db/schema_migrations_test.clj@@ -3313,3 +3313,36 @@"conversations without a blob are untouched")(is (thrown? Exception (t2/query "SELECT state FROM metabot_conversation"))"metabot_conversation.state is gone")))))++(deftest add-field-data-sensitivity-test+ (testing "v64.2026-09-01: data_sensitivity is added to metabase_field and metabase_field_user_settings as nullable columns"+ (impl/test-migrations ["v64.2026-09-01T00:00:00" "v64.2026-09-01T00:00:01"] [migrate!]+ (let [db-id (t2/insert-returning-pk! :metabase_database {:name "Sensitivity Test DB"+ :engine "h2"+ :created_at :%now+ :updated_at :%now+ :details "{}"})+ table-id (t2/insert-returning-pk! :metabase_table {:active true+ :db_id db-id+ :name "a table"+ :created_at :%now+ :updated_at :%now})+ field-id (t2/insert-returning-pk! :metabase_field {:table_id table-id+ :name "email"+ :active true+ :base_type "type/Text"+ :database_type "TEXT"+ :created_at :%now+ :updated_at :%now})]+ (migrate!)+ (testing "an existing field reads NULL"+ (is (nil? (t2/select-one-fn :data_sensitivity :metabase_field :id field-id))))+ (testing "a value writes and reads back on metabase_field"+ (t2/update! :metabase_field field-id {:data_sensitivity "PII"})+ (is (= "PII" (t2/select-one-fn :data_sensitivity :metabase_field :id field-id))))+ (testing "a value writes and reads back on the user-settings mirror"+ (t2/insert! :metabase_field_user_settings {:field_id field-id :data_sensitivity "SYS_TELEMETRY"})+ (is (= "SYS_TELEMETRY" (t2/select-one-fn :data_sensitivity :metabase_field_user_settings :field_id field-id))))+ (testing "the mirror column is nullable"+ (t2/update! :metabase_field_user_settings :field_id field-id {:data_sensitivity nil})+ (is (nil? (t2/select-one-fn :data_sensitivity :metabase_field_user_settings :field_id field-id))))))))diff --git a/test/metabase/lib/schema/metadata_test.cljc b/test/metabase/lib/schema/metadata_test.cljcindex 60df6620a1c..9e13be9892d 100644--- a/test/metabase/lib/schema/metadata_test.cljc+++ b/test/metabase/lib/schema/metadata_test.cljc@@ -170,3 +170,21 @@:metabase.lib.field/original-effective-type :type/Text:metabase.lib.field/simple-display-name "Category: Name":metabase.lib.query/transformation-added-base-type true)))))++(deftest ^:parallel column-data-sensitivity-test+ (testing "every member of the severity-ordered vector validates and the vector has no duplicates"+ (is (= 10 (count lib.schema.metadata/column-data-sensitivity-types)))+ (is (apply distinct? lib.schema.metadata/column-data-sensitivity-types))+ (doseq [member lib.schema.metadata/column-data-sensitivity-types]+ (is (mr/validate ::lib.schema.metadata/column.data-sensitivity member))))+ (testing "the most severe member comes first and PUBLIC comes last"+ (is (= :SEC_KEY (first lib.schema.metadata/column-data-sensitivity-types)))+ (is (= :PUBLIC (last lib.schema.metadata/column-data-sensitivity-types))))+ (testing "normalization keywordizes a string member"+ (is (= :PII (lib/normalize ::lib.schema.metadata/column.data-sensitivity "PII"))))+ (testing "values outside the enum are rejected"+ (are [value] (not (mr/validate ::lib.schema.metadata/column.data-sensitivity value))+ :pii+ :NOT_A_MEMBER+ "PII"+ nil)))diff --git a/test/metabase/lib_be/metadata/jvm_test.clj b/test/metabase/lib_be/metadata/jvm_test.cljindex 2d43c26311e..50621e216ea 100644--- a/test/metabase/lib_be/metadata/jvm_test.clj+++ b/test/metabase/lib_be/metadata/jvm_test.clj@@ -270,6 +270,22 @@(is (=? {:database-partitioned true}(lib.metadata/field mp (:id buyer-id)))))))+(deftest ^:parallel data-sensitivity-metadata-test+ (mt/with-temp [:model/Database db {:engine :h2}+ :model/Table table {:name "CONTACTS" :db_id (:id db)}+ :model/Field labeled {:name "EMAIL" :table_id (:id table) :base_type :type/Text :data_sensitivity :PII}+ :model/Field plain {:name "ID" :table_id (:id table) :base_type :type/Integer}]+ (let [mp (lib.metadata.jvm/application-database-metadata-provider (:id db))]+ (testing "a labeled field carries the label as a keyword under the kebab-case key"+ (let [col (lib.metadata/field mp (:id labeled))]+ (is (=? {:data-sensitivity :PII} col))+ (is (not (me/humanize (mr/explain ::lib.schema.metadata/column col))))))+ (testing "an unlabeled field carries nil"+ (let [col (lib.metadata/field mp (:id plain))]+ (is (contains? col :data-sensitivity))+ (is (nil? (:data-sensitivity col)))+ (is (not (me/humanize (mr/explain ::lib.schema.metadata/column col)))))))))+(deftest ^:parallel instance->metadata-normalize-column-test(testing "instance->metadata should normalize column metadata"(let [legacy-col {:active truediff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.cljindex f1895b86504..58abb3132a7 100644--- a/test/metabase/query_processor/middleware/results_metadata_test.clj+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj@@ -116,6 +116,25 @@(is (= (:updated_at card)(t2/select-one-fn :updated_at :model/Card :id (u/the-id card))))))))+(deftest data-sensitivity-in-result-metadata-test+ (testing "a Field's data_sensitivity label reaches query results through the Lib column metadata"+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :name) {:data_sensitivity :PII}+ (let [query (mt/mbql-query venues {:fields [$id $name]})]+ (mt/with-temp [:model/Card card {:dataset_query query}]+ (let [result (qp/process-query+ (qp/userland-query+ query+ {:card-id (u/the-id card)+ :query-hash (qp.util/query-hash {})}))]+ (when-not (= :completed (:status result))+ (throw (ex-info "Query failed." result)))+ (testing "the label reaches the live query response through the Lib column metadata"+ (is (= [nil :PII]+ (map :data_sensitivity (get-in result [:data :cols])))))+ (testing "the persisted Card result_metadata stores the label as its string form, and only on the labeled column"+ (is (= [nil "PII"]+ (map :data_sensitivity (card-metadata card)))))))))))+(defn- test-card-1 [](let [eid (u/generate-nano-id)]{:dataset_query (mt/native-query {:query "SELECT * FROM VENUES"})diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.cljindex 766e56a61f2..f552780045c 100644--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj@@ -486,3 +486,47 @@:coercion-strategy nil}))](is (some #(contains? (last %) :fingerprint_version) updates)"base_type-change reset still includes fingerprint reset")))))++(deftest base-type-change-leaves-data-sensitivity-test+ (testing "the override write on a base type change does not include data_sensitivity"+ (mt/with-temp [:model/Field {field-id :id} {:data_sensitivity :PII}]+ (let [updates (updates-that-will-be-performed!+ (merge default-metadata+ {:id field-id+ :base-type :type/Text+ :effective-type :type/Text})+ (merge default-metadata+ {:id field-id+ :base-type :type/Integer+ :effective-type :type/Integer+ :data-sensitivity :PII}))]+ (is (= #{"FieldUserSettings" "Field"} (into #{} (map first) updates)))+ (is (not-any? (fn [[_ _ changes]] (contains? changes :data_sensitivity)) updates)))))+ (testing "a user-set label survives a column being dropped and re-created with a different type"+ (mt/with-temp-test-data [["table"+ [{:field-name "field"+ :base-type :type/Text}]+ [["[email protected]"]]]]+ (try+ (sync/sync-table! (t2/select-one :model/Table (mt/id :table)))+ (let [field-id (mt/id :table :field)]+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PII"})+ (sql-jdbc.execute/do-with-connection-with-options+ :h2+ (mt/db)+ {}+ (fn [conn]+ (doseq [sql ["ALTER TABLE \"TABLE\" DROP COLUMN \"FIELD\";"+ "ALTER TABLE \"TABLE\" ADD COLUMN \"FIELD\" INTEGER;"+ "INSERT INTO \"TABLE\"(field) VALUES(1);"]]+ (next.jdbc/execute! conn [sql]))))+ (sync/sync-table! (t2/select-one :model/Table (mt/id :table)))+ (is (=? {:base_type :type/Integer+ :effective_type :type/Integer+ :data_sensitivity :PII}+ (t2/select-one :model/Field :id field-id)))+ (is (=? {:effective_type :type/Integer+ :data_sensitivity :PII}+ (t2/select-one :model/FieldUserSettings :field_id field-id))))+ (finally+ (t2/delete! :model/Database (mt/id)))))))diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.cljindex 86e71eecb2c..3251391bbe2 100644--- a/test/metabase/sync/sync_metadata/fields_test.clj+++ b/test/metabase/sync/sync_metadata/fields_test.clj@@ -591,3 +591,48 @@{a "A", b "B"} (u/index-by :name (t2/select :model/Field :table_id (:id table)))](is (true? (:database_is_nullable a)))(is (false? (:database_is_nullable b)))))))++(deftest data-sensitivity-survives-sync-test+ (testing "a user-set data_sensitivity is untouched by a full sync and by bare Field updates"+ (mt/with-temp-test-data [["sens_table"+ [{:field-name "email", :base-type :type/Text}]+ [["[email protected]"]]]]+ (let [db (mt/db)+ field-id (mt/id :sens_table :email)+ field #(t2/select-one-fn :data_sensitivity :model/Field :id field-id)+ mirror #(t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)]+ (testing "a freshly synced field is unclassified and has no user-settings row"+ (is (nil? (field)))+ (is (not (t2/exists? :model/FieldUserSettings :field_id field-id))))+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PII"})+ (testing "after a full sync"+ (sync/sync-database! db)+ (is (= :PII (field)))+ (is (= :PII (mirror))))+ (testing "after a bare update to the Field with a different label"+ (t2/update! :model/Field field-id {:data_sensitivity :SEC_KEY})+ (is (= :PII (field)))+ (is (= :PII (mirror))))))))++(deftest data-sensitivity-survives-column-drop-and-readd-test+ (testing "a user-set data_sensitivity survives the warehouse column being dropped and added back"+ (mt/with-temp-test-data [["readd_table"+ [{:field-name "email", :base-type :type/Text}]+ [["[email protected]"]]]]+ (try+ (let [db (mt/db)+ db-spec (sql-jdbc.conn/db->pooled-connection-spec db)+ field-id (mt/id :readd_table :email)+ field #(t2/select-one [:model/Field :active :data_sensitivity] :id field-id)]+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PHI"})+ (jdbc/execute! db-spec ["ALTER TABLE \"READD_TABLE\" DROP COLUMN \"EMAIL\";"])+ (sync/sync-database! db)+ (testing "the field is inactive while the column is gone"+ (is (=? {:active false, :data_sensitivity :PHI} (field))))+ (jdbc/execute! db-spec ["ALTER TABLE \"READD_TABLE\" ADD COLUMN \"EMAIL\" VARCHAR;"])+ (sync/sync-database! db)+ (testing "the same Field row is reactivated and keeps its label"+ (is (=? {:active true, :data_sensitivity :PHI} (field)))+ (is (= :PHI (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))+ (finally+ (t2/delete! :model/Database (mt/id)))))))diff --git a/test/metabase/warehouse_schema/models/field_test.clj b/test/metabase/warehouse_schema/models/field_test.cljindex fae02d25f76..a72faad0c93 100644--- a/test/metabase/warehouse_schema/models/field_test.clj+++ b/test/metabase/warehouse_schema/models/field_test.clj@@ -309,3 +309,38 @@:model/Table {table-id :id} {:db_id db-id}](let [field-id (op table-id)](assert-coercion-effective-type-invariant! field-id label)))))))++(deftest data-sensitivity-transform-round-trip-test+ (testing "data_sensitivity is stored as the enum string and read back as a keyword"+ (mt/with-temp [:model/Field {field-id :id} {:data_sensitivity :PII}]+ (is (= "PII" (t2/select-one-fn :data_sensitivity :metabase_field :id field-id)))+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))+ (testing "a field with no label reads nil and has no user-settings row"+ (mt/with-temp [:model/Field {field-id :id}]+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))+ (is (not (t2/exists? :model/FieldUserSettings :field_id field-id))))))++(deftest data-sensitivity-user-setting-overlay-test+ (testing "a user-set label wins over a bare update to the Field"+ (mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]+ (field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})+ (t2/update! :model/Field field-id {:data_sensitivity :PII})+ (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))+ (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))+ (testing "a user-set label also overlays an update that does not mention data_sensitivity"+ (mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]+ (field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})+ (t2/update! :model/Field field-id {:description "touched"})+ (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))+ (testing "a nil user setting does not block a bare update to the Field"+ (mt/with-temp [:model/Field {field-id :id :as field}]+ (field-user-settings/upsert-user-settings field {:data_sensitivity nil})+ (is (t2/exists? :model/FieldUserSettings :field_id field-id))+ (t2/update! :model/Field field-id {:data_sensitivity :PII})+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))+ (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))+ (testing "upsert-user-settings with a nil value clears a previously set label on the mirror"+ (mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]+ (field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})+ (field-user-settings/upsert-user-settings field {:data_sensitivity nil})+ (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))))diff --git a/test/metabase/warehouse_schema_rest/api/field_test.clj b/test/metabase/warehouse_schema_rest/api/field_test.cljindex da0b237235e..5620c2ecace 100644--- a/test/metabase/warehouse_schema_rest/api/field_test.clj+++ b/test/metabase/warehouse_schema_rest/api/field_test.clj@@ -117,7 +117,8 @@:semantic_type:json_unfolding:fk_target_field_id- :nfc_path]))+ :nfc_path+ :data_sensitivity]))(mt/defdataset integer-coerceable[["t" [{:field-name "f"@@ -137,7 +138,8 @@:visibility_type :normal:json_unfolding false:fk_target_field_id nil- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}original-val)))(let [;; set itresponse (mt/user-http-request :crowberto :put 200@@ -148,7 +150,8 @@:semantic_type :type/Name:json_unfolding true:visibility_type :sensitive- :nfc_path ["bob" "dobbs"]})+ :nfc_path ["bob" "dobbs"]+ :data_sensitivity "PII"})updated-val (simple-field-details (t2/select-one :model/Field :id field-id))](testing "response body should be the updated field"(is (= {:name "Field Test"@@ -158,7 +161,8 @@:visibility_type "sensitive":json_unfolding true:fk_target_field_id nil- :nfc_path ["bob" "dobbs"]}+ :nfc_path ["bob" "dobbs"]+ :data_sensitivity "PII"}(simple-field-details response))))(testing "updated value"(is (= {:name "Field Test"@@ -168,12 +172,14 @@:visibility_type :sensitive:json_unfolding true:fk_target_field_id nil- :nfc_path ["bob" "dobbs"]}+ :nfc_path ["bob" "dobbs"]+ :data_sensitivity :PII}updated-val)));; unset it- (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description nil- :semantic_type nil- :nfc_path nil})+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description nil+ :semantic_type nil+ :nfc_path nil+ :data_sensitivity nil})(testing "response"(is (= {:name "Field Test":display_name "yay"@@ -182,9 +188,39 @@:visibility_type :sensitive:json_unfolding true:fk_target_field_id nil- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(simple-field-details (t2/select-one :model/Field :id field-id)))))))))))+(deftest update-field-data-sensitivity-test+ (testing "PUT /api/field/:id"+ (mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]+ (testing "setting data_sensitivity writes the same value to the Field and to the user-settings mirror"+ (is (= "SEC_KEY"+ (:data_sensitivity (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)+ {:data_sensitivity "SEC_KEY"}))))+ (is (= :SEC_KEY (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))+ (is (= :SEC_KEY (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))+ (testing "a null data_sensitivity clears both tables"+ (is (nil? (:data_sensitivity (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)+ {:data_sensitivity nil}))))+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))+ (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))+ (testing "an update that omits data_sensitivity leaves it unchanged"+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PHI"})+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description "unrelated"})+ (is (= :PHI (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))))++(deftest update-field-data-sensitivity-validation-test+ (testing "PUT /api/field/:id rejects data_sensitivity values outside the enum"+ (mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]+ (doseq [bad-value ["pii" "NOT_A_MEMBER" 123 ["PII"]]]+ (testing (pr-str bad-value)+ (is (=? {:errors {:data_sensitivity some?}}+ (mt/user-http-request :crowberto :put 400 (format "field/%d" field-id)+ {:data_sensitivity bad-value})))+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id))))))))+(deftest update-field-test-2(testing "PUT /api/field/:id"(testing "updating coercion strategies"@@ -238,7 +274,9 @@(testing "PUT /api/field/:id"(testing "A field can only be updated by a superuser"(mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]- (mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:name "Field Test 2"})))))+ (mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:name "Field Test 2"})+ (mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:data_sensitivity "PII"})+ (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))))(deftest ^:parallel update-field-hydrated-target-test(testing "PUT /api/field/:id"@@ -640,7 +678,8 @@:semantic_type :type/FK:fk_target_field_id true:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type nil})(testing "after change"@@ -651,7 +690,8 @@:semantic_type nil:fk_target_field_id false:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2)))))))))(deftest update-fk-target-field-id-test-2@@ -670,7 +710,8 @@:semantic_type :type/FK:fk_target_field_id true:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps before-change))))(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-3) {:fk_target_field_id field-id-2})(testing "after change"@@ -682,7 +723,8 @@:semantic_type :type/FK:fk_target_field_id true:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps after-change)))(is (not= (:fk_target_field_id before-change)(:fk_target_field_id after-change)))))))))@@ -699,7 +741,8 @@:semantic_type nil:fk_target_field_id false:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type :type/FK:fk_target_field_id field-id-1})@@ -711,7 +754,8 @@:semantic_type :type/FK:fk_target_field_id true:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2)))))))))(deftest fk-target-field-id-shouldnt-change-test@@ -729,7 +773,8 @@:semantic_type :type/FK:fk_target_field_id true:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:description "foo"})(testing "after change"@@ -740,7 +785,8 @@:semantic_type :type/FK:fk_target_field_id true:json_unfolding false- :nfc_path nil}+ :nfc_path nil+ :data_sensitivity nil}(mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))))))(deftest update-field-type-dimension-testdiff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.cljindex a1b513b8ed7..c82e2c40554 100644--- a/test/metabase/warehouse_schema_rest/api/table_test.clj+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj@@ -393,6 +393,18 @@(mt/user-http-request :rasta :get 200 (format "table/%d/query_metadata" (mt/id :users))))"Make sure that getting the User table does *not* include password info")))))+(deftest query-metadata-data-sensitivity-test+ (testing "GET api/table/:id/query_metadata returns each field's data_sensitivity, nil when unclassified"+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db)}+ :model/Field _ {:table_id (:id table) :name "email" :data_sensitivity :PII}+ :model/Field _ {:table_id (:id table) :name "id"}]+ (is (= {"email" "PII"+ "id" nil}+ (->> (mt/user-http-request :crowberto :get 200 (format "table/%d/query_metadata" (:id table)))+ :fields+ (into {} (map (juxt :name :data_sensitivity)))))))))+(deftest query-metadata-transform-hydration-test(testing "GET /api/table/:id/query_metadata hydrates the source :transform based on transforms availability (GDGT-2523)"(mt/with-temp [:model/Transform transform {:name "My transform"