Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GHY-4377] Data sensitivity table scanning (#81752)

Tyler Plude7d07cc9ac895
diff --git a/src/metabase/analyze/classifiers/data_sensitivity.clj b/src/metabase/analyze/classifiers/data_sensitivity.clj
new file mode 100644
index 00000000000..8259ef62dca
--- /dev/null
+++ b/src/metabase/analyze/classifiers/data_sensitivity.clj
@@ -0,0 +1,272 @@
+(ns metabase.analyze.classifiers.data-sensitivity
+ "Classifier that infers the `data_sensitivity` category of a Field from deterministic rules over its name, base
+ type, semantic type, fingerprint, and the name and entity type of its Table. Pure: no app-DB access, no settings."
+ (:require
+ [clojure.string :as str]
+ [metabase.config.core :as config]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.util :as u]
+ [metabase.util.malli :as mu]))
+
+;; `:type/IPAddress` (Postgres `inet`, ClickHouse `IPv4`/`IPv6`) and `:type/Structured` (JSON, XML, Postgres `macaddr`,
+;; `cidr`) are base types that do not descend from `:type/Text`, so the text rules that should see them list them.
+(def ^:private text-type #{:type/Text})
+(def ^:private text-or-structured-type #{:type/Text :type/Structured})
+(def ^:private telemetry-text-type #{:type/Text :type/IPAddress :type/Structured})
+(def ^:private int-or-text-type #{:type/Integer :type/Text})
+(def ^:private text-or-bool-type #{:type/Text :type/Boolean})
+(def ^:private temporal-or-text-type #{:type/Temporal :type/Text})
+(def ^:private temporal-int-or-text-type #{:type/Temporal :type/Integer :type/Text})
+(def ^:private number-type #{:type/Number})
+(def ^:private any-type #{:type/*})
+
+(defn- name->tokens
+ "Lowercased tokens of a physical column or table name, split on `_`, `-`, `.`, whitespace, and camelCase
+ boundaries, plus every adjacent pair joined with `_`."
+ [s]
+ (let [tokens (->> (str/replace s #"(?<=[a-z0-9])(?=[A-Z])" "_")
+ u/lower-case-en
+ (#(str/split % #"[_\-.\s]+"))
+ (remove str/blank?)
+ vec)]
+ {:tokens (set tokens)
+ :pairs (set (map #(str %1 "_" %2) tokens (rest tokens)))}))
+
+;; Tuples of `[#{token-or-pair ...} set-of-valid-base-types category]`. A field matches when any of its tokens or
+;; adjacent-token pairs is in the set and its base type `isa?` one of the base types. Bare `key`, `token`, `hash`,
+;; `name`, `address`, `city`, `state`, `zip`, `country`, `age` are deliberately absent: they collide with
+;; `foreign_key`, `token_count`, `product_name`, `orders.state`; the semantic-type maps and table boosters below
+;; cover them where the table supplies context.
+(def ^:private token-rules
+ [;; SEC_KEY: credentials, keys, tokens, and hashes that grant access.
+ [#{"password" "passwd" "pwd" "passphrase" "password_hash" "pwd_hash" "salt" "pin" "pin_code"
+ "secret" "secrets" "client_secret" "api_secret" "mfa_secret" "totp_secret" "webhook_secret" "signing_secret"
+ "api_key" "apikey" "access_key" "secret_key" "private_key" "ssh_key" "pgp_key" "gpg_key" "signing_key"
+ "encryption_key" "master_key" "license_key"
+ "api_token" "access_token" "auth_token" "refresh_token" "session_token" "bearer_token" "csrf_token"
+ "oauth_token" "id_token" "reset_token" "verification_token" "device_token" "bearer" "jwt"
+ "otp" "totp" "credential" "credentials" "security_answer" "recovery_code" "recovery_codes" "backup_codes"
+ "connection_string"}
+ text-or-structured-type :SEC_KEY]
+ ;; SYS_TELEMETRY: infrastructure identifiers and machine telemetry.
+ [#{"ip" "ip_address" "ip_addr" "ipaddress" "ipv4" "ipv6" "client_ip" "remote_ip" "remote_addr" "source_ip"
+ "src_ip" "dest_ip" "dst_ip" "server_ip" "host_ip" "mac_address" "mac_addr" "macaddress" "hostname" "host_name"
+ "user_agent" "useragent" "device_fingerprint" "browser_fingerprint"}
+ telemetry-text-type :SYS_TELEMETRY]
+ [#{"device_id" "imei" "imsi" "iccid" "serial_number" "session_id" "trace_id" "span_id" "request_id"
+ "correlation_id"}
+ int-or-text-type :SYS_TELEMETRY]
+ ;; PHI: health status, diagnoses, treatment, and insurance.
+ [#{"diagnosis" "diagnoses" "diagnosis_code" "icd" "icd9" "icd10" "icd_code" "cpt" "cpt_code" "medication"
+ "medications" "prescription" "prescriptions" "allergy" "allergies" "symptom" "symptoms" "immunization"
+ "vaccination" "vaccine" "blood_type" "medical_history" "medical_condition" "health_condition" "mental_health"
+ "therapy" "dosage" "lab_result" "lab_results" "treatment_plan" "clinical_notes" "chief_complaint" "hiv"
+ "hiv_status" "patient" "medical_record"}
+ text-or-structured-type :PHI]
+ [#{"patient_id" "mrn" "npi" "insurance_id" "insurance_number" "rx_number" "rx_id"}
+ int-or-text-type :PHI]
+ [#{"bmi" "blood_pressure" "heart_rate" "glucose" "cholesterol"}
+ any-type :PHI]
+ ;; BIO_GEN: biometric and genetic identifiers.
+ [#{"fingerprint" "fingerprints" "retina" "iris" "face_id" "faceprint" "voiceprint" "face_encoding"
+ "face_embedding" "facial_recognition" "dna" "genome" "genotype" "genetic" "biometric" "biometrics"}
+ any-type :BIO_GEN]
+ ;; PCI_FIN: payment card and financial account data.
+ [#{"card_number" "cardnumber" "card_num" "card_no" "cc_number" "cc_num" "ccnum" "ccn" "credit_card" "creditcard"
+ "debit_card" "debitcard" "pan" "cvv" "cvc" "cvv2" "cvc2" "card_expiry" "card_expiration" "card_last4"
+ "cardholder" "cardholder_name" "iban" "bic" "swift_code" "swift_bic" "routing_number" "routing_no"
+ "aba_number" "sort_code" "account_number" "account_no" "acct_number" "acct_no" "bank_account"}
+ int-or-text-type :PCI_FIN]
+ ;; SENS_PERS: GDPR special categories (race, ethnicity, religion, politics, union, sexual orientation,
+ ;; health-adjacent traits) plus gender and sex by policy (D14).
+ [#{"race" "ethnicity" "ethnic_origin" "religion" "religious_affiliation" "political_affiliation"
+ "political_party" "political_views" "union_member" "union_membership" "trade_union" "sexual_orientation"
+ "sexuality" "gender" "gender_identity" "sex" "disability" "disabilities" "pregnancy" "pregnant"
+ "veteran_status" "criminal_record" "criminal_history" "conviction" "convictions"}
+ text-or-bool-type :SENS_PERS]
+ ;; PII: direct personal identifiers and contact data. Government identifiers fire on any type.
+ [#{"ssn" "social_security" "national_id" "national_identifier" "nin" "nino" "passport"
+ "passport_number" "passport_no" "driver_license" "drivers_license" "driving_license" "driver_licence"
+ "drivers_licence" "dl_number" "license_plate" "licence_plate" "plate_number" "vin" "tax_id" "taxpayer_id"
+ "tin" "itin" "ein" "sin_number" "aadhaar" "aadhar"}
+ any-type :PII]
+ [#{"of_birth" "birth_date" "birthdate" "birthday" "birth_day" "birth_place" "birth_city" "birth_country"}
+ temporal-or-text-type :PII]
+ [#{"dob" "birth_year"}
+ temporal-int-or-text-type :PII]
+ [#{"first_name" "firstname" "last_name" "lastname" "full_name" "fullname" "surname" "given_name" "middle_name"
+ "maiden_name" "family_name" "forename" "username" "user_name" "customer_name" "employee_name" "contact_name"
+ "person_name" "recipient_name" "sender_name"
+ "email" "emails" "email_address" "e_mail" "personal_email" "work_email" "contact_email"
+ "home_address" "mailing_address" "street_address" "postal_address" "billing_address" "shipping_address"
+ "residential_address" "address_line" "address_line1" "address_line2" "address1" "address2" "addr1" "addr2"}
+ text-type :PII]
+ [#{"phone" "telephone" "mobile_phone" "cell_phone" "cellphone" "home_phone" "work_phone" "contact_phone" "fax"}
+ text-type :PII]
+ [#{"phone_number" "phone_no" "phone_num" "mobile_number" "mobile_no" "cell_number" "contact_number" "fax_number"}
+ int-or-text-type :PII]
+ ;; CORP_IP: source code, designs, and proprietary algorithms.
+ [#{"source_code" "sourcecode" "code_snippet" "repo_url" "repository_url" "private_repo" "patent"
+ "patent_number" "design_doc" "design_document" "algorithm" "model_weights" "proprietary"
+ "schematic" "schematics" "blueprint" "blueprints"}
+ text-type :CORP_IP]
+ ;; BIZ_CONF: internal financials and confidential business figures.
+ [#{"salary" "salaries" "compensation" "wage" "wages" "bonus" "payroll" "pay_rate" "hourly_rate" "annual_income"
+ "revenue" "profit" "margin" "gross_margin" "net_income" "operating_income" "ebitda" "cost_basis" "forecast"
+ "budget" "contract_value" "deal_size" "deal_value"}
+ number-type :BIZ_CONF]
+ [#{"salary_band" "salary_grade" "salary_range" "pay_band" "pay_grade" "pay_scale" "compensation_band"
+ "compensation_tier" "compensation_grade" "wage_band"}
+ int-or-text-type :BIZ_CONF]])
+
+;; Tuples of `[regex set-of-valid-base-types category]` matched with `re-find` against the whole lowercased name.
+;; Reserved for true substrings that tokenizing misses (`ssn4`, `passportno`, `cardcvv`); keep short.
+(def ^:private stem-rules
+ [[#"passw" text-type :SEC_KEY]
+ [#"secret(?!ar)" text-type :SEC_KEY]
+ [#"cv[vc]" int-or-text-type :PCI_FIN]
+ [#"iban" int-or-text-type :PCI_FIN]
+ [#"(?<![a-z])ssn" any-type :PII]
+ [#"passport" any-type :PII]
+ [#"birthda" temporal-or-text-type :PII]
+ [#"email" text-type :PII]])
+
+;; Semantic types that imply a category regardless of table. Consulted with `isa?` so descendants match. Also
+;; consulted against the base type when it is itself a semantic type (`:type/IPAddress` is both), since sync
+;; never sets `semantic_type` for those columns.
+(def ^:private semantic-type->category
+ {:type/Email :PII
+ :type/Birthdate :PII
+ :type/IPAddress :SYS_TELEMETRY})
+
+;; Semantic types that imply a category only when the table's `entity_type` is `:entity/UserTable`.
+(def ^:private user-table-semantic-type->category
+ {:type/Name :PII
+ :type/Address :PII
+ :type/City :PII
+ :type/State :PII
+ :type/Country :PII
+ :type/ZipCode :PII
+ :type/Coordinate :PII
+ :type/User :PII})
+
+;; Tuples of `[#{field-token ...} #{table-token ...} category]`. Promotes a weak field signal when the table name
+;; supplies the context.
+(def ^:private table-boosters
+ [[#{"token" "tokens" "key" "keys" "hash" "secret"}
+ #{"session" "sessions" "auth" "oauth" "credential" "credentials" "token" "tokens" "key" "keys" "secret" "secrets"}
+ :SEC_KEY]
+ [#{"notes" "comments" "remarks" "condition" "conditions" "treatment" "treatments" "procedure" "procedures"
+ "result" "results" "history" "record" "records"}
+ #{"patient" "patients" "medical" "clinical" "health" "hospital" "encounter" "encounters" "diagnosis"
+ "prescription" "prescriptions"}
+ :PHI]
+ [#{"template" "templates" "sample" "samples" "scan" "scans" "data" "vector" "embedding"}
+ #{"biometric" "biometrics" "fingerprint" "fingerprints" "facial" "voice" "iris" "retina" "dna" "genetic"
+ "genomic" "genome"}
+ :BIO_GEN]
+ [#{"number" "num" "no" "expiry" "expiration" "exp" "last4" "holder"}
+ #{"card" "cards" "credit" "bank"}
+ :PCI_FIN]
+ [#{"name" "address" "age" "street" "lat" "lon" "lng" "latitude" "longitude" "zipcode" "postcode"}
+ #{"user" "users" "person" "people" "customer" "customers" "employee" "employees" "member" "members" "contact"
+ "contacts" "patient" "patients" "profile" "profiles" "subscriber" "subscribers" "lead" "leads" "applicant"
+ "applicants" "student" "students" "candidate" "candidates"}
+ :PII]
+ [#{"amount" "value" "total"}
+ #{"salary" "salaries" "payroll" "compensation" "deal" "deals" "contract" "contracts" "forecast" "forecasts"
+ "budget" "budgets"}
+ :BIZ_CONF]])
+
+(def ^:private categories
+ (set lib.schema.metadata/column-data-sensitivity-types))
+
+(def ^:private precedence
+ (zipmap lib.schema.metadata/column-data-sensitivity-types (range)))
+
+(when-not config/is-prod?
+ (let [valid-category? #(and (contains? categories %) (not= :PUBLIC %))
+ valid-base-types? #(and (set? %) (seq %) (every? (fn [t] (isa? t :type/*)) %))
+ valid-token? #(re-matches #"[a-z0-9]+(?:_[a-z0-9]+)?" %)]
+ (doseq [[tokens base-types category] token-rules]
+ (assert (and (set? tokens) (every? valid-token? tokens)) (pr-str tokens))
+ (assert (valid-base-types? base-types) (pr-str base-types))
+ (assert (valid-category? category) (pr-str category)))
+ (doseq [[pattern base-types category] stem-rules]
+ (assert (instance? java.util.regex.Pattern pattern) (pr-str pattern))
+ (assert (valid-base-types? base-types) (pr-str base-types))
+ (assert (valid-category? category) (pr-str category)))
+ (doseq [[semantic-type category] (concat semantic-type->category user-table-semantic-type->category)]
+ (assert (isa? semantic-type :Semantic/*) (pr-str semantic-type))
+ (assert (valid-category? category) (pr-str category)))
+ (doseq [[field-tokens table-tokens category] table-boosters]
+ (assert (every? valid-token? field-tokens) (pr-str field-tokens))
+ (assert (every? valid-token? table-tokens) (pr-str table-tokens))
+ (assert (valid-category? category) (pr-str category)))))
+
+(defn- base-type-matches? [base-type base-types]
+ (some (partial isa? base-type) base-types))
+
+(defn- token-matches [{:keys [tokens pairs]} base-type]
+ (let [names (into tokens pairs)]
+ (for [[rule-names base-types category] token-rules
+ :when (and (some rule-names names)
+ (base-type-matches? base-type base-types))]
+ category)))
+
+(defn- stem-matches [lower-name base-type]
+ (for [[pattern base-types category] stem-rules
+ :when (and (base-type-matches? base-type base-types)
+ (re-find pattern lower-name))]
+ category))
+
+(defn- semantic-matches [semantic-type base-type entity-type]
+ (let [types (cond-> #{}
+ semantic-type (conj semantic-type)
+ (isa? base-type :Semantic/*) (conj base-type))]
+ (for [[st category] (cond-> semantic-type->category
+ (= :entity/UserTable entity-type) (merge user-table-semantic-type->category))
+ t types
+ :when (isa? t st)]
+ category)))
+
+(defn- fingerprint-matches [fingerprint base-type]
+ (when (and (isa? base-type :type/Text)
+ (>= (get-in fingerprint [:type :type/Text :percent-email] 0) 0.95))
+ [:PII]))
+
+(defn- booster-matches [{:keys [tokens]} table-name]
+ (when table-name
+ (let [table-tokens (:tokens (name->tokens table-name))]
+ (for [[field-tokens rule-table-tokens category] table-boosters
+ :when (and (some field-tokens tokens)
+ (some rule-table-tokens table-tokens))]
+ category))))
+
+(def ^:private Field
+ [:map
+ [:name :string]
+ [:base_type :keyword]
+ [:semantic_type {:optional true} [:maybe :keyword]]
+ [:fingerprint {:optional true} :any]])
+
+(def ^:private TableContext
+ [:maybe
+ [:map
+ [:name {:optional true} [:maybe :string]]
+ [:entity_type {:optional true} [:maybe :keyword]]]])
+
+(mu/defn infer-data-sensitivity :- [:maybe ::lib.schema.metadata/column.data-sensitivity]
+ "Infer the `data_sensitivity` category of `field` from its name, base type, semantic type, and fingerprint, with
+ `table-context` (`:name`, `:entity_type` of its Table) gating the weaker rules. Returns the highest-precedence
+ matching category, or nil when no rule matches. Never returns `:PUBLIC`."
+ [{field-name :name, :keys [base_type semantic_type fingerprint]} :- Field
+ {table-name :name, :keys [entity_type]} :- TableContext]
+ (let [tokens (name->tokens field-name)]
+ (->> (concat (token-matches tokens base_type)
+ (stem-matches (u/lower-case-en field-name) base_type)
+ (semantic-matches semantic_type base_type entity_type)
+ (fingerprint-matches fingerprint base_type)
+ (booster-matches tokens table-name))
+ (sort-by precedence)
+ first)))
diff --git a/src/metabase/analyze/core.clj b/src/metabase/analyze/core.clj
index 15a722608e2..179005c92c2 100644
--- a/src/metabase/analyze/core.clj
+++ b/src/metabase/analyze/core.clj
@@ -3,6 +3,7 @@
(:require
[metabase.analyze.classifiers.category]
[metabase.analyze.classifiers.core]
+ [metabase.analyze.classifiers.data-sensitivity]
[metabase.analyze.classifiers.name]
[metabase.analyze.fingerprint.fingerprinters]
[metabase.analyze.query-results]
@@ -11,6 +12,7 @@
(comment
metabase.analyze.classifiers.category/keep-me
metabase.analyze.classifiers.core/keep-me
+ metabase.analyze.classifiers.data-sensitivity/keep-me
metabase.analyze.classifiers.name/keep-me
metabase.analyze.fingerprint.fingerprinters/keep-me
metabase.analyze.query-results/keep-me)
@@ -21,6 +23,8 @@
category-cardinality-threshold]
[metabase.analyze.classifiers.core
run-classifiers]
+ [metabase.analyze.classifiers.data-sensitivity
+ infer-data-sensitivity]
[metabase.analyze.classifiers.name
infer-entity-type-by-name
infer-semantic-type-by-name]
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index c12b9a5a692..0b76470740b 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -310,8 +310,9 @@
(def column-data-sensitivity-types
"Possible values for column `:data-sensitivity`, most severe first. The order is the precedence an automated
- classifier uses when several categories match a single column. `nil` means the column has never been classified;
- `:PUBLIC` means it was reviewed and found not sensitive."
+ classifier uses when several categories match a single column. `nil` means the column has never been scanned or
+ labeled; `:PUBLIC` means it was scanned or reviewed and nothing sensitive was found. Whether a person made the call
+ is recorded by the presence of a value in `metabase_field_user_settings`, not by the value itself."
[:SEC_KEY ; Security Credentials & Secrets
:SYS_TELEMETRY ; Infrastructure & System Secrets
:PHI ; Protected Health Information
diff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.clj
index e40eb4490db..f4e9ec2bfa9 100644
--- a/src/metabase/sync/analyze.clj
+++ b/src/metabase/sync/analyze.clj
@@ -5,6 +5,7 @@
and infer field semantic types."
(:require
[metabase.sync.analyze.classify :as classify]
+ [metabase.sync.analyze.data-sensitivity :as sync.data-sensitivity]
[metabase.sync.analyze.fingerprint :as sync.fingerprint]
[metabase.sync.analyze.interestingness :as sync.interestingness]
[metabase.sync.db :as sync.db]
@@ -69,6 +70,7 @@
(classify/classify-fields! table)
(classify/classify-table! table)
(sync.interestingness/score-fields! table)
+ (sync.data-sensitivity/scan-table! table)
(update-fields-last-analyzed! table))
(defn- maybe-log-progress [progress-bar-fn]
@@ -93,6 +95,10 @@
(format "Interestingness scored %d fields, %d failed"
fields-scored fields-failed))
+(defn- data-sensitivity-summary [{:keys [fields-scanned fields-labeled fields-failed]}]
+ (format "Data sensitivity scanned %d fields, labeled %d, %d failed"
+ fields-scanned fields-labeled fields-failed))
+
(defn- make-analyze-steps [log-fn]
[(sync-util/create-sync-step "fingerprint-fields"
#(sync.fingerprint/fingerprint-fields-for-db! % log-fn)
@@ -105,7 +111,10 @@
classify-tables-summary)
(sync-util/create-sync-step "score-interestingness"
#(sync.interestingness/score-fields-for-db! % log-fn)
- interestingness-summary)])
+ interestingness-summary)
+ (sync-util/create-sync-step "classify-data-sensitivity"
+ #(sync.data-sensitivity/scan-fields-for-db! % log-fn)
+ data-sensitivity-summary)])
(mu/defn- analyze-db!*
"Shared core of [[analyze-db!]] and [[analyze-db-explicit!]]: the analysis work, without the
diff --git a/src/metabase/sync/analyze/data_sensitivity.clj b/src/metabase/sync/analyze/data_sensitivity.clj
new file mode 100644
index 00000000000..55503fdfa3d
--- /dev/null
+++ b/src/metabase/sync/analyze/data_sensitivity.clj
@@ -0,0 +1,165 @@
+(ns metabase.sync.analyze.data-sensitivity
+ "Analyze sub-step that labels Fields with a `data_sensitivity` category from
+ [[metabase.analyze.core/infer-data-sensitivity]], writing `:PUBLIC` when no rule matches so each field is scanned
+ once. Selects fields whose `data_sensitivity` is `NULL` (or `PUBLIC` too under `force?`) across every active table
+ in the database, hidden and cruft tables included. Inert unless [[metabase.sync.settings/data-sensitivity-scan-enabled]]
+ is true, except through [[scan-data-sensitivity!]]. User-set labels are protected by the `FieldUserSettings`
+ overlay applied on every Field update. [[reset-data-sensitivity!]] clears the classifier's own labels so a rule
+ change can reach fields that already carry a category."
+ (:require
+ [metabase.analyze.core :as analyze]
+ [metabase.sync.interface :as i]
+ [metabase.sync.settings :as sync.settings]
+ [metabase.sync.util :as sync-util]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [toucan2.core :as t2]
+ [toucan2.realize :as t2.realize]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private Stats
+ [:map
+ [:fields-scanned :int]
+ [:fields-labeled :int]
+ [:fields-failed :int]
+ [:fields-reset {:optional true} :int]])
+
+(def ^:private zero-stats
+ {:fields-scanned 0 :fields-labeled 0 :fields-failed 0})
+
+(def ^:private ScanOptions
+ [:map
+ [:force? {:optional true} [:maybe :boolean]]
+ [:ignore-setting? {:optional true} [:maybe :boolean]]])
+
+(defn- selection-clause [force?]
+ (if force?
+ [:or [:= :data_sensitivity nil] [:= :data_sensitivity "PUBLIC"]]
+ [:= :data_sensitivity nil]))
+
+(defn- skip? [ignore-setting?]
+ (and (not ignore-setting?)
+ (not (sync.settings/data-sensitivity-scan-enabled))))
+
+(mu/defn- fields-to-scan :- [:sequential i/FieldInstance]
+ [table :- i/TableInstance
+ force?]
+ (t2/select :model/Field
+ {:where [:and
+ [:= :table_id (u/the-id table)]
+ [:= :active true]
+ [:not= :visibility_type "retired"]
+ (selection-clause force?)]
+ :order-by [[:id :asc]]}))
+
+(mu/defn- classify-and-save!
+ "Returns the inferred category, nil when no rule matched (in which case `:PUBLIC` was written), or the Exception."
+ [field :- i/FieldInstance
+ table :- i/TableInstance]
+ (sync-util/with-error-handling (format "Error classifying data sensitivity for %s" (sync-util/name-for-logging field))
+ (let [category (analyze/infer-data-sensitivity field {:name (:name table) :entity_type (:entity_type table)})]
+ (t2/update! :model/Field (u/the-id field) {:data_sensitivity (or category :PUBLIC)})
+ category)))
+
+(mu/defn scan-table! :- Stats
+ "Label every unscanned Field in `table`. `fields-scanned` counts every selected field, `fields-labeled` those that
+ matched a category, `fields-failed` those whose classification threw; the remainder were written `:PUBLIC`."
+ [table :- i/TableInstance
+ & {:keys [force? ignore-setting?]} :- [:maybe ScanOptions]]
+ (if (skip? ignore-setting?)
+ zero-stats
+ (let [stats (reduce (fn [stats field]
+ (let [result (classify-and-save! field table)]
+ (log/debugf "Data sensitivity for %s: %s" (sync-util/name-for-logging field)
+ (if (instance? Exception result) "failed" (or result :PUBLIC)))
+ (cond-> (update stats :fields-scanned inc)
+ (instance? Exception result) (update :fields-failed inc)
+ (keyword? result) (update :fields-labeled inc))))
+ zero-stats
+ (fields-to-scan table force?))]
+ (when (pos? (:fields-scanned stats))
+ (log/infof "Data sensitivity scanned %d fields in %s: %d labeled, %d failed"
+ (:fields-scanned stats) (sync-util/name-for-logging table)
+ (:fields-labeled stats) (:fields-failed stats)))
+ stats)))
+
+(mu/defn- table-ids-with-unscanned-fields :- [:maybe [:set pos-int?]]
+ [database :- i/DatabaseInstance
+ force?]
+ (t2/select-fn-set :table_id :model/Field
+ {:select [[:metabase_field.table_id :table_id]]
+ :from [:metabase_field]
+ :join [[:metabase_table] [:= :metabase_field.table_id :metabase_table.id]]
+ :where [:and
+ [:= :metabase_table.db_id (u/the-id database)]
+ [:= :metabase_table.active true]
+ [:= :metabase_field.active true]
+ [:not= :metabase_field.visibility_type "retired"]
+ (selection-clause force?)]
+ :group-by [:metabase_field.table_id]}))
+
+(mu/defn scan-fields-for-db! :- Stats
+ "Label every unscanned Field in every active table of `database`. `log-fn` is accepted for parity with the other
+ analyze steps and not called: the step reports per table through the log, not the progress bar."
+ [database :- i/DatabaseInstance
+ _log-fn
+ & {:keys [force? ignore-setting?]} :- [:maybe ScanOptions]]
+ (if (skip? ignore-setting?)
+ zero-stats
+ (let [table-ids (table-ids-with-unscanned-fields database force?)]
+ (if (empty? table-ids)
+ zero-stats
+ (transduce (comp (map t2.realize/realize)
+ (map #(scan-table! % :force? force? :ignore-setting? true)))
+ (partial merge-with +)
+ zero-stats
+ (t2/reducible-select :model/Table
+ :id [:in table-ids]
+ {:order-by [[:schema :asc] [:name :asc]]}))))))
+
+(defn- scope-clause [database-or-table]
+ (case (t2/model database-or-table)
+ :model/Table [:= :table_id (u/the-id database-or-table)]
+ :model/Database [:in :table_id ^:allow-subquery {:select [:id]
+ :from [:metabase_table]
+ :where [:= :db_id (u/the-id database-or-table)]}]))
+
+(def ^:private classifier-label-clause
+ "A non-null `data_sensitivity` with no value in the `FieldUserSettings` mirror was written by the classifier."
+ [:and
+ [:not= :data_sensitivity nil]
+ [:not [:exists ^:allow-subquery {:select [1]
+ :from [[:metabase_field_user_settings :s]]
+ :where [:and
+ [:= :s.field_id :metabase_field.id]
+ [:not= :s.data_sensitivity nil]]}]]])
+
+(mu/defn reset-data-sensitivity! :- :int
+ "REPL entry point: clear every classifier-written `data_sensitivity` in a Database or a single Table so the next
+ scan recomputes them, including fields that carry a category and would otherwise never be reselected. Labels with
+ a value in the `FieldUserSettings` mirror are human-set and untouched. Returns the number of fields cleared."
+ [database-or-table :- [:or i/DatabaseInstance i/TableInstance]]
+ (let [n (t2/query-one {:update :metabase_field
+ :set {:data_sensitivity nil}
+ :where [:and (scope-clause database-or-table) classifier-label-clause]})]
+ (log/infof "Data sensitivity reset %d classifier-written labels in %s" n (sync-util/name-for-logging database-or-table))
+ n))
+
+(mu/defn scan-data-sensitivity! :- Stats
+ "REPL entry point: scan a Database or a single Table regardless of the `data-sensitivity-scan-enabled` setting.
+ With `:force? true` fields already labeled `:PUBLIC` are rescanned too; fields carrying a category are not. With
+ `:reset? true` every classifier-written label is cleared first via [[reset-data-sensitivity!]] so the whole scope
+ is recomputed under the current rules, and `:fields-reset` is added to the stats. User-set labels survive in
+ every mode because the `FieldUserSettings` overlay is applied on every Field update."
+ [database-or-table :- [:or i/DatabaseInstance i/TableInstance]
+ & {:keys [force? reset?]} :- [:maybe [:map
+ [:force? {:optional true} [:maybe :boolean]]
+ [:reset? {:optional true} [:maybe :boolean]]]]]
+ (let [reset (when reset? (reset-data-sensitivity! database-or-table))
+ stats (case (t2/model database-or-table)
+ :model/Table (scan-table! database-or-table :force? force? :ignore-setting? true)
+ :model/Database (scan-fields-for-db! database-or-table (constantly nil) :force? force? :ignore-setting? true))]
+ (cond-> stats
+ reset (assoc :fields-reset reset))))
diff --git a/src/metabase/sync/core.clj b/src/metabase/sync/core.clj
index 4617e6f4833..c51ead83195 100644
--- a/src/metabase/sync/core.clj
+++ b/src/metabase/sync/core.clj
@@ -3,6 +3,7 @@
-- Tables and Fields."
(:require
[metabase.sync.analyze]
+ [metabase.sync.analyze.data-sensitivity]
[metabase.sync.field-values]
[metabase.sync.sync]
[metabase.sync.sync-metadata]
@@ -12,6 +13,7 @@
(comment
metabase.sync.analyze/keep-me
+ metabase.sync.analyze.data-sensitivity/keep-me
metabase.sync.field-values/keep-me
metabase.sync.sync/keep-me
metabase.sync.sync-metadata/keep-me
@@ -22,6 +24,9 @@
[metabase.sync.analyze
analyze-db!
analyze-db-explicit!]
+ [metabase.sync.analyze.data-sensitivity
+ reset-data-sensitivity!
+ scan-data-sensitivity!]
[metabase.sync.field-values
update-field-values!
update-field-values-for-table!]
diff --git a/src/metabase/sync/settings.clj b/src/metabase/sync/settings.clj
index feafda677c7..fdec3a5030d 100644
--- a/src/metabase/sync/settings.clj
+++ b/src/metabase/sync/settings.clj
@@ -57,3 +57,15 @@
:export? true
:type :integer
:default 10000)
+
+(defsetting data-sensitivity-scan-enabled
+ "When true, the analyze phase of sync labels every unlabeled field with a data_sensitivity category inferred from
+ its name, types, and fingerprint, writing PUBLIC when nothing matches. Metadata only: the label does not mask or
+ restrict anything. Labels set by a user are never overwritten."
+ :type :boolean
+ :default false
+ :setter :none
+ :visibility :internal
+ :export? false
+ :doc "Scans run with the scheduled analyze pass and with Sync database schema now on a database's admin
+ page. Fields that already carry a label, including PUBLIC, are not rescanned.")
diff --git a/test/metabase/analyze/classifiers/data_sensitivity_test.clj b/test/metabase/analyze/classifiers/data_sensitivity_test.clj
new file mode 100644
index 00000000000..163ddd617ca
--- /dev/null
+++ b/test/metabase/analyze/classifiers/data_sensitivity_test.clj
@@ -0,0 +1,383 @@
+(ns metabase.analyze.classifiers.data-sensitivity-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.analyze.classifiers.data-sensitivity :as classifiers.data-sensitivity]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]))
+
+(defn- infer
+ ([field-name base-type]
+ (infer field-name base-type nil nil))
+ ([field-name base-type semantic-type table-context]
+ (classifiers.data-sensitivity/infer-data-sensitivity
+ {:name field-name :base_type base-type :semantic_type semantic-type}
+ table-context)))
+
+(deftest ^:parallel name->tokens-test
+ (testing "separators and camelCase all tokenize to the same tokens and adjacent pair"
+ (doseq [s ["firstName" "first_name" "FIRST-NAME" "first.name" "first name"]]
+ (testing (pr-str s)
+ (is (= {:tokens #{"first" "name"} :pairs #{"first_name"}}
+ (#'classifiers.data-sensitivity/name->tokens s))))))
+ (testing "tokens are whole words, not substrings"
+ (let [{:keys [tokens]} (#'classifiers.data-sensitivity/name->tokens "zip_code")]
+ (is (= #{"zip" "code"} tokens))
+ (is (not (contains? tokens "ip")))))
+ (testing "pairs are adjacent only"
+ (is (= #{"date_of" "of_birth"}
+ (:pairs (#'classifiers.data-sensitivity/name->tokens "date_of_birth")))))
+ (testing "a single token has no pairs"
+ (is (= {:tokens #{"ssn"} :pairs #{}}
+ (#'classifiers.data-sensitivity/name->tokens "SSN")))))
+
+(deftest ^:parallel positive-cases-per-category-test
+ (doseq [[field-name base-type semantic-type table-context expected]
+ [["password" :type/Text nil nil :SEC_KEY]
+ ["apiKey" :type/Text nil nil :SEC_KEY]
+ ["refresh_token" :type/Text nil nil :SEC_KEY]
+ ["client_secret" :type/Text nil nil :SEC_KEY]
+ ["salt" :type/Text nil nil :SEC_KEY]
+ ["ip_address" :type/Text nil nil :SYS_TELEMETRY]
+ ["client_ip" :type/Text nil nil :SYS_TELEMETRY]
+ ["user_agent" :type/Text nil nil :SYS_TELEMETRY]
+ ["mac_address" :type/Text nil nil :SYS_TELEMETRY]
+ ["device_id" :type/Integer nil nil :SYS_TELEMETRY]
+ ["diagnosis" :type/Text nil nil :PHI]
+ ["diagnosis_code" :type/Text nil nil :PHI]
+ ["icd10" :type/Text nil nil :PHI]
+ ["patient_id" :type/Integer nil nil :PHI]
+ ["bmi" :type/Float nil nil :PHI]
+ ["dna" :type/Text nil nil :BIO_GEN]
+ ["fingerprint" :type/Text nil nil :BIO_GEN]
+ ["retina_scan" :type/Text nil nil :BIO_GEN]
+ ["face_id" :type/Text nil nil :BIO_GEN]
+ ["card_number" :type/Integer nil nil :PCI_FIN]
+ ["credit_card" :type/Text nil nil :PCI_FIN]
+ ["cvv" :type/Integer nil nil :PCI_FIN]
+ ["iban" :type/Text nil nil :PCI_FIN]
+ ["routing_number" :type/Text nil nil :PCI_FIN]
+ ["gender" :type/Text nil nil :SENS_PERS]
+ ["sex" :type/Text nil nil :SENS_PERS]
+ ["ethnicity" :type/Text nil nil :SENS_PERS]
+ ["religion" :type/Text nil nil :SENS_PERS]
+ ["is_pregnant" :type/Boolean nil nil :SENS_PERS]
+ ["ssn" :type/Text nil nil :PII]
+ ["passport_number" :type/Text nil nil :PII]
+ ["date_of_birth" :type/Date nil nil :PII]
+ ["email_address" :type/Text nil nil :PII]
+ ["phone" :type/Text nil nil :PII]
+ ["phone_number" :type/Integer nil nil :PII]
+ ["shipping_address" :type/Text nil nil :PII]
+ ["source_code" :type/Text nil nil :CORP_IP]
+ ["repo_url" :type/Text nil nil :CORP_IP]
+ ["algorithm" :type/Text nil nil :CORP_IP]
+ ["salary" :type/Float nil nil :BIZ_CONF]
+ ["revenue" :type/Decimal nil nil :BIZ_CONF]
+ ["gross_margin" :type/Float nil nil :BIZ_CONF]]]
+ (testing (pr-str (list 'infer-data-sensitivity field-name base-type semantic-type table-context))
+ (is (= expected
+ (infer field-name base-type semantic-type table-context))))))
+
+(deftest ^:parallel base-type-gates-token-rules-test
+ (testing "a token rule only fires when the base type isa? one of the rule's base types"
+ (is (= :BIZ_CONF (infer "salary" :type/Integer)))
+ (is (nil? (infer "salary" :type/Text)))
+ (is (= :SEC_KEY (infer "password" :type/Text)))
+ (is (nil? (infer "password" :type/Integer)))
+ (is (= :SENS_PERS (infer "race" :type/Text)))
+ (is (nil? (infer "race_id" :type/Integer)))
+ (is (= :PHI (infer "patient" :type/Text)))
+ (is (nil? (infer "patient_count" :type/Integer)))))
+
+(deftest ^:parallel non-text-base-types-test
+ (testing "IPAddress and Structured base types, which are not descendants of :type/Text, match the text rules that list them"
+ (is (= :SYS_TELEMETRY (infer "client_ip" :type/IPAddress)))
+ (is (= :SYS_TELEMETRY (infer "mac_address" :type/Structured)))
+ (is (= :SEC_KEY (infer "credentials" :type/JSON)))
+ (is (= :SEC_KEY (infer "password" :type/XML)))
+ (is (= :PHI (infer "medical_history" :type/JSON))))
+ (testing "an IPAddress base type implies SYS_TELEMETRY whatever the column is named"
+ (is (= :SYS_TELEMETRY (infer "foo" :type/IPAddress))))
+ (testing "rules that stay text-only do not match structured columns"
+ (is (nil? (infer "home_address" :type/JSON)))
+ (is (nil? (infer "source_code" :type/JSON)))
+ (is (nil? (infer "settings" :type/JSON)))))
+
+(deftest ^:parallel integer-and-band-variants-test
+ (testing "dob matches as an integer as well as a date or text"
+ (is (= :PII (infer "dob" :type/Integer)))
+ (is (= :PII (infer "dob" :type/Date)))
+ (is (= :PII (infer "dob" :type/Text)))
+ (is (nil? (infer "dob" :type/Boolean)))
+ (is (nil? (infer "birthday" :type/Integer))))
+ (testing "salary and compensation bands match as text or integer codes while bare salary stays numeric-only"
+ (is (= :BIZ_CONF (infer "salary_band" :type/Text)))
+ (is (= :BIZ_CONF (infer "compensation_tier" :type/Text)))
+ (is (= :BIZ_CONF (infer "pay_grade" :type/Integer)))
+ (is (nil? (infer "salary" :type/Text)))
+ (is (nil? (infer "salary_currency" :type/Text)))))
+
+(deftest ^:parallel false-positives-test
+ (testing "names that share a substring or token with a rule but are not sensitive return nil"
+ (doseq [[field-name base-type]
+ [["zip_code" :type/Text]
+ ["ship_date" :type/Date]
+ ["shipping_address_id" :type/Integer]
+ ["description" :type/Text]
+ ["monkey" :type/Text]
+ ["foreign_key" :type/Integer]
+ ["primary_key" :type/Integer]
+ ["key_id" :type/Integer]
+ ["token_count" :type/Integer]
+ ["session_count" :type/Integer]
+ ["hostname_count" :type/Integer]
+ ["content_hash" :type/Text]
+ ["adobe_id" :type/Text]
+ ["sussex" :type/Text]
+ ["product_name" :type/Text]
+ ["filename" :type/Text]
+ ["age" :type/Integer]
+ ["address" :type/Text]
+ ["city" :type/Text]
+ ["state" :type/Text]
+ ["country" :type/Text]
+ ["name" :type/Text]
+ ["phone_id" :type/Integer]
+ ["password_changed_at" :type/DateTime]
+ ["passwordless_login_count" :type/Integer]
+ ["emailed_at" :type/DateTime]
+ ["email_count" :type/Integer]
+ ["businessName" :type/Text]
+ ["business_name" :type/Text]
+ ["secretary" :type/Text]
+ ["disabled" :type/Boolean]
+ ["condition" :type/Text]
+ ["treatment" :type/Text]
+ ["margin_top" :type/Text]
+ ["token" :type/Text]]]
+ (testing (pr-str field-name)
+ (is (nil? (infer field-name base-type))))))
+ (testing "geography semantic types alone do not imply PII outside a UserTable"
+ (is (nil? (infer "state" :type/Text :type/State {:entity_type :entity/GenericTable})))
+ (is (nil? (infer "state" :type/Text :type/State {:name "orders"})))))
+
+(deftest ^:parallel stem-rules-test
+ (testing "stems catch spellings the tokenizer splits differently"
+ (is (= :PII (infer "ssn4" :type/Text)))
+ (is (= :PII (infer "last4ssn" :type/Text)))
+ (is (= :PII (infer "passportno" :type/Text)))
+ (is (= :PII (infer "userbirthdate" :type/Date)))
+ (is (= :PII (infer "useremail" :type/Text)))
+ (is (= :SEC_KEY (infer "passwd" :type/Text)))
+ (is (= :SEC_KEY (infer "clientsecret" :type/Text)))
+ (is (= :PCI_FIN (infer "cardcvv" :type/Text)))
+ (is (= :PCI_FIN (infer "ibannumber" :type/Text))))
+ (testing "stems are bounded so common words containing them do not match"
+ (is (nil? (infer "businessname" :type/Text)))
+ (is (nil? (infer "secretary" :type/Text)))
+ (is (nil? (infer "birthday_reminder_sent" :type/Boolean)))))
+
+(deftest ^:parallel precedence-test
+ (testing "the earliest category in column-data-sensitivity-types wins when several rules match"
+ (is (= :PCI_FIN (infer "ssn_card_number" :type/Text)))
+ (is (= :SEC_KEY (infer "ssn_password" :type/Text)))
+ (is (= :SEC_KEY (infer "salary_password" :type/Text)))
+ (is (= :SYS_TELEMETRY (infer "device_fingerprint" :type/Text)))
+ (is (= :SEC_KEY (infer "device_token" :type/Text)))
+ (is (= :PHI (infer "patient_email" :type/Text)))))
+
+(deftest ^:parallel user-table-gate-test
+ (testing "geography and name semantic types imply PII only on a UserTable"
+ (doseq [semantic-type [:type/Name :type/Address :type/City :type/State :type/Country :type/ZipCode
+ :type/Latitude :type/Longitude :type/User :type/Author]]
+ (testing (pr-str semantic-type)
+ (is (nil? (infer "col" :type/Text semantic-type {:entity_type :entity/GenericTable})))
+ (is (nil? (infer "col" :type/Text semantic-type nil)))
+ (is (= :PII (infer "col" :type/Text semantic-type {:entity_type :entity/UserTable}))))))
+ (testing "explicit name tokens fire regardless of table"
+ (is (= :PII (infer "first_name" :type/Text nil {:entity_type :entity/GenericTable})))
+ (is (= :PII (infer "first_name" :type/Text nil {:entity_type :entity/UserTable}))))
+ (testing "ungated semantic types fire regardless of table"
+ (is (= :PII (infer "col_3" :type/Text :type/Email {:entity_type :entity/GenericTable})))
+ (is (= :PII (infer "col_4" :type/Date :type/Birthdate {:entity_type :entity/GenericTable})))
+ (is (= :SYS_TELEMETRY (infer "col_5" :type/Text :type/IPAddress {:entity_type :entity/GenericTable}))))
+ (testing "semantic types outside both maps do not match"
+ (doseq [semantic-type [:type/Category :type/Title :type/Description :type/Company :type/Income :type/Cost
+ :type/PK :type/FK :type/CreationTimestamp]]
+ (testing (pr-str semantic-type)
+ (is (nil? (infer "kind" :type/Text semantic-type {:entity_type :entity/UserTable})))))))
+
+(deftest ^:parallel table-booster-test
+ (testing "a weak field token is promoted only when the table name supplies the context"
+ (is (nil? (infer "notes" :type/Text nil nil)))
+ (is (nil? (infer "notes" :type/Text nil {:name "products"})))
+ (is (= :PHI (infer "notes" :type/Text nil {:name "patient_visits"})))
+ (is (= :PHI (infer "NOTES" :type/Text nil {:name "PATIENT_VISITS"})))
+ (is (= :PHI (infer "condition" :type/Text nil {:name "patients"})))
+ (is (= :SEC_KEY (infer "token" :type/Text nil {:name "sessions"})))
+ (is (= :SEC_KEY (infer "key" :type/Text nil {:name "api_keys"})))
+ (is (nil? (infer "key" :type/Text nil {:name "settings"})))
+ (is (= :BIO_GEN (infer "template" :type/Text nil {:name "fingerprint_templates"})))
+ (is (= :PCI_FIN (infer "number" :type/Text nil {:name "credit_cards"})))
+ (is (= :PCI_FIN (infer "last4" :type/Text nil {:name "cards"})))
+ (is (nil? (infer "number" :type/Text nil {:name "invoices"})))
+ (is (= :PII (infer "name" :type/Text nil {:name "customers"})))
+ (is (= :PII (infer "age" :type/Integer nil {:name "users"})))
+ (is (= :PII (infer "address" :type/Text nil {:name "employees"})))
+ (is (nil? (infer "name" :type/Text nil {:name "products"})))
+ (is (= :BIZ_CONF (infer "amount" :type/Float nil {:name "deals"})))
+ (is (nil? (infer "amount" :type/Float nil {:name "orders"})))))
+
+(deftest ^:parallel fingerprint-rule-test
+ (let [infer-fp (fn [base-type percent-email]
+ (classifiers.data-sensitivity/infer-data-sensitivity
+ {:name "col_7"
+ :base_type base-type
+ :fingerprint {:type {:type/Text {:percent-email percent-email}}}}
+ nil))]
+ (testing "a text column whose values are almost all emails is PII"
+ (is (= :PII (infer-fp :type/Text 0.97))))
+ (testing "below the threshold nothing matches"
+ (is (nil? (infer-fp :type/Text 0.5))))
+ (testing "the rule only applies to text base types"
+ (is (nil? (infer-fp :type/Integer 0.97))))
+ (testing "a missing fingerprint is not an error"
+ (is (nil? (infer "col_7" :type/Text))))))
+
+(def sample-database-fields
+ "`[table entity-type field base-type semantic-type]` for every column of the Sample Database, as synced on H2."
+ [["ACCOUNTS" :entity/UserTable "ACTIVE_SUBSCRIPTION" :type/Boolean nil]
+ ["ACCOUNTS" :entity/UserTable "CANCELED_AT" :type/DateTime :type/CancelationTimestamp]
+ ["ACCOUNTS" :entity/UserTable "COUNTRY" :type/Text :type/Country]
+ ["ACCOUNTS" :entity/UserTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]
+ ["ACCOUNTS" :entity/UserTable "EMAIL" :type/Text :type/Email]
+ ["ACCOUNTS" :entity/UserTable "FIRST_NAME" :type/Text :type/Name]
+ ["ACCOUNTS" :entity/UserTable "ID" :type/BigInteger :type/PK]
+ ["ACCOUNTS" :entity/UserTable "LAST_NAME" :type/Text :type/Name]
+ ["ACCOUNTS" :entity/UserTable "LATITUDE" :type/Float :type/Latitude]
+ ["ACCOUNTS" :entity/UserTable "LEGACY_PLAN" :type/Boolean nil]
+ ["ACCOUNTS" :entity/UserTable "LONGITUDE" :type/Float :type/Longitude]
+ ["ACCOUNTS" :entity/UserTable "PLAN" :type/Text :type/Category]
+ ["ACCOUNTS" :entity/UserTable "SEATS" :type/Integer nil]
+ ["ACCOUNTS" :entity/UserTable "SOURCE" :type/Text :type/Source]
+ ["ACCOUNTS" :entity/UserTable "TRIAL_CONVERTED" :type/Boolean nil]
+ ["ACCOUNTS" :entity/UserTable "TRIAL_ENDS_AT" :type/DateTime nil]
+ ["ANALYTIC_EVENTS" :entity/EventTable "ACCOUNT_ID" :type/BigInteger :type/FK]
+ ["ANALYTIC_EVENTS" :entity/EventTable "BUTTON_LABEL" :type/Text :type/Category]
+ ["ANALYTIC_EVENTS" :entity/EventTable "EVENT" :type/Text :type/Category]
+ ["ANALYTIC_EVENTS" :entity/EventTable "ID" :type/BigInteger :type/PK]
+ ["ANALYTIC_EVENTS" :entity/EventTable "PAGE_URL" :type/Text :type/URL]
+ ["ANALYTIC_EVENTS" :entity/EventTable "TIMESTAMP" :type/DateTime nil]
+ ["FEEDBACK" :entity/GenericTable "ACCOUNT_ID" :type/BigInteger :type/FK]
+ ["FEEDBACK" :entity/GenericTable "BODY" :type/Text nil]
+ ["FEEDBACK" :entity/GenericTable "DATE_RECEIVED" :type/DateTime nil]
+ ["FEEDBACK" :entity/GenericTable "EMAIL" :type/Text :type/Email]
+ ["FEEDBACK" :entity/GenericTable "ID" :type/BigInteger :type/PK]
+ ["FEEDBACK" :entity/GenericTable "RATING" :type/Integer :type/Score]
+ ["FEEDBACK" :entity/GenericTable "RATING_MAPPED" :type/Text :type/Category]
+ ["INVOICES" :entity/GenericTable "ACCOUNT_ID" :type/BigInteger :type/FK]
+ ["INVOICES" :entity/GenericTable "DATE_RECEIVED" :type/DateTime nil]
+ ["INVOICES" :entity/GenericTable "EXPECTED_INVOICE" :type/Boolean nil]
+ ["INVOICES" :entity/GenericTable "ID" :type/BigInteger :type/PK]
+ ["INVOICES" :entity/GenericTable "PAYMENT" :type/Float nil]
+ ["INVOICES" :entity/GenericTable "PLAN" :type/Text :type/Category]
+ ["ORDERS" :entity/TransactionTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]
+ ["ORDERS" :entity/TransactionTable "DISCOUNT" :type/Float :type/Discount]
+ ["ORDERS" :entity/TransactionTable "ID" :type/BigInteger :type/PK]
+ ["ORDERS" :entity/TransactionTable "PRODUCT_ID" :type/Integer :type/FK]
+ ["ORDERS" :entity/TransactionTable "QUANTITY" :type/Integer :type/Quantity]
+ ["ORDERS" :entity/TransactionTable "SUBTOTAL" :type/Float nil]
+ ["ORDERS" :entity/TransactionTable "TAX" :type/Float nil]
+ ["ORDERS" :entity/TransactionTable "TOTAL" :type/Float nil]
+ ["ORDERS" :entity/TransactionTable "USER_ID" :type/Integer :type/FK]
+ ["PEOPLE" :entity/UserTable "ADDRESS" :type/Text nil]
+ ["PEOPLE" :entity/UserTable "BIRTH_DATE" :type/Date nil]
+ ["PEOPLE" :entity/UserTable "CITY" :type/Text :type/City]
+ ["PEOPLE" :entity/UserTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]
+ ["PEOPLE" :entity/UserTable "EMAIL" :type/Text :type/Email]
+ ["PEOPLE" :entity/UserTable "ID" :type/BigInteger :type/PK]
+ ["PEOPLE" :entity/UserTable "LATITUDE" :type/Float :type/Latitude]
+ ["PEOPLE" :entity/UserTable "LONGITUDE" :type/Float :type/Longitude]
+ ["PEOPLE" :entity/UserTable "NAME" :type/Text :type/Name]
+ ["PEOPLE" :entity/UserTable "PASSWORD" :type/Text nil]
+ ["PEOPLE" :entity/UserTable "SOURCE" :type/Text :type/Source]
+ ["PEOPLE" :entity/UserTable "STATE" :type/Text :type/State]
+ ["PEOPLE" :entity/UserTable "ZIP" :type/Text :type/ZipCode]
+ ["PRODUCTS" :entity/ProductTable "CATEGORY" :type/Text :type/Category]
+ ["PRODUCTS" :entity/ProductTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]
+ ["PRODUCTS" :entity/ProductTable "EAN" :type/Text nil]
+ ["PRODUCTS" :entity/ProductTable "ID" :type/BigInteger :type/PK]
+ ["PRODUCTS" :entity/ProductTable "PRICE" :type/Float nil]
+ ["PRODUCTS" :entity/ProductTable "RATING" :type/Float :type/Score]
+ ["PRODUCTS" :entity/ProductTable "TITLE" :type/Text :type/Title]
+ ["PRODUCTS" :entity/ProductTable "VENDOR" :type/Text :type/Company]
+ ["REVIEWS" :entity/GenericTable "BODY" :type/Text :type/Description]
+ ["REVIEWS" :entity/GenericTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]
+ ["REVIEWS" :entity/GenericTable "ID" :type/BigInteger :type/PK]
+ ["REVIEWS" :entity/GenericTable "PRODUCT_ID" :type/Integer :type/FK]
+ ["REVIEWS" :entity/GenericTable "RATING" :type/Integer :type/Score]
+ ["REVIEWS" :entity/GenericTable "REVIEWER" :type/Text nil]])
+
+(def sample-database-expectations
+ "Expected `data_sensitivity` for every Sample Database column after one scan, keyed by `[table field]`. Every
+ column absent from this map is expected to be `:PUBLIC`."
+ {["ACCOUNTS" "COUNTRY"] :PII
+ ["ACCOUNTS" "EMAIL"] :PII
+ ["ACCOUNTS" "FIRST_NAME"] :PII
+ ["ACCOUNTS" "LAST_NAME"] :PII
+ ["ACCOUNTS" "LATITUDE"] :PII
+ ["ACCOUNTS" "LONGITUDE"] :PII
+ ["FEEDBACK" "EMAIL"] :PII
+ ["PEOPLE" "ADDRESS"] :PII
+ ["PEOPLE" "BIRTH_DATE"] :PII
+ ["PEOPLE" "CITY"] :PII
+ ["PEOPLE" "EMAIL"] :PII
+ ["PEOPLE" "LATITUDE"] :PII
+ ["PEOPLE" "LONGITUDE"] :PII
+ ["PEOPLE" "NAME"] :PII
+ ["PEOPLE" "PASSWORD"] :SEC_KEY
+ ["PEOPLE" "STATE"] :PII
+ ["PEOPLE" "ZIP"] :PII})
+
+(deftest ^:parallel sample-database-test
+ (testing "every Sample Database column classifies to its expected category, or nil where PUBLIC is expected"
+ (doseq [[table entity-type field base-type semantic-type] sample-database-fields]
+ (testing (str table "." field)
+ (is (= (get sample-database-expectations [table field] :PUBLIC)
+ (or (infer field base-type semantic-type {:name table :entity_type entity-type})
+ :PUBLIC)))))))
+
+(deftest ^:parallel rule-data-invariants-test
+ (let [categories (set lib.schema.metadata/column-data-sensitivity-types)
+ valid-category? (fn [c] (and (contains? categories c) (not= :PUBLIC c)))
+ valid-token? (fn [t] (re-matches #"[a-z0-9]+(?:_[a-z0-9]+)?" t))
+ base-types-ok? (fn [bts] (and (set? bts) (seq bts) (every? #(isa? % :type/*) bts)))]
+ (testing "no rule outputs :PUBLIC and every output is an enum member"
+ (doseq [[_ _ category] (concat @#'classifiers.data-sensitivity/token-rules
+ @#'classifiers.data-sensitivity/stem-rules
+ @#'classifiers.data-sensitivity/table-boosters)]
+ (is (valid-category? category) (pr-str category)))
+ (doseq [[_ category] (concat @#'classifiers.data-sensitivity/semantic-type->category
+ @#'classifiers.data-sensitivity/user-table-semantic-type->category)]
+ (is (valid-category? category) (pr-str category))))
+ (testing "token rules use lowercase tokens or single underscore pairs and valid base types"
+ (doseq [[tokens base-types _] @#'classifiers.data-sensitivity/token-rules]
+ (is (set? tokens))
+ (is (every? valid-token? tokens) (pr-str tokens))
+ (is (base-types-ok? base-types) (pr-str base-types))))
+ (testing "every rule token can be produced by the tokenizer, so no rule is unreachable"
+ (doseq [[tokens _ _] @#'classifiers.data-sensitivity/token-rules
+ token tokens
+ :let [{:keys [tokens pairs]} (#'classifiers.data-sensitivity/name->tokens token)]]
+ (is (contains? (into tokens pairs) token) token)))
+ (testing "stem rules are compiled patterns with valid base types"
+ (doseq [[pattern base-types _] @#'classifiers.data-sensitivity/stem-rules]
+ (is (instance? java.util.regex.Pattern pattern))
+ (is (base-types-ok? base-types) (pr-str base-types))))
+ (testing "semantic maps key on semantic types"
+ (doseq [[semantic-type _] (concat @#'classifiers.data-sensitivity/semantic-type->category
+ @#'classifiers.data-sensitivity/user-table-semantic-type->category)]
+ (is (isa? semantic-type :Semantic/*) (pr-str semantic-type))))
+ (testing "booster token sets are lowercase tokens"
+ (doseq [[field-tokens table-tokens _] @#'classifiers.data-sensitivity/table-boosters]
+ (is (every? valid-token? field-tokens) (pr-str field-tokens))
+ (is (every? valid-token? table-tokens) (pr-str table-tokens))))))
diff --git a/test/metabase/sync/analyze/data_sensitivity_test.clj b/test/metabase/sync/analyze/data_sensitivity_test.clj
new file mode 100644
index 00000000000..2337cd4acf5
--- /dev/null
+++ b/test/metabase/sync/analyze/data_sensitivity_test.clj
@@ -0,0 +1,285 @@
+(ns metabase.sync.analyze.data-sensitivity-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.analyze.core :as analyze]
+ [metabase.sync.analyze.data-sensitivity :as sync.data-sensitivity]
+ [metabase.sync.core :as sync]
+ [metabase.test :as mt]
+ [metabase.util :as u]
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
+ [toucan2.core :as t2]))
+
+(defn- label [field-or-id]
+ (t2/select-one-fn :data_sensitivity :model/Field :id (u/the-id field-or-id)))
+
+(defn- mirror-label [field-or-id]
+ (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id (u/the-id field-or-id)))
+
+(defn- selected-names [table force?]
+ (set (map :name (#'sync.data-sensitivity/fields-to-scan table force?))))
+
+(deftest ^:parallel fields-to-scan-test
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "app_users"}
+ :model/Field _ {:table_id (:id table) :name "unscanned" :base_type :type/Text}
+ :model/Field _ {:table_id (:id table) :name "public" :base_type :type/Text :data_sensitivity :PUBLIC}
+ :model/Field _ {:table_id (:id table) :name "labeled" :base_type :type/Text :data_sensitivity :PII}
+ :model/Field _ {:table_id (:id table) :name "retired" :base_type :type/Text :visibility_type :retired}
+ :model/Field _ {:table_id (:id table) :name "sensitive" :base_type :type/Text :visibility_type :sensitive}
+ :model/Field _ {:table_id (:id table) :name "hidden" :base_type :type/Text :visibility_type :hidden}
+ :model/Field _ {:table_id (:id table) :name "details" :base_type :type/Text :visibility_type :details-only}
+ :model/Field _ {:table_id (:id table) :name "inactive" :base_type :type/Text :active false}]
+ (testing "a scheduled scan selects NULL fields of any visibility except retired, never inactive or labeled ones"
+ (is (= #{"unscanned" "sensitive" "hidden" "details"}
+ (selected-names table false))))
+ (testing "a forced scan also reselects PUBLIC fields but never categorized ones"
+ (is (= #{"unscanned" "sensitive" "hidden" "details" "public"}
+ (selected-names table true))))))
+
+(deftest ^:parallel table-ids-with-unscanned-fields-test
+ (mt/with-temp [:model/Database db {}
+ :model/Database other-db {}
+ :model/Table hidden {:db_id (:id db) :name "hidden_t" :visibility_type :hidden}
+ :model/Table cruft {:db_id (:id db) :name "cruft_t" :visibility_type :cruft}
+ :model/Table done {:db_id (:id db) :name "done_t"}
+ :model/Table inactive {:db_id (:id db) :name "inactive_t" :active false}
+ :model/Table foreign {:db_id (:id other-db) :name "foreign_t"}
+ :model/Field _ {:table_id (:id hidden) :name "a" :base_type :type/Text}
+ :model/Field _ {:table_id (:id cruft) :name "b" :base_type :type/Text}
+ :model/Field _ {:table_id (:id done) :name "c" :base_type :type/Text :data_sensitivity :PUBLIC}
+ :model/Field _ {:table_id (:id done) :name "d" :base_type :type/Text :data_sensitivity :PII}
+ :model/Field _ {:table_id (:id inactive) :name "e" :base_type :type/Text}
+ :model/Field _ {:table_id (:id foreign) :name "f" :base_type :type/Text}]
+ (testing "hidden and cruft tables with unscanned fields are selected; converged, inactive, and foreign tables are not"
+ (is (= #{(:id hidden) (:id cruft)}
+ (#'sync.data-sensitivity/table-ids-with-unscanned-fields db false))))
+ (testing "force reselects the table whose only unlabeled field is PUBLIC"
+ (is (= #{(:id hidden) (:id cruft) (:id done)}
+ (#'sync.data-sensitivity/table-ids-with-unscanned-fields db true))))))
+
+(deftest scan-fields-for-db-converges-test
+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "app_users" :entity_type :entity/UserTable}
+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}
+ :model/Field foo {:table_id (:id table) :name "foo" :base_type :type/Text}]
+ (testing "the first scan labels every unscanned field, PUBLIC when nothing matches, without creating mirror rows"
+ (is (= {:fields-scanned 2 :fields-labeled 1 :fields-failed 0}
+ (sync.data-sensitivity/scan-fields-for-db! db nil)))
+ (is (= :PII (label ssn)))
+ (is (= :PUBLIC (label foo)))
+ (is (nil? (mirror-label ssn)))
+ (is (nil? (mirror-label foo))))
+ (testing "a second scan selects nothing"
+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}
+ (sync.data-sensitivity/scan-fields-for-db! db nil)))))))
+
+(deftest setting-gates-scans-test
+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled false]
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "app_users"}
+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}]
+ (testing "with the setting off both entry points return zero stats and write nothing"
+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}
+ (sync.data-sensitivity/scan-fields-for-db! db nil)))
+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}
+ (sync.data-sensitivity/scan-table! table)))
+ (is (nil? (label ssn))))
+ (testing "the REPL entry point ignores the setting"
+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0}
+ (sync.data-sensitivity/scan-data-sensitivity! db)))
+ (is (= :PII (label ssn)))))))
+
+(deftest force-rescans-public-only-test
+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "app_users"}
+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}
+ :model/Field foo {:table_id (:id table) :name "foo" :base_type :type/Text}]
+ (sync.data-sensitivity/scan-fields-for-db! db nil)
+ (is (= :PUBLIC (label foo)))
+ (with-redefs [analyze/infer-data-sensitivity (constantly :PHI)]
+ (testing "a scheduled scan does not revisit PUBLIC fields even when the rules would now match"
+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}
+ (sync.data-sensitivity/scan-fields-for-db! db nil)))
+ (is (= :PUBLIC (label foo))))
+ (testing "a forced scan revisits PUBLIC fields and leaves categorized fields alone"
+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0}
+ (sync.data-sensitivity/scan-data-sensitivity! db :force? true)))
+ (is (= :PHI (label foo)))
+ (is (= :PII (label ssn))))))))
+
+(deftest user-label-in-mirror-wins-test
+ (testing "a mirror-only user label is what the field reads after the classifier writes"
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "app_users"}
+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}
+ :model/Field notes {:table_id (:id table) :name "notes" :base_type :type/Text}]
+ (field-user-settings/upsert-user-settings ssn {:data_sensitivity :PUBLIC})
+ (field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})
+ (is (nil? (label ssn)))
+ (is (nil? (label notes)))
+ (testing "stats count the rule result, the overlay decides what is stored"
+ (is (= {:fields-scanned 2 :fields-labeled 1 :fields-failed 0}
+ (sync.data-sensitivity/scan-data-sensitivity! db))))
+ (is (= :PUBLIC (label ssn)))
+ (is (= :PUBLIC (mirror-label ssn)))
+ (is (= :PHI (label notes)))
+ (is (= :PHI (mirror-label notes)))
+ (testing "a forced rescan still cannot override the user's PUBLIC"
+ (sync.data-sensitivity/scan-data-sensitivity! db :force? true)
+ (is (= :PUBLIC (label ssn)))))))
+
+(deftest reset-data-sensitivity-test
+ (mt/with-temp [:model/Database db {}
+ :model/Database other-db {}
+ :model/Table users {:db_id (:id db) :name "app_users"}
+ :model/Table orders {:db_id (:id db) :name "orders"}
+ :model/Table foreign {:db_id (:id other-db) :name "app_users"}
+ :model/Field ssn {:table_id (:id users) :name "ssn" :base_type :type/Text}
+ :model/Field foo {:table_id (:id users) :name "foo" :base_type :type/Text}
+ :model/Field notes {:table_id (:id users) :name "notes" :base_type :type/Text}
+ :model/Field email {:table_id (:id users) :name "email" :base_type :type/Text}
+ :model/Field total {:table_id (:id orders) :name "total" :base_type :type/Float}
+ :model/Field far {:table_id (:id foreign) :name "ssn" :base_type :type/Text}]
+ (field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})
+ (t2/insert! :model/FieldUserSettings {:field_id (:id email)})
+ (sync.data-sensitivity/scan-data-sensitivity! db)
+ (sync.data-sensitivity/scan-data-sensitivity! other-db)
+ (is (= [:PII :PUBLIC :PHI :PII :PUBLIC :PII]
+ (map label [ssn foo notes email total far])))
+ (testing "a table scope clears only that table's classifier labels"
+ (is (= 1 (sync.data-sensitivity/reset-data-sensitivity! orders)))
+ (is (nil? (label total)))
+ (is (= :PII (label ssn))))
+ (testing "a database scope clears categories and PUBLIC alike, including fields with a label-less mirror row"
+ (is (= 3 (sync.data-sensitivity/reset-data-sensitivity! db)))
+ (is (= [nil nil nil] (map label [ssn foo email]))))
+ (testing "a label backed by the mirror is human-set and survives"
+ (is (= :PHI (label notes)))
+ (is (= :PHI (mirror-label notes))))
+ (testing "other databases are untouched"
+ (is (= :PII (label far))))
+ (testing "a second reset finds nothing"
+ (is (zero? (sync.data-sensitivity/reset-data-sensitivity! db))))))
+
+(deftest scan-with-reset-relabels-categorized-fields-test
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "app_users"}
+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}
+ :model/Field notes {:table_id (:id table) :name "notes" :base_type :type/Text}]
+ (field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})
+ (sync.data-sensitivity/scan-data-sensitivity! db)
+ (is (= :PII (label ssn)))
+ (with-redefs [analyze/infer-data-sensitivity (constantly :SEC_KEY)]
+ (testing "a forced scan leaves a categorized field on its old label"
+ (sync.data-sensitivity/scan-data-sensitivity! db :force? true)
+ (is (= :PII (label ssn))))
+ (testing "a reset scan recomputes it under the current rules and reports the reset count"
+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0 :fields-reset 1}
+ (sync.data-sensitivity/scan-data-sensitivity! db :reset? true)))
+ (is (= :SEC_KEY (label ssn))))
+ (testing "the user's label is neither reset nor rescanned"
+ (is (= :PHI (label notes)))
+ (is (= :PHI (mirror-label notes)))))))
+
+(deftest classification-failure-is-counted-and-retried-test
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "app_users"}
+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}]
+ (let [calls (atom 0)]
+ (with-redefs [analyze/infer-data-sensitivity (fn [& _] (swap! calls inc) (throw (ex-info "boom" {})))]
+ (testing "a throwing rule counts as scanned and failed and leaves the field unscanned"
+ (is (= {:fields-scanned 1 :fields-labeled 0 :fields-failed 1}
+ (sync.data-sensitivity/scan-data-sensitivity! db)))
+ (is (nil? (label ssn))))
+ (testing "the next scan retries the field"
+ (sync.data-sensitivity/scan-data-sensitivity! db)
+ (is (= 2 @calls)))))))
+
+(deftest sync-database-end-to-end-test
+ (testing "a full sync with the setting on labels every field of every table using this run's inferred table context"
+ (mt/with-temp-test-data [["app_users"
+ [{:field-name "ssn", :base-type :type/Text}
+ {:field-name "email", :base-type :type/Text}
+ {:field-name "card_number", :base-type :type/Text}
+ {:field-name "city", :base-type :type/Text}
+ {:field-name "notes", :base-type :type/Text}
+ {:field-name "foo", :base-type :type/Text}]
+ [["123-45-6789" "[email protected]" "4111111111111111" "Berlin" "called back twice" "x"]]]
+ ["products"
+ [{:field-name "title", :base-type :type/Text}
+ {:field-name "city", :base-type :type/Text}
+ {:field-name "price", :base-type :type/Float}]
+ [["Widget" "Berlin" 9.99]]]]
+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]
+ (sync/sync-database! (mt/db)))
+ (let [labels (fn [table]
+ (into {} (map (juxt (comp u/lower-case-en :name) :data_sensitivity))
+ (t2/select [:model/Field :name :data_sensitivity] :table_id (mt/id table))))]
+ (testing "app_users is a UserTable, so the gated city rule fires alongside the explicit name rules"
+ (is (= :entity/UserTable (t2/select-one-fn :entity_type :model/Table :id (mt/id :app_users))))
+ (is (= {"id" :PUBLIC
+ "ssn" :PII
+ "email" :PII
+ "card_number" :PCI_FIN
+ "city" :PII
+ "notes" :PUBLIC
+ "foo" :PUBLIC}
+ (labels :app_users))))
+ (testing "products is not a UserTable, so city stays PUBLIC"
+ (is (= {"id" :PUBLIC "title" :PUBLIC "city" :PUBLIC "price" :PUBLIC}
+ (labels :products))))
+ (testing "no mirror rows were created"
+ (is (zero? (t2/count :model/FieldUserSettings
+ :field_id [:in (t2/select-pks-set :model/Field :table_id [:in [(mt/id :app_users) (mt/id :products)]])]))))
+ (testing "the analyze step recorded its counts in task_history"
+ (is (=? {:status :success
+ :task_details {:fields-scanned 11 :fields-labeled 4 :fields-failed 0}}
+ (t2/select-one :model/TaskHistory :db_id (mt/id) :task "classify-data-sensitivity"
+ {:order-by [[:id :desc]]}))))))))
+
+(mt/defdataset inet-columns
+ [["connections"
+ [{:field-name "client_ip", :base-type {:native "inet"}, :effective-type :type/IPAddress}
+ {:field-name "addr", :base-type {:native "inet"}, :effective-type :type/IPAddress}]
+ [[[:raw "'192.168.1.1'::inet"] [:raw "'10.4.4.15'::inet"]]]]])
+
+(deftest scan-postgres-inet-columns-test
+ (mt/test-driver :postgres
+ (testing "inet columns sync as :type/IPAddress base and semantic type and are labeled SYS_TELEMETRY whatever their name"
+ (mt/dataset inet-columns
+ (let [field-ids (t2/select-pks-set :model/Field :table_id (mt/id :connections) :name [:in ["client_ip" "addr"]])]
+ (t2/update! :model/Field :id [:in field-ids] {:data_sensitivity nil})
+ (is (= #{[:type/IPAddress :type/IPAddress]}
+ (t2/select-fn-set (juxt :base_type :semantic_type) :model/Field :id [:in field-ids])))
+ (sync/scan-data-sensitivity! (mt/db))
+ (is (= {"client_ip" :SYS_TELEMETRY "addr" :SYS_TELEMETRY}
+ (t2/select-fn->fn :name :data_sensitivity :model/Field :id [:in field-ids]))))))))
+
+(deftest scan-non-text-base-types-test
+ (testing "IPAddress and JSON base types with no semantic type, as ClickHouse and MySQL sync them, are labeled from the base type"
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db) :name "connections"}
+ :model/Field ip {:table_id (:id table) :name "client_ip" :base_type :type/IPAddress}
+ :model/Field addr {:table_id (:id table) :name "addr" :base_type :type/IPAddress}
+ :model/Field creds {:table_id (:id table) :name "credentials" :base_type :type/JSON}
+ :model/Field meta {:table_id (:id table) :name "metadata" :base_type :type/JSON}]
+ (is (= {:fields-scanned 4 :fields-labeled 3 :fields-failed 0}
+ (sync.data-sensitivity/scan-data-sensitivity! table)))
+ (is (= :SYS_TELEMETRY (label ip)))
+ (is (= :SYS_TELEMETRY (label addr)))
+ (is (= :SEC_KEY (label creds)))
+ (is (= :PUBLIC (label meta))))))
+
+(deftest scan-single-table-test
+ (mt/with-temp [:model/Database db {}
+ :model/Table users {:db_id (:id db) :name "app_users"}
+ :model/Table other {:db_id (:id db) :name "other"}
+ :model/Field ssn {:table_id (:id users) :name "ssn" :base_type :type/Text}
+ :model/Field email {:table_id (:id other) :name "email" :base_type :type/Text}]
+ (testing "scanning a Table instance labels only that table's fields"
+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0}
+ (sync.data-sensitivity/scan-data-sensitivity! users)))
+ (is (= :PII (label ssn)))
+ (is (nil? (label email))))))
diff --git a/test/metabase/sync/analyze_test.clj b/test/metabase/sync/analyze_test.clj
index 2f2dcdb3d2f..d55a10c2c1a 100644
--- a/test/metabase/sync/analyze_test.clj
+++ b/test/metabase/sync/analyze_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase.analyze.classifiers.category :as classifiers.category]
+ [metabase.analyze.classifiers.data-sensitivity :as classifiers.data-sensitivity]
[metabase.analyze.classifiers.name :as classifiers.name]
[metabase.analyze.classifiers.no-preview-display
:as classifiers.no-preview-display]
@@ -99,6 +100,11 @@
(testing "Make sure we survive table classification failing"
(sync-survives-crash?! classifiers.name/infer-entity-type-by-name)))
+(deftest survive-data-sensitivity-errors
+ (testing "sync survives the data sensitivity rules failing; the setting must be on for the rules to be reached"
+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]
+ (sync-survives-crash?! classifiers.data-sensitivity/infer-data-sensitivity))))
+
(defn- classified-semantic-type [values]
(let [field (mi/instance :model/Field {:base_type :type/Text})]
(:semantic_type (classifiers.text-fingerprint/infer-semantic-type
@@ -227,7 +233,8 @@
:model/Field _ (fake-field table)]
(let [results (analyze-table! table)]
(testing "has the steps performed"
- (is (= ["fingerprint-fields" "classify-fields" "classify-tables" "score-interestingness"]
+ (is (= ["fingerprint-fields" "classify-fields" "classify-tables" "score-interestingness"
+ "classify-data-sensitivity"]
(->> results :steps (map first)))))
(testing "has start and finish times"
(is (seq (select-keys results [:start-time :end-time]))))))))
diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.clj
index 3251391bbe2..ea9e5e402ae 100644
--- a/test/metabase/sync/sync_metadata/fields_test.clj
+++ b/test/metabase/sync/sync_metadata/fields_test.clj
@@ -614,6 +614,28 @@
(is (= :PII (field)))
(is (= :PII (mirror))))))))
+(deftest data-sensitivity-classifier-respects-user-label-test
+ (testing "a user-set data_sensitivity wins over the classifier's inference while sibling fields get labeled"
+ (mt/with-temp-test-data [["app_users"
+ [{:field-name "email", :base-type :type/Text}
+ {:field-name "ssn", :base-type :type/Text}
+ {:field-name "notes", :base-type :type/Text}]
+ [["[email protected]" "123-45-6789" "called back twice"]]]]
+ (let [field #(t2/select-one-fn :data_sensitivity :model/Field :id (mt/id :app_users %))
+ mirror #(t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id (mt/id :app_users %))]
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" (mt/id :app_users :email)) {:data_sensitivity "PUBLIC"})
+ (is (nil? (field :ssn)))
+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]
+ (sync/sync-database! (mt/db)))
+ (testing "the user's PUBLIC on email survives even though the rules say PII"
+ (is (= :PUBLIC (field :email)))
+ (is (= :PUBLIC (mirror :email))))
+ (testing "the classifier labels the unlabeled siblings without creating mirror rows"
+ (is (= :PII (field :ssn)))
+ (is (= :PUBLIC (field :notes)))
+ (is (nil? (mirror :ssn)))
+ (is (nil? (mirror :notes))))))))
+
(deftest data-sensitivity-survives-column-drop-and-readd-test
(testing "a user-set data_sensitivity survives the warehouse column being dropped and added back"
(mt/with-temp-test-data [["readd_table"
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 7a56771c15e..afd96d612e2 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -1748,6 +1748,28 @@
(is (pos? (t2/count :model/Table :db_id db-id))
"Sync-now must populate tables even when disable-auto-sync is on")))))))
+(deftest sync-schema-labels-data-sensitivity-test
+ (testing "POST /api/database/:id/sync_schema runs the data sensitivity step when the setting is on"
+ ;; Same shape as the test above: create the Database under disable-auto-sync so only the explicit request syncs
+ ;; it, and run the quick task synchronously so the labels can be asserted after the request returns.
+ (let [details (:details (mt/db))]
+ (mt/with-temporary-setting-values [disable-auto-sync true
+ data-sensitivity-scan-enabled true]
+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2"
+ :details details
+ :initial_sync_status "incomplete"}]
+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f]
+ (binding [driver.settings/*allow-testing-h2-connections* true]
+ (f)))]
+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id)))
+ (let [label (fn [table-name field-name]
+ (t2/select-one-fn :data_sensitivity :model/Field
+ :table_id (t2/select-one-pk :model/Table :db_id db-id :name table-name)
+ :name field-name))]
+ (is (= :PII (label "PEOPLE" "EMAIL")))
+ (is (= :SEC_KEY (label "PEOPLE" "PASSWORD")))
+ (is (= :PUBLIC (label "ORDERS" "TOTAL")))))))))
+
(deftest sync-schema-executes-when-executor-busy-test
(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"
(let [sync-called? (promise)