The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GHY-4377] Data sensitivity table scanning (#81752)
Tyler Plude7d07cc9ac895
diff --git a/src/metabase/analyze/classifiers/data_sensitivity.clj b/src/metabase/analyze/classifiers/data_sensitivity.cljnew file mode 100644index 00000000000..8259ef62dca--- /dev/null+++ b/src/metabase/analyze/classifiers/data_sensitivity.clj@@ -0,0 +1,272 @@+(ns metabase.analyze.classifiers.data-sensitivity+ "Classifier that infers the `data_sensitivity` category of a Field from deterministic rules over its name, base+ type, semantic type, fingerprint, and the name and entity type of its Table. Pure: no app-DB access, no settings."+ (:require+ [clojure.string :as str]+ [metabase.config.core :as config]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.util :as u]+ [metabase.util.malli :as mu]))++;; `:type/IPAddress` (Postgres `inet`, ClickHouse `IPv4`/`IPv6`) and `:type/Structured` (JSON, XML, Postgres `macaddr`,+;; `cidr`) are base types that do not descend from `:type/Text`, so the text rules that should see them list them.+(def ^:private text-type #{:type/Text})+(def ^:private text-or-structured-type #{:type/Text :type/Structured})+(def ^:private telemetry-text-type #{:type/Text :type/IPAddress :type/Structured})+(def ^:private int-or-text-type #{:type/Integer :type/Text})+(def ^:private text-or-bool-type #{:type/Text :type/Boolean})+(def ^:private temporal-or-text-type #{:type/Temporal :type/Text})+(def ^:private temporal-int-or-text-type #{:type/Temporal :type/Integer :type/Text})+(def ^:private number-type #{:type/Number})+(def ^:private any-type #{:type/*})++(defn- name->tokens+ "Lowercased tokens of a physical column or table name, split on `_`, `-`, `.`, whitespace, and camelCase+ boundaries, plus every adjacent pair joined with `_`."+ [s]+ (let [tokens (->> (str/replace s #"(?<=[a-z0-9])(?=[A-Z])" "_")+ u/lower-case-en+ (#(str/split % #"[_\-.\s]+"))+ (remove str/blank?)+ vec)]+ {:tokens (set tokens)+ :pairs (set (map #(str %1 "_" %2) tokens (rest tokens)))}))++;; Tuples of `[#{token-or-pair ...} set-of-valid-base-types category]`. A field matches when any of its tokens or+;; adjacent-token pairs is in the set and its base type `isa?` one of the base types. Bare `key`, `token`, `hash`,+;; `name`, `address`, `city`, `state`, `zip`, `country`, `age` are deliberately absent: they collide with+;; `foreign_key`, `token_count`, `product_name`, `orders.state`; the semantic-type maps and table boosters below+;; cover them where the table supplies context.+(def ^:private token-rules+ [;; SEC_KEY: credentials, keys, tokens, and hashes that grant access.+ [#{"password" "passwd" "pwd" "passphrase" "password_hash" "pwd_hash" "salt" "pin" "pin_code"+ "secret" "secrets" "client_secret" "api_secret" "mfa_secret" "totp_secret" "webhook_secret" "signing_secret"+ "api_key" "apikey" "access_key" "secret_key" "private_key" "ssh_key" "pgp_key" "gpg_key" "signing_key"+ "encryption_key" "master_key" "license_key"+ "api_token" "access_token" "auth_token" "refresh_token" "session_token" "bearer_token" "csrf_token"+ "oauth_token" "id_token" "reset_token" "verification_token" "device_token" "bearer" "jwt"+ "otp" "totp" "credential" "credentials" "security_answer" "recovery_code" "recovery_codes" "backup_codes"+ "connection_string"}+ text-or-structured-type :SEC_KEY]+ ;; SYS_TELEMETRY: infrastructure identifiers and machine telemetry.+ [#{"ip" "ip_address" "ip_addr" "ipaddress" "ipv4" "ipv6" "client_ip" "remote_ip" "remote_addr" "source_ip"+ "src_ip" "dest_ip" "dst_ip" "server_ip" "host_ip" "mac_address" "mac_addr" "macaddress" "hostname" "host_name"+ "user_agent" "useragent" "device_fingerprint" "browser_fingerprint"}+ telemetry-text-type :SYS_TELEMETRY]+ [#{"device_id" "imei" "imsi" "iccid" "serial_number" "session_id" "trace_id" "span_id" "request_id"+ "correlation_id"}+ int-or-text-type :SYS_TELEMETRY]+ ;; PHI: health status, diagnoses, treatment, and insurance.+ [#{"diagnosis" "diagnoses" "diagnosis_code" "icd" "icd9" "icd10" "icd_code" "cpt" "cpt_code" "medication"+ "medications" "prescription" "prescriptions" "allergy" "allergies" "symptom" "symptoms" "immunization"+ "vaccination" "vaccine" "blood_type" "medical_history" "medical_condition" "health_condition" "mental_health"+ "therapy" "dosage" "lab_result" "lab_results" "treatment_plan" "clinical_notes" "chief_complaint" "hiv"+ "hiv_status" "patient" "medical_record"}+ text-or-structured-type :PHI]+ [#{"patient_id" "mrn" "npi" "insurance_id" "insurance_number" "rx_number" "rx_id"}+ int-or-text-type :PHI]+ [#{"bmi" "blood_pressure" "heart_rate" "glucose" "cholesterol"}+ any-type :PHI]+ ;; BIO_GEN: biometric and genetic identifiers.+ [#{"fingerprint" "fingerprints" "retina" "iris" "face_id" "faceprint" "voiceprint" "face_encoding"+ "face_embedding" "facial_recognition" "dna" "genome" "genotype" "genetic" "biometric" "biometrics"}+ any-type :BIO_GEN]+ ;; PCI_FIN: payment card and financial account data.+ [#{"card_number" "cardnumber" "card_num" "card_no" "cc_number" "cc_num" "ccnum" "ccn" "credit_card" "creditcard"+ "debit_card" "debitcard" "pan" "cvv" "cvc" "cvv2" "cvc2" "card_expiry" "card_expiration" "card_last4"+ "cardholder" "cardholder_name" "iban" "bic" "swift_code" "swift_bic" "routing_number" "routing_no"+ "aba_number" "sort_code" "account_number" "account_no" "acct_number" "acct_no" "bank_account"}+ int-or-text-type :PCI_FIN]+ ;; SENS_PERS: GDPR special categories (race, ethnicity, religion, politics, union, sexual orientation,+ ;; health-adjacent traits) plus gender and sex by policy (D14).+ [#{"race" "ethnicity" "ethnic_origin" "religion" "religious_affiliation" "political_affiliation"+ "political_party" "political_views" "union_member" "union_membership" "trade_union" "sexual_orientation"+ "sexuality" "gender" "gender_identity" "sex" "disability" "disabilities" "pregnancy" "pregnant"+ "veteran_status" "criminal_record" "criminal_history" "conviction" "convictions"}+ text-or-bool-type :SENS_PERS]+ ;; PII: direct personal identifiers and contact data. Government identifiers fire on any type.+ [#{"ssn" "social_security" "national_id" "national_identifier" "nin" "nino" "passport"+ "passport_number" "passport_no" "driver_license" "drivers_license" "driving_license" "driver_licence"+ "drivers_licence" "dl_number" "license_plate" "licence_plate" "plate_number" "vin" "tax_id" "taxpayer_id"+ "tin" "itin" "ein" "sin_number" "aadhaar" "aadhar"}+ any-type :PII]+ [#{"of_birth" "birth_date" "birthdate" "birthday" "birth_day" "birth_place" "birth_city" "birth_country"}+ temporal-or-text-type :PII]+ [#{"dob" "birth_year"}+ temporal-int-or-text-type :PII]+ [#{"first_name" "firstname" "last_name" "lastname" "full_name" "fullname" "surname" "given_name" "middle_name"+ "maiden_name" "family_name" "forename" "username" "user_name" "customer_name" "employee_name" "contact_name"+ "person_name" "recipient_name" "sender_name"+ "email" "emails" "email_address" "e_mail" "personal_email" "work_email" "contact_email"+ "home_address" "mailing_address" "street_address" "postal_address" "billing_address" "shipping_address"+ "residential_address" "address_line" "address_line1" "address_line2" "address1" "address2" "addr1" "addr2"}+ text-type :PII]+ [#{"phone" "telephone" "mobile_phone" "cell_phone" "cellphone" "home_phone" "work_phone" "contact_phone" "fax"}+ text-type :PII]+ [#{"phone_number" "phone_no" "phone_num" "mobile_number" "mobile_no" "cell_number" "contact_number" "fax_number"}+ int-or-text-type :PII]+ ;; CORP_IP: source code, designs, and proprietary algorithms.+ [#{"source_code" "sourcecode" "code_snippet" "repo_url" "repository_url" "private_repo" "patent"+ "patent_number" "design_doc" "design_document" "algorithm" "model_weights" "proprietary"+ "schematic" "schematics" "blueprint" "blueprints"}+ text-type :CORP_IP]+ ;; BIZ_CONF: internal financials and confidential business figures.+ [#{"salary" "salaries" "compensation" "wage" "wages" "bonus" "payroll" "pay_rate" "hourly_rate" "annual_income"+ "revenue" "profit" "margin" "gross_margin" "net_income" "operating_income" "ebitda" "cost_basis" "forecast"+ "budget" "contract_value" "deal_size" "deal_value"}+ number-type :BIZ_CONF]+ [#{"salary_band" "salary_grade" "salary_range" "pay_band" "pay_grade" "pay_scale" "compensation_band"+ "compensation_tier" "compensation_grade" "wage_band"}+ int-or-text-type :BIZ_CONF]])++;; Tuples of `[regex set-of-valid-base-types category]` matched with `re-find` against the whole lowercased name.+;; Reserved for true substrings that tokenizing misses (`ssn4`, `passportno`, `cardcvv`); keep short.+(def ^:private stem-rules+ [[#"passw" text-type :SEC_KEY]+ [#"secret(?!ar)" text-type :SEC_KEY]+ [#"cv[vc]" int-or-text-type :PCI_FIN]+ [#"iban" int-or-text-type :PCI_FIN]+ [#"(?<![a-z])ssn" any-type :PII]+ [#"passport" any-type :PII]+ [#"birthda" temporal-or-text-type :PII]+ [#"email" text-type :PII]])++;; Semantic types that imply a category regardless of table. Consulted with `isa?` so descendants match. Also+;; consulted against the base type when it is itself a semantic type (`:type/IPAddress` is both), since sync+;; never sets `semantic_type` for those columns.+(def ^:private semantic-type->category+ {:type/Email :PII+ :type/Birthdate :PII+ :type/IPAddress :SYS_TELEMETRY})++;; Semantic types that imply a category only when the table's `entity_type` is `:entity/UserTable`.+(def ^:private user-table-semantic-type->category+ {:type/Name :PII+ :type/Address :PII+ :type/City :PII+ :type/State :PII+ :type/Country :PII+ :type/ZipCode :PII+ :type/Coordinate :PII+ :type/User :PII})++;; Tuples of `[#{field-token ...} #{table-token ...} category]`. Promotes a weak field signal when the table name+;; supplies the context.+(def ^:private table-boosters+ [[#{"token" "tokens" "key" "keys" "hash" "secret"}+ #{"session" "sessions" "auth" "oauth" "credential" "credentials" "token" "tokens" "key" "keys" "secret" "secrets"}+ :SEC_KEY]+ [#{"notes" "comments" "remarks" "condition" "conditions" "treatment" "treatments" "procedure" "procedures"+ "result" "results" "history" "record" "records"}+ #{"patient" "patients" "medical" "clinical" "health" "hospital" "encounter" "encounters" "diagnosis"+ "prescription" "prescriptions"}+ :PHI]+ [#{"template" "templates" "sample" "samples" "scan" "scans" "data" "vector" "embedding"}+ #{"biometric" "biometrics" "fingerprint" "fingerprints" "facial" "voice" "iris" "retina" "dna" "genetic"+ "genomic" "genome"}+ :BIO_GEN]+ [#{"number" "num" "no" "expiry" "expiration" "exp" "last4" "holder"}+ #{"card" "cards" "credit" "bank"}+ :PCI_FIN]+ [#{"name" "address" "age" "street" "lat" "lon" "lng" "latitude" "longitude" "zipcode" "postcode"}+ #{"user" "users" "person" "people" "customer" "customers" "employee" "employees" "member" "members" "contact"+ "contacts" "patient" "patients" "profile" "profiles" "subscriber" "subscribers" "lead" "leads" "applicant"+ "applicants" "student" "students" "candidate" "candidates"}+ :PII]+ [#{"amount" "value" "total"}+ #{"salary" "salaries" "payroll" "compensation" "deal" "deals" "contract" "contracts" "forecast" "forecasts"+ "budget" "budgets"}+ :BIZ_CONF]])++(def ^:private categories+ (set lib.schema.metadata/column-data-sensitivity-types))++(def ^:private precedence+ (zipmap lib.schema.metadata/column-data-sensitivity-types (range)))++(when-not config/is-prod?+ (let [valid-category? #(and (contains? categories %) (not= :PUBLIC %))+ valid-base-types? #(and (set? %) (seq %) (every? (fn [t] (isa? t :type/*)) %))+ valid-token? #(re-matches #"[a-z0-9]+(?:_[a-z0-9]+)?" %)]+ (doseq [[tokens base-types category] token-rules]+ (assert (and (set? tokens) (every? valid-token? tokens)) (pr-str tokens))+ (assert (valid-base-types? base-types) (pr-str base-types))+ (assert (valid-category? category) (pr-str category)))+ (doseq [[pattern base-types category] stem-rules]+ (assert (instance? java.util.regex.Pattern pattern) (pr-str pattern))+ (assert (valid-base-types? base-types) (pr-str base-types))+ (assert (valid-category? category) (pr-str category)))+ (doseq [[semantic-type category] (concat semantic-type->category user-table-semantic-type->category)]+ (assert (isa? semantic-type :Semantic/*) (pr-str semantic-type))+ (assert (valid-category? category) (pr-str category)))+ (doseq [[field-tokens table-tokens category] table-boosters]+ (assert (every? valid-token? field-tokens) (pr-str field-tokens))+ (assert (every? valid-token? table-tokens) (pr-str table-tokens))+ (assert (valid-category? category) (pr-str category)))))++(defn- base-type-matches? [base-type base-types]+ (some (partial isa? base-type) base-types))++(defn- token-matches [{:keys [tokens pairs]} base-type]+ (let [names (into tokens pairs)]+ (for [[rule-names base-types category] token-rules+ :when (and (some rule-names names)+ (base-type-matches? base-type base-types))]+ category)))++(defn- stem-matches [lower-name base-type]+ (for [[pattern base-types category] stem-rules+ :when (and (base-type-matches? base-type base-types)+ (re-find pattern lower-name))]+ category))++(defn- semantic-matches [semantic-type base-type entity-type]+ (let [types (cond-> #{}+ semantic-type (conj semantic-type)+ (isa? base-type :Semantic/*) (conj base-type))]+ (for [[st category] (cond-> semantic-type->category+ (= :entity/UserTable entity-type) (merge user-table-semantic-type->category))+ t types+ :when (isa? t st)]+ category)))++(defn- fingerprint-matches [fingerprint base-type]+ (when (and (isa? base-type :type/Text)+ (>= (get-in fingerprint [:type :type/Text :percent-email] 0) 0.95))+ [:PII]))++(defn- booster-matches [{:keys [tokens]} table-name]+ (when table-name+ (let [table-tokens (:tokens (name->tokens table-name))]+ (for [[field-tokens rule-table-tokens category] table-boosters+ :when (and (some field-tokens tokens)+ (some rule-table-tokens table-tokens))]+ category))))++(def ^:private Field+ [:map+ [:name :string]+ [:base_type :keyword]+ [:semantic_type {:optional true} [:maybe :keyword]]+ [:fingerprint {:optional true} :any]])++(def ^:private TableContext+ [:maybe+ [:map+ [:name {:optional true} [:maybe :string]]+ [:entity_type {:optional true} [:maybe :keyword]]]])++(mu/defn infer-data-sensitivity :- [:maybe ::lib.schema.metadata/column.data-sensitivity]+ "Infer the `data_sensitivity` category of `field` from its name, base type, semantic type, and fingerprint, with+ `table-context` (`:name`, `:entity_type` of its Table) gating the weaker rules. Returns the highest-precedence+ matching category, or nil when no rule matches. Never returns `:PUBLIC`."+ [{field-name :name, :keys [base_type semantic_type fingerprint]} :- Field+ {table-name :name, :keys [entity_type]} :- TableContext]+ (let [tokens (name->tokens field-name)]+ (->> (concat (token-matches tokens base_type)+ (stem-matches (u/lower-case-en field-name) base_type)+ (semantic-matches semantic_type base_type entity_type)+ (fingerprint-matches fingerprint base_type)+ (booster-matches tokens table-name))+ (sort-by precedence)+ first)))diff --git a/src/metabase/analyze/core.clj b/src/metabase/analyze/core.cljindex 15a722608e2..179005c92c2 100644--- a/src/metabase/analyze/core.clj+++ b/src/metabase/analyze/core.clj@@ -3,6 +3,7 @@(:require[metabase.analyze.classifiers.category][metabase.analyze.classifiers.core]+ [metabase.analyze.classifiers.data-sensitivity][metabase.analyze.classifiers.name][metabase.analyze.fingerprint.fingerprinters][metabase.analyze.query-results]@@ -11,6 +12,7 @@(commentmetabase.analyze.classifiers.category/keep-memetabase.analyze.classifiers.core/keep-me+ metabase.analyze.classifiers.data-sensitivity/keep-memetabase.analyze.classifiers.name/keep-memetabase.analyze.fingerprint.fingerprinters/keep-memetabase.analyze.query-results/keep-me)@@ -21,6 +23,8 @@category-cardinality-threshold][metabase.analyze.classifiers.corerun-classifiers]+ [metabase.analyze.classifiers.data-sensitivity+ infer-data-sensitivity][metabase.analyze.classifiers.nameinfer-entity-type-by-nameinfer-semantic-type-by-name]diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex c12b9a5a692..0b76470740b 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -310,8 +310,9 @@(def column-data-sensitivity-types"Possible values for column `:data-sensitivity`, most severe first. The order is the precedence an automated- classifier uses when several categories match a single column. `nil` means the column has never been classified;- `:PUBLIC` means it was reviewed and found not sensitive."+ classifier uses when several categories match a single column. `nil` means the column has never been scanned or+ labeled; `:PUBLIC` means it was scanned or reviewed and nothing sensitive was found. Whether a person made the call+ is recorded by the presence of a value in `metabase_field_user_settings`, not by the value itself."[:SEC_KEY ; Security Credentials & Secrets:SYS_TELEMETRY ; Infrastructure & System Secrets:PHI ; Protected Health Informationdiff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.cljindex e40eb4490db..f4e9ec2bfa9 100644--- a/src/metabase/sync/analyze.clj+++ b/src/metabase/sync/analyze.clj@@ -5,6 +5,7 @@and infer field semantic types."(:require[metabase.sync.analyze.classify :as classify]+ [metabase.sync.analyze.data-sensitivity :as sync.data-sensitivity][metabase.sync.analyze.fingerprint :as sync.fingerprint][metabase.sync.analyze.interestingness :as sync.interestingness][metabase.sync.db :as sync.db]@@ -69,6 +70,7 @@(classify/classify-fields! table)(classify/classify-table! table)(sync.interestingness/score-fields! table)+ (sync.data-sensitivity/scan-table! table)(update-fields-last-analyzed! table))(defn- maybe-log-progress [progress-bar-fn]@@ -93,6 +95,10 @@(format "Interestingness scored %d fields, %d failed"fields-scored fields-failed))+(defn- data-sensitivity-summary [{:keys [fields-scanned fields-labeled fields-failed]}]+ (format "Data sensitivity scanned %d fields, labeled %d, %d failed"+ fields-scanned fields-labeled fields-failed))+(defn- make-analyze-steps [log-fn][(sync-util/create-sync-step "fingerprint-fields"#(sync.fingerprint/fingerprint-fields-for-db! % log-fn)@@ -105,7 +111,10 @@classify-tables-summary)(sync-util/create-sync-step "score-interestingness"#(sync.interestingness/score-fields-for-db! % log-fn)- interestingness-summary)])+ interestingness-summary)+ (sync-util/create-sync-step "classify-data-sensitivity"+ #(sync.data-sensitivity/scan-fields-for-db! % log-fn)+ data-sensitivity-summary)])(mu/defn- analyze-db!*"Shared core of [[analyze-db!]] and [[analyze-db-explicit!]]: the analysis work, without thediff --git a/src/metabase/sync/analyze/data_sensitivity.clj b/src/metabase/sync/analyze/data_sensitivity.cljnew file mode 100644index 00000000000..55503fdfa3d--- /dev/null+++ b/src/metabase/sync/analyze/data_sensitivity.clj@@ -0,0 +1,165 @@+(ns metabase.sync.analyze.data-sensitivity+ "Analyze sub-step that labels Fields with a `data_sensitivity` category from+ [[metabase.analyze.core/infer-data-sensitivity]], writing `:PUBLIC` when no rule matches so each field is scanned+ once. Selects fields whose `data_sensitivity` is `NULL` (or `PUBLIC` too under `force?`) across every active table+ in the database, hidden and cruft tables included. Inert unless [[metabase.sync.settings/data-sensitivity-scan-enabled]]+ is true, except through [[scan-data-sensitivity!]]. User-set labels are protected by the `FieldUserSettings`+ overlay applied on every Field update. [[reset-data-sensitivity!]] clears the classifier's own labels so a rule+ change can reach fields that already carry a category."+ (:require+ [metabase.analyze.core :as analyze]+ [metabase.sync.interface :as i]+ [metabase.sync.settings :as sync.settings]+ [metabase.sync.util :as sync-util]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [toucan2.core :as t2]+ [toucan2.realize :as t2.realize]))++(set! *warn-on-reflection* true)++(def ^:private Stats+ [:map+ [:fields-scanned :int]+ [:fields-labeled :int]+ [:fields-failed :int]+ [:fields-reset {:optional true} :int]])++(def ^:private zero-stats+ {:fields-scanned 0 :fields-labeled 0 :fields-failed 0})++(def ^:private ScanOptions+ [:map+ [:force? {:optional true} [:maybe :boolean]]+ [:ignore-setting? {:optional true} [:maybe :boolean]]])++(defn- selection-clause [force?]+ (if force?+ [:or [:= :data_sensitivity nil] [:= :data_sensitivity "PUBLIC"]]+ [:= :data_sensitivity nil]))++(defn- skip? [ignore-setting?]+ (and (not ignore-setting?)+ (not (sync.settings/data-sensitivity-scan-enabled))))++(mu/defn- fields-to-scan :- [:sequential i/FieldInstance]+ [table :- i/TableInstance+ force?]+ (t2/select :model/Field+ {:where [:and+ [:= :table_id (u/the-id table)]+ [:= :active true]+ [:not= :visibility_type "retired"]+ (selection-clause force?)]+ :order-by [[:id :asc]]}))++(mu/defn- classify-and-save!+ "Returns the inferred category, nil when no rule matched (in which case `:PUBLIC` was written), or the Exception."+ [field :- i/FieldInstance+ table :- i/TableInstance]+ (sync-util/with-error-handling (format "Error classifying data sensitivity for %s" (sync-util/name-for-logging field))+ (let [category (analyze/infer-data-sensitivity field {:name (:name table) :entity_type (:entity_type table)})]+ (t2/update! :model/Field (u/the-id field) {:data_sensitivity (or category :PUBLIC)})+ category)))++(mu/defn scan-table! :- Stats+ "Label every unscanned Field in `table`. `fields-scanned` counts every selected field, `fields-labeled` those that+ matched a category, `fields-failed` those whose classification threw; the remainder were written `:PUBLIC`."+ [table :- i/TableInstance+ & {:keys [force? ignore-setting?]} :- [:maybe ScanOptions]]+ (if (skip? ignore-setting?)+ zero-stats+ (let [stats (reduce (fn [stats field]+ (let [result (classify-and-save! field table)]+ (log/debugf "Data sensitivity for %s: %s" (sync-util/name-for-logging field)+ (if (instance? Exception result) "failed" (or result :PUBLIC)))+ (cond-> (update stats :fields-scanned inc)+ (instance? Exception result) (update :fields-failed inc)+ (keyword? result) (update :fields-labeled inc))))+ zero-stats+ (fields-to-scan table force?))]+ (when (pos? (:fields-scanned stats))+ (log/infof "Data sensitivity scanned %d fields in %s: %d labeled, %d failed"+ (:fields-scanned stats) (sync-util/name-for-logging table)+ (:fields-labeled stats) (:fields-failed stats)))+ stats)))++(mu/defn- table-ids-with-unscanned-fields :- [:maybe [:set pos-int?]]+ [database :- i/DatabaseInstance+ force?]+ (t2/select-fn-set :table_id :model/Field+ {:select [[:metabase_field.table_id :table_id]]+ :from [:metabase_field]+ :join [[:metabase_table] [:= :metabase_field.table_id :metabase_table.id]]+ :where [:and+ [:= :metabase_table.db_id (u/the-id database)]+ [:= :metabase_table.active true]+ [:= :metabase_field.active true]+ [:not= :metabase_field.visibility_type "retired"]+ (selection-clause force?)]+ :group-by [:metabase_field.table_id]}))++(mu/defn scan-fields-for-db! :- Stats+ "Label every unscanned Field in every active table of `database`. `log-fn` is accepted for parity with the other+ analyze steps and not called: the step reports per table through the log, not the progress bar."+ [database :- i/DatabaseInstance+ _log-fn+ & {:keys [force? ignore-setting?]} :- [:maybe ScanOptions]]+ (if (skip? ignore-setting?)+ zero-stats+ (let [table-ids (table-ids-with-unscanned-fields database force?)]+ (if (empty? table-ids)+ zero-stats+ (transduce (comp (map t2.realize/realize)+ (map #(scan-table! % :force? force? :ignore-setting? true)))+ (partial merge-with +)+ zero-stats+ (t2/reducible-select :model/Table+ :id [:in table-ids]+ {:order-by [[:schema :asc] [:name :asc]]}))))))++(defn- scope-clause [database-or-table]+ (case (t2/model database-or-table)+ :model/Table [:= :table_id (u/the-id database-or-table)]+ :model/Database [:in :table_id ^:allow-subquery {:select [:id]+ :from [:metabase_table]+ :where [:= :db_id (u/the-id database-or-table)]}]))++(def ^:private classifier-label-clause+ "A non-null `data_sensitivity` with no value in the `FieldUserSettings` mirror was written by the classifier."+ [:and+ [:not= :data_sensitivity nil]+ [:not [:exists ^:allow-subquery {:select [1]+ :from [[:metabase_field_user_settings :s]]+ :where [:and+ [:= :s.field_id :metabase_field.id]+ [:not= :s.data_sensitivity nil]]}]]])++(mu/defn reset-data-sensitivity! :- :int+ "REPL entry point: clear every classifier-written `data_sensitivity` in a Database or a single Table so the next+ scan recomputes them, including fields that carry a category and would otherwise never be reselected. Labels with+ a value in the `FieldUserSettings` mirror are human-set and untouched. Returns the number of fields cleared."+ [database-or-table :- [:or i/DatabaseInstance i/TableInstance]]+ (let [n (t2/query-one {:update :metabase_field+ :set {:data_sensitivity nil}+ :where [:and (scope-clause database-or-table) classifier-label-clause]})]+ (log/infof "Data sensitivity reset %d classifier-written labels in %s" n (sync-util/name-for-logging database-or-table))+ n))++(mu/defn scan-data-sensitivity! :- Stats+ "REPL entry point: scan a Database or a single Table regardless of the `data-sensitivity-scan-enabled` setting.+ With `:force? true` fields already labeled `:PUBLIC` are rescanned too; fields carrying a category are not. With+ `:reset? true` every classifier-written label is cleared first via [[reset-data-sensitivity!]] so the whole scope+ is recomputed under the current rules, and `:fields-reset` is added to the stats. User-set labels survive in+ every mode because the `FieldUserSettings` overlay is applied on every Field update."+ [database-or-table :- [:or i/DatabaseInstance i/TableInstance]+ & {:keys [force? reset?]} :- [:maybe [:map+ [:force? {:optional true} [:maybe :boolean]]+ [:reset? {:optional true} [:maybe :boolean]]]]]+ (let [reset (when reset? (reset-data-sensitivity! database-or-table))+ stats (case (t2/model database-or-table)+ :model/Table (scan-table! database-or-table :force? force? :ignore-setting? true)+ :model/Database (scan-fields-for-db! database-or-table (constantly nil) :force? force? :ignore-setting? true))]+ (cond-> stats+ reset (assoc :fields-reset reset))))diff --git a/src/metabase/sync/core.clj b/src/metabase/sync/core.cljindex 4617e6f4833..c51ead83195 100644--- a/src/metabase/sync/core.clj+++ b/src/metabase/sync/core.clj@@ -3,6 +3,7 @@-- Tables and Fields."(:require[metabase.sync.analyze]+ [metabase.sync.analyze.data-sensitivity][metabase.sync.field-values][metabase.sync.sync][metabase.sync.sync-metadata]@@ -12,6 +13,7 @@(commentmetabase.sync.analyze/keep-me+ metabase.sync.analyze.data-sensitivity/keep-memetabase.sync.field-values/keep-memetabase.sync.sync/keep-memetabase.sync.sync-metadata/keep-me@@ -22,6 +24,9 @@[metabase.sync.analyzeanalyze-db!analyze-db-explicit!]+ [metabase.sync.analyze.data-sensitivity+ reset-data-sensitivity!+ scan-data-sensitivity!][metabase.sync.field-valuesupdate-field-values!update-field-values-for-table!]diff --git a/src/metabase/sync/settings.clj b/src/metabase/sync/settings.cljindex feafda677c7..fdec3a5030d 100644--- a/src/metabase/sync/settings.clj+++ b/src/metabase/sync/settings.clj@@ -57,3 +57,15 @@:export? true:type :integer:default 10000)++(defsetting data-sensitivity-scan-enabled+ "When true, the analyze phase of sync labels every unlabeled field with a data_sensitivity category inferred from+ its name, types, and fingerprint, writing PUBLIC when nothing matches. Metadata only: the label does not mask or+ restrict anything. Labels set by a user are never overwritten."+ :type :boolean+ :default false+ :setter :none+ :visibility :internal+ :export? false+ :doc "Scans run with the scheduled analyze pass and with Sync database schema now on a database's admin+ page. Fields that already carry a label, including PUBLIC, are not rescanned.")diff --git a/test/metabase/analyze/classifiers/data_sensitivity_test.clj b/test/metabase/analyze/classifiers/data_sensitivity_test.cljnew file mode 100644index 00000000000..163ddd617ca--- /dev/null+++ b/test/metabase/analyze/classifiers/data_sensitivity_test.clj@@ -0,0 +1,383 @@+(ns metabase.analyze.classifiers.data-sensitivity-test+ (:require+ [clojure.test :refer :all]+ [metabase.analyze.classifiers.data-sensitivity :as classifiers.data-sensitivity]+ [metabase.lib.schema.metadata :as lib.schema.metadata]))++(defn- infer+ ([field-name base-type]+ (infer field-name base-type nil nil))+ ([field-name base-type semantic-type table-context]+ (classifiers.data-sensitivity/infer-data-sensitivity+ {:name field-name :base_type base-type :semantic_type semantic-type}+ table-context)))++(deftest ^:parallel name->tokens-test+ (testing "separators and camelCase all tokenize to the same tokens and adjacent pair"+ (doseq [s ["firstName" "first_name" "FIRST-NAME" "first.name" "first name"]]+ (testing (pr-str s)+ (is (= {:tokens #{"first" "name"} :pairs #{"first_name"}}+ (#'classifiers.data-sensitivity/name->tokens s))))))+ (testing "tokens are whole words, not substrings"+ (let [{:keys [tokens]} (#'classifiers.data-sensitivity/name->tokens "zip_code")]+ (is (= #{"zip" "code"} tokens))+ (is (not (contains? tokens "ip")))))+ (testing "pairs are adjacent only"+ (is (= #{"date_of" "of_birth"}+ (:pairs (#'classifiers.data-sensitivity/name->tokens "date_of_birth")))))+ (testing "a single token has no pairs"+ (is (= {:tokens #{"ssn"} :pairs #{}}+ (#'classifiers.data-sensitivity/name->tokens "SSN")))))++(deftest ^:parallel positive-cases-per-category-test+ (doseq [[field-name base-type semantic-type table-context expected]+ [["password" :type/Text nil nil :SEC_KEY]+ ["apiKey" :type/Text nil nil :SEC_KEY]+ ["refresh_token" :type/Text nil nil :SEC_KEY]+ ["client_secret" :type/Text nil nil :SEC_KEY]+ ["salt" :type/Text nil nil :SEC_KEY]+ ["ip_address" :type/Text nil nil :SYS_TELEMETRY]+ ["client_ip" :type/Text nil nil :SYS_TELEMETRY]+ ["user_agent" :type/Text nil nil :SYS_TELEMETRY]+ ["mac_address" :type/Text nil nil :SYS_TELEMETRY]+ ["device_id" :type/Integer nil nil :SYS_TELEMETRY]+ ["diagnosis" :type/Text nil nil :PHI]+ ["diagnosis_code" :type/Text nil nil :PHI]+ ["icd10" :type/Text nil nil :PHI]+ ["patient_id" :type/Integer nil nil :PHI]+ ["bmi" :type/Float nil nil :PHI]+ ["dna" :type/Text nil nil :BIO_GEN]+ ["fingerprint" :type/Text nil nil :BIO_GEN]+ ["retina_scan" :type/Text nil nil :BIO_GEN]+ ["face_id" :type/Text nil nil :BIO_GEN]+ ["card_number" :type/Integer nil nil :PCI_FIN]+ ["credit_card" :type/Text nil nil :PCI_FIN]+ ["cvv" :type/Integer nil nil :PCI_FIN]+ ["iban" :type/Text nil nil :PCI_FIN]+ ["routing_number" :type/Text nil nil :PCI_FIN]+ ["gender" :type/Text nil nil :SENS_PERS]+ ["sex" :type/Text nil nil :SENS_PERS]+ ["ethnicity" :type/Text nil nil :SENS_PERS]+ ["religion" :type/Text nil nil :SENS_PERS]+ ["is_pregnant" :type/Boolean nil nil :SENS_PERS]+ ["ssn" :type/Text nil nil :PII]+ ["passport_number" :type/Text nil nil :PII]+ ["date_of_birth" :type/Date nil nil :PII]+ ["email_address" :type/Text nil nil :PII]+ ["phone" :type/Text nil nil :PII]+ ["phone_number" :type/Integer nil nil :PII]+ ["shipping_address" :type/Text nil nil :PII]+ ["source_code" :type/Text nil nil :CORP_IP]+ ["repo_url" :type/Text nil nil :CORP_IP]+ ["algorithm" :type/Text nil nil :CORP_IP]+ ["salary" :type/Float nil nil :BIZ_CONF]+ ["revenue" :type/Decimal nil nil :BIZ_CONF]+ ["gross_margin" :type/Float nil nil :BIZ_CONF]]]+ (testing (pr-str (list 'infer-data-sensitivity field-name base-type semantic-type table-context))+ (is (= expected+ (infer field-name base-type semantic-type table-context))))))++(deftest ^:parallel base-type-gates-token-rules-test+ (testing "a token rule only fires when the base type isa? one of the rule's base types"+ (is (= :BIZ_CONF (infer "salary" :type/Integer)))+ (is (nil? (infer "salary" :type/Text)))+ (is (= :SEC_KEY (infer "password" :type/Text)))+ (is (nil? (infer "password" :type/Integer)))+ (is (= :SENS_PERS (infer "race" :type/Text)))+ (is (nil? (infer "race_id" :type/Integer)))+ (is (= :PHI (infer "patient" :type/Text)))+ (is (nil? (infer "patient_count" :type/Integer)))))++(deftest ^:parallel non-text-base-types-test+ (testing "IPAddress and Structured base types, which are not descendants of :type/Text, match the text rules that list them"+ (is (= :SYS_TELEMETRY (infer "client_ip" :type/IPAddress)))+ (is (= :SYS_TELEMETRY (infer "mac_address" :type/Structured)))+ (is (= :SEC_KEY (infer "credentials" :type/JSON)))+ (is (= :SEC_KEY (infer "password" :type/XML)))+ (is (= :PHI (infer "medical_history" :type/JSON))))+ (testing "an IPAddress base type implies SYS_TELEMETRY whatever the column is named"+ (is (= :SYS_TELEMETRY (infer "foo" :type/IPAddress))))+ (testing "rules that stay text-only do not match structured columns"+ (is (nil? (infer "home_address" :type/JSON)))+ (is (nil? (infer "source_code" :type/JSON)))+ (is (nil? (infer "settings" :type/JSON)))))++(deftest ^:parallel integer-and-band-variants-test+ (testing "dob matches as an integer as well as a date or text"+ (is (= :PII (infer "dob" :type/Integer)))+ (is (= :PII (infer "dob" :type/Date)))+ (is (= :PII (infer "dob" :type/Text)))+ (is (nil? (infer "dob" :type/Boolean)))+ (is (nil? (infer "birthday" :type/Integer))))+ (testing "salary and compensation bands match as text or integer codes while bare salary stays numeric-only"+ (is (= :BIZ_CONF (infer "salary_band" :type/Text)))+ (is (= :BIZ_CONF (infer "compensation_tier" :type/Text)))+ (is (= :BIZ_CONF (infer "pay_grade" :type/Integer)))+ (is (nil? (infer "salary" :type/Text)))+ (is (nil? (infer "salary_currency" :type/Text)))))++(deftest ^:parallel false-positives-test+ (testing "names that share a substring or token with a rule but are not sensitive return nil"+ (doseq [[field-name base-type]+ [["zip_code" :type/Text]+ ["ship_date" :type/Date]+ ["shipping_address_id" :type/Integer]+ ["description" :type/Text]+ ["monkey" :type/Text]+ ["foreign_key" :type/Integer]+ ["primary_key" :type/Integer]+ ["key_id" :type/Integer]+ ["token_count" :type/Integer]+ ["session_count" :type/Integer]+ ["hostname_count" :type/Integer]+ ["content_hash" :type/Text]+ ["adobe_id" :type/Text]+ ["sussex" :type/Text]+ ["product_name" :type/Text]+ ["filename" :type/Text]+ ["age" :type/Integer]+ ["address" :type/Text]+ ["city" :type/Text]+ ["state" :type/Text]+ ["country" :type/Text]+ ["name" :type/Text]+ ["phone_id" :type/Integer]+ ["password_changed_at" :type/DateTime]+ ["passwordless_login_count" :type/Integer]+ ["emailed_at" :type/DateTime]+ ["email_count" :type/Integer]+ ["businessName" :type/Text]+ ["business_name" :type/Text]+ ["secretary" :type/Text]+ ["disabled" :type/Boolean]+ ["condition" :type/Text]+ ["treatment" :type/Text]+ ["margin_top" :type/Text]+ ["token" :type/Text]]]+ (testing (pr-str field-name)+ (is (nil? (infer field-name base-type))))))+ (testing "geography semantic types alone do not imply PII outside a UserTable"+ (is (nil? (infer "state" :type/Text :type/State {:entity_type :entity/GenericTable})))+ (is (nil? (infer "state" :type/Text :type/State {:name "orders"})))))++(deftest ^:parallel stem-rules-test+ (testing "stems catch spellings the tokenizer splits differently"+ (is (= :PII (infer "ssn4" :type/Text)))+ (is (= :PII (infer "last4ssn" :type/Text)))+ (is (= :PII (infer "passportno" :type/Text)))+ (is (= :PII (infer "userbirthdate" :type/Date)))+ (is (= :PII (infer "useremail" :type/Text)))+ (is (= :SEC_KEY (infer "passwd" :type/Text)))+ (is (= :SEC_KEY (infer "clientsecret" :type/Text)))+ (is (= :PCI_FIN (infer "cardcvv" :type/Text)))+ (is (= :PCI_FIN (infer "ibannumber" :type/Text))))+ (testing "stems are bounded so common words containing them do not match"+ (is (nil? (infer "businessname" :type/Text)))+ (is (nil? (infer "secretary" :type/Text)))+ (is (nil? (infer "birthday_reminder_sent" :type/Boolean)))))++(deftest ^:parallel precedence-test+ (testing "the earliest category in column-data-sensitivity-types wins when several rules match"+ (is (= :PCI_FIN (infer "ssn_card_number" :type/Text)))+ (is (= :SEC_KEY (infer "ssn_password" :type/Text)))+ (is (= :SEC_KEY (infer "salary_password" :type/Text)))+ (is (= :SYS_TELEMETRY (infer "device_fingerprint" :type/Text)))+ (is (= :SEC_KEY (infer "device_token" :type/Text)))+ (is (= :PHI (infer "patient_email" :type/Text)))))++(deftest ^:parallel user-table-gate-test+ (testing "geography and name semantic types imply PII only on a UserTable"+ (doseq [semantic-type [:type/Name :type/Address :type/City :type/State :type/Country :type/ZipCode+ :type/Latitude :type/Longitude :type/User :type/Author]]+ (testing (pr-str semantic-type)+ (is (nil? (infer "col" :type/Text semantic-type {:entity_type :entity/GenericTable})))+ (is (nil? (infer "col" :type/Text semantic-type nil)))+ (is (= :PII (infer "col" :type/Text semantic-type {:entity_type :entity/UserTable}))))))+ (testing "explicit name tokens fire regardless of table"+ (is (= :PII (infer "first_name" :type/Text nil {:entity_type :entity/GenericTable})))+ (is (= :PII (infer "first_name" :type/Text nil {:entity_type :entity/UserTable}))))+ (testing "ungated semantic types fire regardless of table"+ (is (= :PII (infer "col_3" :type/Text :type/Email {:entity_type :entity/GenericTable})))+ (is (= :PII (infer "col_4" :type/Date :type/Birthdate {:entity_type :entity/GenericTable})))+ (is (= :SYS_TELEMETRY (infer "col_5" :type/Text :type/IPAddress {:entity_type :entity/GenericTable}))))+ (testing "semantic types outside both maps do not match"+ (doseq [semantic-type [:type/Category :type/Title :type/Description :type/Company :type/Income :type/Cost+ :type/PK :type/FK :type/CreationTimestamp]]+ (testing (pr-str semantic-type)+ (is (nil? (infer "kind" :type/Text semantic-type {:entity_type :entity/UserTable})))))))++(deftest ^:parallel table-booster-test+ (testing "a weak field token is promoted only when the table name supplies the context"+ (is (nil? (infer "notes" :type/Text nil nil)))+ (is (nil? (infer "notes" :type/Text nil {:name "products"})))+ (is (= :PHI (infer "notes" :type/Text nil {:name "patient_visits"})))+ (is (= :PHI (infer "NOTES" :type/Text nil {:name "PATIENT_VISITS"})))+ (is (= :PHI (infer "condition" :type/Text nil {:name "patients"})))+ (is (= :SEC_KEY (infer "token" :type/Text nil {:name "sessions"})))+ (is (= :SEC_KEY (infer "key" :type/Text nil {:name "api_keys"})))+ (is (nil? (infer "key" :type/Text nil {:name "settings"})))+ (is (= :BIO_GEN (infer "template" :type/Text nil {:name "fingerprint_templates"})))+ (is (= :PCI_FIN (infer "number" :type/Text nil {:name "credit_cards"})))+ (is (= :PCI_FIN (infer "last4" :type/Text nil {:name "cards"})))+ (is (nil? (infer "number" :type/Text nil {:name "invoices"})))+ (is (= :PII (infer "name" :type/Text nil {:name "customers"})))+ (is (= :PII (infer "age" :type/Integer nil {:name "users"})))+ (is (= :PII (infer "address" :type/Text nil {:name "employees"})))+ (is (nil? (infer "name" :type/Text nil {:name "products"})))+ (is (= :BIZ_CONF (infer "amount" :type/Float nil {:name "deals"})))+ (is (nil? (infer "amount" :type/Float nil {:name "orders"})))))++(deftest ^:parallel fingerprint-rule-test+ (let [infer-fp (fn [base-type percent-email]+ (classifiers.data-sensitivity/infer-data-sensitivity+ {:name "col_7"+ :base_type base-type+ :fingerprint {:type {:type/Text {:percent-email percent-email}}}}+ nil))]+ (testing "a text column whose values are almost all emails is PII"+ (is (= :PII (infer-fp :type/Text 0.97))))+ (testing "below the threshold nothing matches"+ (is (nil? (infer-fp :type/Text 0.5))))+ (testing "the rule only applies to text base types"+ (is (nil? (infer-fp :type/Integer 0.97))))+ (testing "a missing fingerprint is not an error"+ (is (nil? (infer "col_7" :type/Text))))))++(def sample-database-fields+ "`[table entity-type field base-type semantic-type]` for every column of the Sample Database, as synced on H2."+ [["ACCOUNTS" :entity/UserTable "ACTIVE_SUBSCRIPTION" :type/Boolean nil]+ ["ACCOUNTS" :entity/UserTable "CANCELED_AT" :type/DateTime :type/CancelationTimestamp]+ ["ACCOUNTS" :entity/UserTable "COUNTRY" :type/Text :type/Country]+ ["ACCOUNTS" :entity/UserTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]+ ["ACCOUNTS" :entity/UserTable "EMAIL" :type/Text :type/Email]+ ["ACCOUNTS" :entity/UserTable "FIRST_NAME" :type/Text :type/Name]+ ["ACCOUNTS" :entity/UserTable "ID" :type/BigInteger :type/PK]+ ["ACCOUNTS" :entity/UserTable "LAST_NAME" :type/Text :type/Name]+ ["ACCOUNTS" :entity/UserTable "LATITUDE" :type/Float :type/Latitude]+ ["ACCOUNTS" :entity/UserTable "LEGACY_PLAN" :type/Boolean nil]+ ["ACCOUNTS" :entity/UserTable "LONGITUDE" :type/Float :type/Longitude]+ ["ACCOUNTS" :entity/UserTable "PLAN" :type/Text :type/Category]+ ["ACCOUNTS" :entity/UserTable "SEATS" :type/Integer nil]+ ["ACCOUNTS" :entity/UserTable "SOURCE" :type/Text :type/Source]+ ["ACCOUNTS" :entity/UserTable "TRIAL_CONVERTED" :type/Boolean nil]+ ["ACCOUNTS" :entity/UserTable "TRIAL_ENDS_AT" :type/DateTime nil]+ ["ANALYTIC_EVENTS" :entity/EventTable "ACCOUNT_ID" :type/BigInteger :type/FK]+ ["ANALYTIC_EVENTS" :entity/EventTable "BUTTON_LABEL" :type/Text :type/Category]+ ["ANALYTIC_EVENTS" :entity/EventTable "EVENT" :type/Text :type/Category]+ ["ANALYTIC_EVENTS" :entity/EventTable "ID" :type/BigInteger :type/PK]+ ["ANALYTIC_EVENTS" :entity/EventTable "PAGE_URL" :type/Text :type/URL]+ ["ANALYTIC_EVENTS" :entity/EventTable "TIMESTAMP" :type/DateTime nil]+ ["FEEDBACK" :entity/GenericTable "ACCOUNT_ID" :type/BigInteger :type/FK]+ ["FEEDBACK" :entity/GenericTable "BODY" :type/Text nil]+ ["FEEDBACK" :entity/GenericTable "DATE_RECEIVED" :type/DateTime nil]+ ["FEEDBACK" :entity/GenericTable "EMAIL" :type/Text :type/Email]+ ["FEEDBACK" :entity/GenericTable "ID" :type/BigInteger :type/PK]+ ["FEEDBACK" :entity/GenericTable "RATING" :type/Integer :type/Score]+ ["FEEDBACK" :entity/GenericTable "RATING_MAPPED" :type/Text :type/Category]+ ["INVOICES" :entity/GenericTable "ACCOUNT_ID" :type/BigInteger :type/FK]+ ["INVOICES" :entity/GenericTable "DATE_RECEIVED" :type/DateTime nil]+ ["INVOICES" :entity/GenericTable "EXPECTED_INVOICE" :type/Boolean nil]+ ["INVOICES" :entity/GenericTable "ID" :type/BigInteger :type/PK]+ ["INVOICES" :entity/GenericTable "PAYMENT" :type/Float nil]+ ["INVOICES" :entity/GenericTable "PLAN" :type/Text :type/Category]+ ["ORDERS" :entity/TransactionTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]+ ["ORDERS" :entity/TransactionTable "DISCOUNT" :type/Float :type/Discount]+ ["ORDERS" :entity/TransactionTable "ID" :type/BigInteger :type/PK]+ ["ORDERS" :entity/TransactionTable "PRODUCT_ID" :type/Integer :type/FK]+ ["ORDERS" :entity/TransactionTable "QUANTITY" :type/Integer :type/Quantity]+ ["ORDERS" :entity/TransactionTable "SUBTOTAL" :type/Float nil]+ ["ORDERS" :entity/TransactionTable "TAX" :type/Float nil]+ ["ORDERS" :entity/TransactionTable "TOTAL" :type/Float nil]+ ["ORDERS" :entity/TransactionTable "USER_ID" :type/Integer :type/FK]+ ["PEOPLE" :entity/UserTable "ADDRESS" :type/Text nil]+ ["PEOPLE" :entity/UserTable "BIRTH_DATE" :type/Date nil]+ ["PEOPLE" :entity/UserTable "CITY" :type/Text :type/City]+ ["PEOPLE" :entity/UserTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]+ ["PEOPLE" :entity/UserTable "EMAIL" :type/Text :type/Email]+ ["PEOPLE" :entity/UserTable "ID" :type/BigInteger :type/PK]+ ["PEOPLE" :entity/UserTable "LATITUDE" :type/Float :type/Latitude]+ ["PEOPLE" :entity/UserTable "LONGITUDE" :type/Float :type/Longitude]+ ["PEOPLE" :entity/UserTable "NAME" :type/Text :type/Name]+ ["PEOPLE" :entity/UserTable "PASSWORD" :type/Text nil]+ ["PEOPLE" :entity/UserTable "SOURCE" :type/Text :type/Source]+ ["PEOPLE" :entity/UserTable "STATE" :type/Text :type/State]+ ["PEOPLE" :entity/UserTable "ZIP" :type/Text :type/ZipCode]+ ["PRODUCTS" :entity/ProductTable "CATEGORY" :type/Text :type/Category]+ ["PRODUCTS" :entity/ProductTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]+ ["PRODUCTS" :entity/ProductTable "EAN" :type/Text nil]+ ["PRODUCTS" :entity/ProductTable "ID" :type/BigInteger :type/PK]+ ["PRODUCTS" :entity/ProductTable "PRICE" :type/Float nil]+ ["PRODUCTS" :entity/ProductTable "RATING" :type/Float :type/Score]+ ["PRODUCTS" :entity/ProductTable "TITLE" :type/Text :type/Title]+ ["PRODUCTS" :entity/ProductTable "VENDOR" :type/Text :type/Company]+ ["REVIEWS" :entity/GenericTable "BODY" :type/Text :type/Description]+ ["REVIEWS" :entity/GenericTable "CREATED_AT" :type/DateTime :type/CreationTimestamp]+ ["REVIEWS" :entity/GenericTable "ID" :type/BigInteger :type/PK]+ ["REVIEWS" :entity/GenericTable "PRODUCT_ID" :type/Integer :type/FK]+ ["REVIEWS" :entity/GenericTable "RATING" :type/Integer :type/Score]+ ["REVIEWS" :entity/GenericTable "REVIEWER" :type/Text nil]])++(def sample-database-expectations+ "Expected `data_sensitivity` for every Sample Database column after one scan, keyed by `[table field]`. Every+ column absent from this map is expected to be `:PUBLIC`."+ {["ACCOUNTS" "COUNTRY"] :PII+ ["ACCOUNTS" "EMAIL"] :PII+ ["ACCOUNTS" "FIRST_NAME"] :PII+ ["ACCOUNTS" "LAST_NAME"] :PII+ ["ACCOUNTS" "LATITUDE"] :PII+ ["ACCOUNTS" "LONGITUDE"] :PII+ ["FEEDBACK" "EMAIL"] :PII+ ["PEOPLE" "ADDRESS"] :PII+ ["PEOPLE" "BIRTH_DATE"] :PII+ ["PEOPLE" "CITY"] :PII+ ["PEOPLE" "EMAIL"] :PII+ ["PEOPLE" "LATITUDE"] :PII+ ["PEOPLE" "LONGITUDE"] :PII+ ["PEOPLE" "NAME"] :PII+ ["PEOPLE" "PASSWORD"] :SEC_KEY+ ["PEOPLE" "STATE"] :PII+ ["PEOPLE" "ZIP"] :PII})++(deftest ^:parallel sample-database-test+ (testing "every Sample Database column classifies to its expected category, or nil where PUBLIC is expected"+ (doseq [[table entity-type field base-type semantic-type] sample-database-fields]+ (testing (str table "." field)+ (is (= (get sample-database-expectations [table field] :PUBLIC)+ (or (infer field base-type semantic-type {:name table :entity_type entity-type})+ :PUBLIC)))))))++(deftest ^:parallel rule-data-invariants-test+ (let [categories (set lib.schema.metadata/column-data-sensitivity-types)+ valid-category? (fn [c] (and (contains? categories c) (not= :PUBLIC c)))+ valid-token? (fn [t] (re-matches #"[a-z0-9]+(?:_[a-z0-9]+)?" t))+ base-types-ok? (fn [bts] (and (set? bts) (seq bts) (every? #(isa? % :type/*) bts)))]+ (testing "no rule outputs :PUBLIC and every output is an enum member"+ (doseq [[_ _ category] (concat @#'classifiers.data-sensitivity/token-rules+ @#'classifiers.data-sensitivity/stem-rules+ @#'classifiers.data-sensitivity/table-boosters)]+ (is (valid-category? category) (pr-str category)))+ (doseq [[_ category] (concat @#'classifiers.data-sensitivity/semantic-type->category+ @#'classifiers.data-sensitivity/user-table-semantic-type->category)]+ (is (valid-category? category) (pr-str category))))+ (testing "token rules use lowercase tokens or single underscore pairs and valid base types"+ (doseq [[tokens base-types _] @#'classifiers.data-sensitivity/token-rules]+ (is (set? tokens))+ (is (every? valid-token? tokens) (pr-str tokens))+ (is (base-types-ok? base-types) (pr-str base-types))))+ (testing "every rule token can be produced by the tokenizer, so no rule is unreachable"+ (doseq [[tokens _ _] @#'classifiers.data-sensitivity/token-rules+ token tokens+ :let [{:keys [tokens pairs]} (#'classifiers.data-sensitivity/name->tokens token)]]+ (is (contains? (into tokens pairs) token) token)))+ (testing "stem rules are compiled patterns with valid base types"+ (doseq [[pattern base-types _] @#'classifiers.data-sensitivity/stem-rules]+ (is (instance? java.util.regex.Pattern pattern))+ (is (base-types-ok? base-types) (pr-str base-types))))+ (testing "semantic maps key on semantic types"+ (doseq [[semantic-type _] (concat @#'classifiers.data-sensitivity/semantic-type->category+ @#'classifiers.data-sensitivity/user-table-semantic-type->category)]+ (is (isa? semantic-type :Semantic/*) (pr-str semantic-type))))+ (testing "booster token sets are lowercase tokens"+ (doseq [[field-tokens table-tokens _] @#'classifiers.data-sensitivity/table-boosters]+ (is (every? valid-token? field-tokens) (pr-str field-tokens))+ (is (every? valid-token? table-tokens) (pr-str table-tokens))))))diff --git a/test/metabase/sync/analyze/data_sensitivity_test.clj b/test/metabase/sync/analyze/data_sensitivity_test.cljnew file mode 100644index 00000000000..2337cd4acf5--- /dev/null+++ b/test/metabase/sync/analyze/data_sensitivity_test.clj@@ -0,0 +1,285 @@+(ns metabase.sync.analyze.data-sensitivity-test+ (:require+ [clojure.test :refer :all]+ [metabase.analyze.core :as analyze]+ [metabase.sync.analyze.data-sensitivity :as sync.data-sensitivity]+ [metabase.sync.core :as sync]+ [metabase.test :as mt]+ [metabase.util :as u]+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]+ [toucan2.core :as t2]))++(defn- label [field-or-id]+ (t2/select-one-fn :data_sensitivity :model/Field :id (u/the-id field-or-id)))++(defn- mirror-label [field-or-id]+ (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id (u/the-id field-or-id)))++(defn- selected-names [table force?]+ (set (map :name (#'sync.data-sensitivity/fields-to-scan table force?))))++(deftest ^:parallel fields-to-scan-test+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "app_users"}+ :model/Field _ {:table_id (:id table) :name "unscanned" :base_type :type/Text}+ :model/Field _ {:table_id (:id table) :name "public" :base_type :type/Text :data_sensitivity :PUBLIC}+ :model/Field _ {:table_id (:id table) :name "labeled" :base_type :type/Text :data_sensitivity :PII}+ :model/Field _ {:table_id (:id table) :name "retired" :base_type :type/Text :visibility_type :retired}+ :model/Field _ {:table_id (:id table) :name "sensitive" :base_type :type/Text :visibility_type :sensitive}+ :model/Field _ {:table_id (:id table) :name "hidden" :base_type :type/Text :visibility_type :hidden}+ :model/Field _ {:table_id (:id table) :name "details" :base_type :type/Text :visibility_type :details-only}+ :model/Field _ {:table_id (:id table) :name "inactive" :base_type :type/Text :active false}]+ (testing "a scheduled scan selects NULL fields of any visibility except retired, never inactive or labeled ones"+ (is (= #{"unscanned" "sensitive" "hidden" "details"}+ (selected-names table false))))+ (testing "a forced scan also reselects PUBLIC fields but never categorized ones"+ (is (= #{"unscanned" "sensitive" "hidden" "details" "public"}+ (selected-names table true))))))++(deftest ^:parallel table-ids-with-unscanned-fields-test+ (mt/with-temp [:model/Database db {}+ :model/Database other-db {}+ :model/Table hidden {:db_id (:id db) :name "hidden_t" :visibility_type :hidden}+ :model/Table cruft {:db_id (:id db) :name "cruft_t" :visibility_type :cruft}+ :model/Table done {:db_id (:id db) :name "done_t"}+ :model/Table inactive {:db_id (:id db) :name "inactive_t" :active false}+ :model/Table foreign {:db_id (:id other-db) :name "foreign_t"}+ :model/Field _ {:table_id (:id hidden) :name "a" :base_type :type/Text}+ :model/Field _ {:table_id (:id cruft) :name "b" :base_type :type/Text}+ :model/Field _ {:table_id (:id done) :name "c" :base_type :type/Text :data_sensitivity :PUBLIC}+ :model/Field _ {:table_id (:id done) :name "d" :base_type :type/Text :data_sensitivity :PII}+ :model/Field _ {:table_id (:id inactive) :name "e" :base_type :type/Text}+ :model/Field _ {:table_id (:id foreign) :name "f" :base_type :type/Text}]+ (testing "hidden and cruft tables with unscanned fields are selected; converged, inactive, and foreign tables are not"+ (is (= #{(:id hidden) (:id cruft)}+ (#'sync.data-sensitivity/table-ids-with-unscanned-fields db false))))+ (testing "force reselects the table whose only unlabeled field is PUBLIC"+ (is (= #{(:id hidden) (:id cruft) (:id done)}+ (#'sync.data-sensitivity/table-ids-with-unscanned-fields db true))))))++(deftest scan-fields-for-db-converges-test+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "app_users" :entity_type :entity/UserTable}+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}+ :model/Field foo {:table_id (:id table) :name "foo" :base_type :type/Text}]+ (testing "the first scan labels every unscanned field, PUBLIC when nothing matches, without creating mirror rows"+ (is (= {:fields-scanned 2 :fields-labeled 1 :fields-failed 0}+ (sync.data-sensitivity/scan-fields-for-db! db nil)))+ (is (= :PII (label ssn)))+ (is (= :PUBLIC (label foo)))+ (is (nil? (mirror-label ssn)))+ (is (nil? (mirror-label foo))))+ (testing "a second scan selects nothing"+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}+ (sync.data-sensitivity/scan-fields-for-db! db nil)))))))++(deftest setting-gates-scans-test+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled false]+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "app_users"}+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}]+ (testing "with the setting off both entry points return zero stats and write nothing"+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}+ (sync.data-sensitivity/scan-fields-for-db! db nil)))+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}+ (sync.data-sensitivity/scan-table! table)))+ (is (nil? (label ssn))))+ (testing "the REPL entry point ignores the setting"+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0}+ (sync.data-sensitivity/scan-data-sensitivity! db)))+ (is (= :PII (label ssn)))))))++(deftest force-rescans-public-only-test+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "app_users"}+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}+ :model/Field foo {:table_id (:id table) :name "foo" :base_type :type/Text}]+ (sync.data-sensitivity/scan-fields-for-db! db nil)+ (is (= :PUBLIC (label foo)))+ (with-redefs [analyze/infer-data-sensitivity (constantly :PHI)]+ (testing "a scheduled scan does not revisit PUBLIC fields even when the rules would now match"+ (is (= {:fields-scanned 0 :fields-labeled 0 :fields-failed 0}+ (sync.data-sensitivity/scan-fields-for-db! db nil)))+ (is (= :PUBLIC (label foo))))+ (testing "a forced scan revisits PUBLIC fields and leaves categorized fields alone"+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0}+ (sync.data-sensitivity/scan-data-sensitivity! db :force? true)))+ (is (= :PHI (label foo)))+ (is (= :PII (label ssn))))))))++(deftest user-label-in-mirror-wins-test+ (testing "a mirror-only user label is what the field reads after the classifier writes"+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "app_users"}+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}+ :model/Field notes {:table_id (:id table) :name "notes" :base_type :type/Text}]+ (field-user-settings/upsert-user-settings ssn {:data_sensitivity :PUBLIC})+ (field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})+ (is (nil? (label ssn)))+ (is (nil? (label notes)))+ (testing "stats count the rule result, the overlay decides what is stored"+ (is (= {:fields-scanned 2 :fields-labeled 1 :fields-failed 0}+ (sync.data-sensitivity/scan-data-sensitivity! db))))+ (is (= :PUBLIC (label ssn)))+ (is (= :PUBLIC (mirror-label ssn)))+ (is (= :PHI (label notes)))+ (is (= :PHI (mirror-label notes)))+ (testing "a forced rescan still cannot override the user's PUBLIC"+ (sync.data-sensitivity/scan-data-sensitivity! db :force? true)+ (is (= :PUBLIC (label ssn)))))))++(deftest reset-data-sensitivity-test+ (mt/with-temp [:model/Database db {}+ :model/Database other-db {}+ :model/Table users {:db_id (:id db) :name "app_users"}+ :model/Table orders {:db_id (:id db) :name "orders"}+ :model/Table foreign {:db_id (:id other-db) :name "app_users"}+ :model/Field ssn {:table_id (:id users) :name "ssn" :base_type :type/Text}+ :model/Field foo {:table_id (:id users) :name "foo" :base_type :type/Text}+ :model/Field notes {:table_id (:id users) :name "notes" :base_type :type/Text}+ :model/Field email {:table_id (:id users) :name "email" :base_type :type/Text}+ :model/Field total {:table_id (:id orders) :name "total" :base_type :type/Float}+ :model/Field far {:table_id (:id foreign) :name "ssn" :base_type :type/Text}]+ (field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})+ (t2/insert! :model/FieldUserSettings {:field_id (:id email)})+ (sync.data-sensitivity/scan-data-sensitivity! db)+ (sync.data-sensitivity/scan-data-sensitivity! other-db)+ (is (= [:PII :PUBLIC :PHI :PII :PUBLIC :PII]+ (map label [ssn foo notes email total far])))+ (testing "a table scope clears only that table's classifier labels"+ (is (= 1 (sync.data-sensitivity/reset-data-sensitivity! orders)))+ (is (nil? (label total)))+ (is (= :PII (label ssn))))+ (testing "a database scope clears categories and PUBLIC alike, including fields with a label-less mirror row"+ (is (= 3 (sync.data-sensitivity/reset-data-sensitivity! db)))+ (is (= [nil nil nil] (map label [ssn foo email]))))+ (testing "a label backed by the mirror is human-set and survives"+ (is (= :PHI (label notes)))+ (is (= :PHI (mirror-label notes))))+ (testing "other databases are untouched"+ (is (= :PII (label far))))+ (testing "a second reset finds nothing"+ (is (zero? (sync.data-sensitivity/reset-data-sensitivity! db))))))++(deftest scan-with-reset-relabels-categorized-fields-test+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "app_users"}+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}+ :model/Field notes {:table_id (:id table) :name "notes" :base_type :type/Text}]+ (field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})+ (sync.data-sensitivity/scan-data-sensitivity! db)+ (is (= :PII (label ssn)))+ (with-redefs [analyze/infer-data-sensitivity (constantly :SEC_KEY)]+ (testing "a forced scan leaves a categorized field on its old label"+ (sync.data-sensitivity/scan-data-sensitivity! db :force? true)+ (is (= :PII (label ssn))))+ (testing "a reset scan recomputes it under the current rules and reports the reset count"+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0 :fields-reset 1}+ (sync.data-sensitivity/scan-data-sensitivity! db :reset? true)))+ (is (= :SEC_KEY (label ssn))))+ (testing "the user's label is neither reset nor rescanned"+ (is (= :PHI (label notes)))+ (is (= :PHI (mirror-label notes)))))))++(deftest classification-failure-is-counted-and-retried-test+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "app_users"}+ :model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}]+ (let [calls (atom 0)]+ (with-redefs [analyze/infer-data-sensitivity (fn [& _] (swap! calls inc) (throw (ex-info "boom" {})))]+ (testing "a throwing rule counts as scanned and failed and leaves the field unscanned"+ (is (= {:fields-scanned 1 :fields-labeled 0 :fields-failed 1}+ (sync.data-sensitivity/scan-data-sensitivity! db)))+ (is (nil? (label ssn))))+ (testing "the next scan retries the field"+ (sync.data-sensitivity/scan-data-sensitivity! db)+ (is (= 2 @calls)))))))++(deftest sync-database-end-to-end-test+ (testing "a full sync with the setting on labels every field of every table using this run's inferred table context"+ (mt/with-temp-test-data [["app_users"+ [{:field-name "ssn", :base-type :type/Text}+ {:field-name "email", :base-type :type/Text}+ {:field-name "card_number", :base-type :type/Text}+ {:field-name "city", :base-type :type/Text}+ {:field-name "notes", :base-type :type/Text}+ {:field-name "foo", :base-type :type/Text}]+ [["123-45-6789" "[email protected]" "4111111111111111" "Berlin" "called back twice" "x"]]]+ ["products"+ [{:field-name "title", :base-type :type/Text}+ {:field-name "city", :base-type :type/Text}+ {:field-name "price", :base-type :type/Float}]+ [["Widget" "Berlin" 9.99]]]]+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]+ (sync/sync-database! (mt/db)))+ (let [labels (fn [table]+ (into {} (map (juxt (comp u/lower-case-en :name) :data_sensitivity))+ (t2/select [:model/Field :name :data_sensitivity] :table_id (mt/id table))))]+ (testing "app_users is a UserTable, so the gated city rule fires alongside the explicit name rules"+ (is (= :entity/UserTable (t2/select-one-fn :entity_type :model/Table :id (mt/id :app_users))))+ (is (= {"id" :PUBLIC+ "ssn" :PII+ "email" :PII+ "card_number" :PCI_FIN+ "city" :PII+ "notes" :PUBLIC+ "foo" :PUBLIC}+ (labels :app_users))))+ (testing "products is not a UserTable, so city stays PUBLIC"+ (is (= {"id" :PUBLIC "title" :PUBLIC "city" :PUBLIC "price" :PUBLIC}+ (labels :products))))+ (testing "no mirror rows were created"+ (is (zero? (t2/count :model/FieldUserSettings+ :field_id [:in (t2/select-pks-set :model/Field :table_id [:in [(mt/id :app_users) (mt/id :products)]])]))))+ (testing "the analyze step recorded its counts in task_history"+ (is (=? {:status :success+ :task_details {:fields-scanned 11 :fields-labeled 4 :fields-failed 0}}+ (t2/select-one :model/TaskHistory :db_id (mt/id) :task "classify-data-sensitivity"+ {:order-by [[:id :desc]]}))))))))++(mt/defdataset inet-columns+ [["connections"+ [{:field-name "client_ip", :base-type {:native "inet"}, :effective-type :type/IPAddress}+ {:field-name "addr", :base-type {:native "inet"}, :effective-type :type/IPAddress}]+ [[[:raw "'192.168.1.1'::inet"] [:raw "'10.4.4.15'::inet"]]]]])++(deftest scan-postgres-inet-columns-test+ (mt/test-driver :postgres+ (testing "inet columns sync as :type/IPAddress base and semantic type and are labeled SYS_TELEMETRY whatever their name"+ (mt/dataset inet-columns+ (let [field-ids (t2/select-pks-set :model/Field :table_id (mt/id :connections) :name [:in ["client_ip" "addr"]])]+ (t2/update! :model/Field :id [:in field-ids] {:data_sensitivity nil})+ (is (= #{[:type/IPAddress :type/IPAddress]}+ (t2/select-fn-set (juxt :base_type :semantic_type) :model/Field :id [:in field-ids])))+ (sync/scan-data-sensitivity! (mt/db))+ (is (= {"client_ip" :SYS_TELEMETRY "addr" :SYS_TELEMETRY}+ (t2/select-fn->fn :name :data_sensitivity :model/Field :id [:in field-ids]))))))))++(deftest scan-non-text-base-types-test+ (testing "IPAddress and JSON base types with no semantic type, as ClickHouse and MySQL sync them, are labeled from the base type"+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db) :name "connections"}+ :model/Field ip {:table_id (:id table) :name "client_ip" :base_type :type/IPAddress}+ :model/Field addr {:table_id (:id table) :name "addr" :base_type :type/IPAddress}+ :model/Field creds {:table_id (:id table) :name "credentials" :base_type :type/JSON}+ :model/Field meta {:table_id (:id table) :name "metadata" :base_type :type/JSON}]+ (is (= {:fields-scanned 4 :fields-labeled 3 :fields-failed 0}+ (sync.data-sensitivity/scan-data-sensitivity! table)))+ (is (= :SYS_TELEMETRY (label ip)))+ (is (= :SYS_TELEMETRY (label addr)))+ (is (= :SEC_KEY (label creds)))+ (is (= :PUBLIC (label meta))))))++(deftest scan-single-table-test+ (mt/with-temp [:model/Database db {}+ :model/Table users {:db_id (:id db) :name "app_users"}+ :model/Table other {:db_id (:id db) :name "other"}+ :model/Field ssn {:table_id (:id users) :name "ssn" :base_type :type/Text}+ :model/Field email {:table_id (:id other) :name "email" :base_type :type/Text}]+ (testing "scanning a Table instance labels only that table's fields"+ (is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0}+ (sync.data-sensitivity/scan-data-sensitivity! users)))+ (is (= :PII (label ssn)))+ (is (nil? (label email))))))diff --git a/test/metabase/sync/analyze_test.clj b/test/metabase/sync/analyze_test.cljindex 2f2dcdb3d2f..d55a10c2c1a 100644--- a/test/metabase/sync/analyze_test.clj+++ b/test/metabase/sync/analyze_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase.analyze.classifiers.category :as classifiers.category]+ [metabase.analyze.classifiers.data-sensitivity :as classifiers.data-sensitivity][metabase.analyze.classifiers.name :as classifiers.name][metabase.analyze.classifiers.no-preview-display:as classifiers.no-preview-display]@@ -99,6 +100,11 @@(testing "Make sure we survive table classification failing"(sync-survives-crash?! classifiers.name/infer-entity-type-by-name)))+(deftest survive-data-sensitivity-errors+ (testing "sync survives the data sensitivity rules failing; the setting must be on for the rules to be reached"+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]+ (sync-survives-crash?! classifiers.data-sensitivity/infer-data-sensitivity))))+(defn- classified-semantic-type [values](let [field (mi/instance :model/Field {:base_type :type/Text})](:semantic_type (classifiers.text-fingerprint/infer-semantic-type@@ -227,7 +233,8 @@:model/Field _ (fake-field table)](let [results (analyze-table! table)](testing "has the steps performed"- (is (= ["fingerprint-fields" "classify-fields" "classify-tables" "score-interestingness"]+ (is (= ["fingerprint-fields" "classify-fields" "classify-tables" "score-interestingness"+ "classify-data-sensitivity"](->> results :steps (map first)))))(testing "has start and finish times"(is (seq (select-keys results [:start-time :end-time]))))))))diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.cljindex 3251391bbe2..ea9e5e402ae 100644--- a/test/metabase/sync/sync_metadata/fields_test.clj+++ b/test/metabase/sync/sync_metadata/fields_test.clj@@ -614,6 +614,28 @@(is (= :PII (field)))(is (= :PII (mirror))))))))+(deftest data-sensitivity-classifier-respects-user-label-test+ (testing "a user-set data_sensitivity wins over the classifier's inference while sibling fields get labeled"+ (mt/with-temp-test-data [["app_users"+ [{:field-name "email", :base-type :type/Text}+ {:field-name "ssn", :base-type :type/Text}+ {:field-name "notes", :base-type :type/Text}]+ [["[email protected]" "123-45-6789" "called back twice"]]]]+ (let [field #(t2/select-one-fn :data_sensitivity :model/Field :id (mt/id :app_users %))+ mirror #(t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id (mt/id :app_users %))]+ (mt/user-http-request :crowberto :put 200 (format "field/%d" (mt/id :app_users :email)) {:data_sensitivity "PUBLIC"})+ (is (nil? (field :ssn)))+ (mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]+ (sync/sync-database! (mt/db)))+ (testing "the user's PUBLIC on email survives even though the rules say PII"+ (is (= :PUBLIC (field :email)))+ (is (= :PUBLIC (mirror :email))))+ (testing "the classifier labels the unlabeled siblings without creating mirror rows"+ (is (= :PII (field :ssn)))+ (is (= :PUBLIC (field :notes)))+ (is (nil? (mirror :ssn)))+ (is (nil? (mirror :notes))))))))+(deftest data-sensitivity-survives-column-drop-and-readd-test(testing "a user-set data_sensitivity survives the warehouse column being dropped and added back"(mt/with-temp-test-data [["readd_table"diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 7a56771c15e..afd96d612e2 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -1748,6 +1748,28 @@(is (pos? (t2/count :model/Table :db_id db-id))"Sync-now must populate tables even when disable-auto-sync is on")))))))+(deftest sync-schema-labels-data-sensitivity-test+ (testing "POST /api/database/:id/sync_schema runs the data sensitivity step when the setting is on"+ ;; Same shape as the test above: create the Database under disable-auto-sync so only the explicit request syncs+ ;; it, and run the quick task synchronously so the labels can be asserted after the request returns.+ (let [details (:details (mt/db))]+ (mt/with-temporary-setting-values [disable-auto-sync true+ data-sensitivity-scan-enabled true]+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2"+ :details details+ :initial_sync_status "incomplete"}]+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f]+ (binding [driver.settings/*allow-testing-h2-connections* true]+ (f)))]+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id)))+ (let [label (fn [table-name field-name]+ (t2/select-one-fn :data_sensitivity :model/Field+ :table_id (t2/select-one-pk :model/Table :db_id db-id :name table-name)+ :name field-name))]+ (is (= :PII (label "PEOPLE" "EMAIL")))+ (is (= :SEC_KEY (label "PEOPLE" "PASSWORD")))+ (is (= :PUBLIC (label "ORDERS" "TOTAL")))))))))+(deftest sync-schema-executes-when-executor-busy-test(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"(let [sync-called? (promise)