Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Keep field-values for detail-only fields (#67540)

metamben353260e5832c
diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
index 72ca6b2ae57..f2359df89cd 100644
--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj
+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
@@ -239,7 +239,8 @@
(when semantic-type
{:semantic-type semantic-type})
(when (and json? (driver/database-supports? driver :nested-field-columns db))
- {:visibility-type :details-only})))))))
+ {:visibility-type :details-only
+ :preview-display false})))))))
(defn describe-table-fields-xf
"Returns a transducer for computing metadata about the fields in a table"
diff --git a/src/metabase/sync/sync_metadata/fields/our_metadata.clj b/src/metabase/sync/sync_metadata/fields/our_metadata.clj
index 88079434eee..b55ab1b20a5 100644
--- a/src/metabase/sync/sync_metadata/fields/our_metadata.clj
+++ b/src/metabase/sync/sync_metadata/fields/our_metadata.clj
@@ -35,7 +35,8 @@
:database-position (:database_position field)
:database-partitioned (:database_partitioned field)
:database-required (:database_required field)
- :visibility-type (:visibility_type field)}
+ :visibility-type (:visibility_type field)
+ :preview-display (:preview_display field)}
(u/remove-nils
{:pk? (:database_is_pk field)
:database-is-generated (:database_is_generated field)
@@ -77,7 +78,7 @@
:parent_id :id :description :database_position :nfc_path
:database_is_auto_increment :database_required
:database_default :database_is_generated :database_is_nullable :database_is_pk
- :database_partitioned :json_unfolding :position]
+ :database_partitioned :json_unfolding :position :preview_display]
:table_id (u/the-id table)
:active true
{:order-by table/field-order-rule}))
diff --git a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
index 24513559724..6a8f3720e40 100644
--- a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
+++ b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
@@ -15,9 +15,12 @@
[metabase.warehouse-schema.models.field-user-settings :as schema.field-user-settings]
[toucan2.core :as t2]))
+(defn- crufty-field? [db field-metadata]
+ (crufty/name? (:name field-metadata)
+ (some-> db :settings :auto-cruft-columns)))
+
(defn- compute-new-visibility-type [db field-metadata]
- (if (crufty/name? (:name field-metadata)
- (some-> db :settings :auto-cruft-columns))
+ (if (crufty-field? db field-metadata)
:details-only
;; n.b. if it was auto-crufted in the past, removing it from auto-cruft will NOT make it visible because old
;; visibility-type will be :details-only. This only changes things to be hidden. If you want to make it visible
@@ -45,7 +48,8 @@
old-database-is-nullable :database-is-nullable
old-db-partitioned :database-partitioned
old-db-required :database-required
- old-visibility-type :visibility-type} metabase-field
+ old-visibility-type :visibility-type
+ old-preview-display :preview-display} metabase-field
{new-database-type :database-type
new-base-type :base-type
new-field-comment :field-comment
@@ -92,8 +96,11 @@
new-db-generated? (not= old-database-is-generated new-database-is-generated)
new-db-nullable? (not= old-database-is-nullable new-database-is-nullable)
new-db-partitioned? (not= new-db-partitioned old-db-partitioned)
- new-db-required? (not= old-db-required new-db-required)
- new-visibility-type? (not= old-visibility-type new-visibility-type)
+ new-db-required? (not= old-db-required new-db-required)
+ new-visibility-type? (not= old-visibility-type new-visibility-type)
+ ;; set preview_display=false for crufty fields (prevents FieldValues from being created)
+ is-crufty? (crufty-field? database field-metadata)
+ set-preview-display-false? (and is-crufty? old-preview-display)
;; calculate combined updates
updates
@@ -197,7 +204,9 @@
new-db-required)
{:database_required new-db-required})
(when new-visibility-type?
- {:visibility_type new-visibility-type}))]
+ {:visibility_type new-visibility-type})
+ (when set-preview-display-false?
+ {:preview_display false}))]
;; if any updates need to be done, do them and return 1 (because 1 Field was updated), otherwise return 0
(if (and (seq updates)
(pos? (t2/update! :model/Field (u/the-id metabase-field) updates)))
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index 3396862cbaf..41b65597c85 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -252,15 +252,19 @@
[field-or-field-id]
(if-not (map? field-or-field-id)
(let [field-id (u/the-id field-or-field-id)]
- (recur (or (t2/select-one ['Field :base_type :visibility_type :has_field_values] :id field-id)
+ (recur (or (t2/select-one ['Field :base_type :visibility_type :has_field_values :preview_display] :id field-id)
(throw (ex-info (tru "Field {0} does not exist." field-id)
{:field-id field-id, :status-code 404})))))
(let [{base-type :base_type
visibility-type :visibility_type
- has-field-values :has_field_values} field-or-field-id]
+ has-field-values :has_field_values
+ preview-display :preview_display} field-or-field-id]
(boolean
(and
- (not (contains? #{:retired :sensitive :hidden :details-only} (keyword visibility-type)))
+ (not (contains? #{:retired :sensitive :hidden} (keyword visibility-type)))
+ ;; preview_display is set to false by sync for fields that shouldn't have FieldValues
+ ;; (e.g. long text fields, JSON columns, auto-cruft columns). Defaults to true if not provided.
+ (not (false? preview-display))
(not (isa? (keyword base-type) :type/field-values-unsupported))
(not (= (keyword base-type) :type/*))
(#{:list :auto-list} (keyword has-field-values)))))))
diff --git a/test/metabase/sync/sync_metadata/fields/our_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/our_metadata_test.clj
index 1e250804b43..6c228ad374d 100644
--- a/test/metabase/sync/sync_metadata/fields/our_metadata_test.clj
+++ b/test/metabase/sync/sync_metadata/fields/our_metadata_test.clj
@@ -22,7 +22,8 @@
:pk? true
:database-required false
:database-is-auto-increment true
- :json-unfolding false}
+ :json-unfolding false
+ :preview-display true}
{:name "buyer"
:database-type "OBJECT"
:base-type :type/Dictionary
@@ -31,6 +32,7 @@
:database-required false
:database-is-auto-increment false
:json-unfolding false
+ :preview-display true
:nested-fields #{{:name "name"
:database-type "VARCHAR"
:base-type :type/Text
@@ -38,7 +40,8 @@
:pk? false
:database-required false
:json-unfolding false
- :database-is-auto-increment false}
+ :database-is-auto-increment false
+ :preview-display true}
{:name "cc"
:database-type "VARCHAR"
:base-type :type/Text
@@ -46,7 +49,8 @@
:pk? false
:database-required false
:json-unfolding false
- :database-is-auto-increment false}}}
+ :database-is-auto-increment false
+ :preview-display true}}}
{:name "ts"
:database-type "BIGINT"
:base-type :type/BigInteger
@@ -55,7 +59,8 @@
:pk? false
:database-is-auto-increment false
:json-unfolding false
- :database-required false}
+ :database-required false
+ :preview-display true}
{:name "toucan"
:database-type "OBJECT"
:base-type :type/Dictionary
@@ -64,6 +69,7 @@
:database-required false
:database-is-auto-increment false
:json-unfolding false
+ :preview-display true
:nested-fields #{{:name "name"
:database-type "VARCHAR"
:base-type :type/Text
@@ -71,7 +77,8 @@
:pk? false
:database-required false
:json-unfolding false
- :database-is-auto-increment false}
+ :database-is-auto-increment false
+ :preview-display true}
{:name "details"
:database-type "OBJECT"
:base-type :type/Dictionary
@@ -80,6 +87,7 @@
:database-required false
:json-unfolding false
:database-is-auto-increment false
+ :preview-display true
:nested-fields #{{:name "weight"
:database-type "DECIMAL"
:base-type :type/Decimal
@@ -88,7 +96,8 @@
:pk? false
:database-required false
:json-unfolding false
- :database-is-auto-increment false}
+ :database-is-auto-increment false
+ :preview-display true}
{:name "age"
:database-type "INT"
:base-type :type/Integer
@@ -96,7 +105,8 @@
:pk? false
:database-required false
:json-unfolding false
- :database-is-auto-increment false}}}}}}
+ :database-is-auto-increment false
+ :preview-display true}}}}}}
(let [transactions-table-id (u/the-id (t2/select-one-pk :model/Table :db_id (u/the-id db), :name "transactions"))
remove-ids-and-nil-vals (partial walk/postwalk #(if-not (map? %)
diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
index 1b95ce28a5b..a0b9468f4ed 100644
--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
@@ -14,7 +14,10 @@
;; use alphabetical field_order by default because the default, database, will update the position
(updates-that-will-be-performed! new-metadata-from-sync metadata-in-application-db {:field_order :alphabetical}))
([new-metadata-from-sync metadata-in-application-db table]
- (mt/with-temp [:model/Database db {}
+ (updates-that-will-be-performed! new-metadata-from-sync metadata-in-application-db table {}))
+ ([new-metadata-from-sync metadata-in-application-db table db-settings]
+ (mt/with-temp [:model/Database db (cond-> {}
+ (seq db-settings) (assoc :settings db-settings))
:model/Table table (assoc table :db_id (u/the-id db))]
(let [update-operations (atom [])]
(with-redefs [t2/update! (fn [model id updates]
@@ -220,6 +223,33 @@
:database-is-auto-increment false
:json-unfolding false})))))
+(deftest crufty-field-sets-preview-display-false-test
+ (testing "When a field becomes crufty (matches auto-cruft pattern), preview_display should be set to false"
+ (testing "field with preview_display=true should have it set to false when crufty"
+ (is (= [["Field" 1 {:visibility_type :details-only
+ :preview_display false}]]
+ (updates-that-will-be-performed!
+ ;; new metadata from sync - field name matches auto-cruft pattern
+ (merge default-metadata {:name "updated_at"})
+ ;; existing field in application DB with preview_display=true
+ (merge default-metadata {:name "updated_at"
+ :id 1
+ :visibility-type :normal
+ :preview-display true})
+ {:field_order :alphabetical}
+ ;; database settings with auto-cruft pattern matching "updated_at"
+ {:auto-cruft-columns ["updated_at"]}))))
+ (testing "field already with preview_display=false should not be updated again"
+ (is (= [["Field" 1 {:visibility_type :details-only}]]
+ (updates-that-will-be-performed!
+ (merge default-metadata {:name "updated_at"})
+ (merge default-metadata {:name "updated_at"
+ :id 1
+ :visibility-type :normal
+ :preview-display false})
+ {:field_order :alphabetical}
+ {:auto-cruft-columns ["updated_at"]}))))))
+
(deftest dont-overwrite-semantic-type-test
(testing "We should not override non-nil `semantic_type`s"
(is (= []
diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.clj
index c12aaaa2a03..421e4a58f88 100644
--- a/test/metabase/sync/sync_metadata/fields_test.clj
+++ b/test/metabase/sync/sync_metadata/fields_test.clj
@@ -468,6 +468,23 @@
(mapv :visibility_type)
frequencies))))))
+(deftest auto-cruft-sets-preview-display-false-test
+ (testing "When a field becomes crufty via auto-cruft settings, preview_display should be set to false (#10851)"
+ (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery}]
+ (sync-metadata/sync-db-metadata! db)
+ (let [details-field (t2/select-one :model/Field :name "details"
+ :table_id [:in (map :id (t2/select :model/Table :db_id (u/the-id db)))])]
+ (is (=? {:visibility_type :normal
+ :preview_display true}
+ details-field)
+ "before auto-cruft: field has default visibility and preview_display")
+ (t2/update! :model/Database (u/the-id db) {:settings {:auto-cruft-columns ["details"]}})
+ (sync-metadata/sync-db-metadata! (t2/select-one :model/Database (u/the-id db)))
+ (is (=? {:visibility_type :details-only
+ :preview_display false}
+ (t2/select-one :model/Field :id (:id details-field)))
+ "after auto-cruft: field should have visibility_type=details-only AND preview_display=false")))))
+
(deftest sync-fields-resilient-to-non-existence-test
(testing "[[sync-fields/sync-fields-for-table!]] doesn't crash on a non-existent table (SEM-39)"
(mt/test-drivers (mt/normal-drivers)
diff --git a/test/metabase/warehouse_schema/models/field_values_test.clj b/test/metabase/warehouse_schema/models/field_values_test.clj
index aa68e3f89c3..2677e8eb34c 100644
--- a/test/metabase/warehouse_schema/models/field_values_test.clj
+++ b/test/metabase/warehouse_schema/models/field_values_test.clj
@@ -94,7 +94,7 @@
:base_type "type/Boolean"}
true}
- "retired/sensitive/hidden/details-only fields should always be excluded"
+ "retired/sensitive/hidden fields should always be excluded"
{{:base_type :type/Boolean
:has_field_values :list
:visibility_type :retired}
@@ -118,16 +118,30 @@
{:has_field_values :list
:visibility_type :hidden
:base_type :type/Text}
- false
+ false}
- {:base_type :type/Boolean
+ "details-only fields should be included (fixes #10851)"
+ {{:base_type :type/Boolean
:has_field_values :list
:visibility_type :details-only}
- false
+ true
{:has_field_values :list
:visibility_type :details-only
:base_type :type/Text}
+ true}
+
+ "fields with preview_display=false should be excluded"
+ {{:base_type :type/Boolean
+ :has_field_values :list
+ :visibility_type :normal
+ :preview_display false}
+ false
+
+ {:has_field_values :list
+ :visibility_type :details-only
+ :base_type :type/Text
+ :preview_display false}
false}
"date/time based fields should always be excluded"