The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Keep field-values for detail-only fields (#67540)
metamben353260e5832c
diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.cljindex 72ca6b2ae57..f2359df89cd 100644--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj@@ -239,7 +239,8 @@(when semantic-type{:semantic-type semantic-type})(when (and json? (driver/database-supports? driver :nested-field-columns db))- {:visibility-type :details-only})))))))+ {:visibility-type :details-only+ :preview-display false})))))))(defn describe-table-fields-xf"Returns a transducer for computing metadata about the fields in a table"diff --git a/src/metabase/sync/sync_metadata/fields/our_metadata.clj b/src/metabase/sync/sync_metadata/fields/our_metadata.cljindex 88079434eee..b55ab1b20a5 100644--- a/src/metabase/sync/sync_metadata/fields/our_metadata.clj+++ b/src/metabase/sync/sync_metadata/fields/our_metadata.clj@@ -35,7 +35,8 @@:database-position (:database_position field):database-partitioned (:database_partitioned field):database-required (:database_required field)- :visibility-type (:visibility_type field)}+ :visibility-type (:visibility_type field)+ :preview-display (:preview_display field)}(u/remove-nils{:pk? (:database_is_pk field):database-is-generated (:database_is_generated field)@@ -77,7 +78,7 @@:parent_id :id :description :database_position :nfc_path:database_is_auto_increment :database_required:database_default :database_is_generated :database_is_nullable :database_is_pk- :database_partitioned :json_unfolding :position]+ :database_partitioned :json_unfolding :position :preview_display]:table_id (u/the-id table):active true{:order-by table/field-order-rule}))diff --git a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj b/src/metabase/sync/sync_metadata/fields/sync_metadata.cljindex 24513559724..6a8f3720e40 100644--- a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj+++ b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj@@ -15,9 +15,12 @@[metabase.warehouse-schema.models.field-user-settings :as schema.field-user-settings][toucan2.core :as t2]))+(defn- crufty-field? [db field-metadata]+ (crufty/name? (:name field-metadata)+ (some-> db :settings :auto-cruft-columns)))+(defn- compute-new-visibility-type [db field-metadata]- (if (crufty/name? (:name field-metadata)- (some-> db :settings :auto-cruft-columns))+ (if (crufty-field? db field-metadata):details-only;; n.b. if it was auto-crufted in the past, removing it from auto-cruft will NOT make it visible because old;; visibility-type will be :details-only. This only changes things to be hidden. If you want to make it visible@@ -45,7 +48,8 @@old-database-is-nullable :database-is-nullableold-db-partitioned :database-partitionedold-db-required :database-required- old-visibility-type :visibility-type} metabase-field+ old-visibility-type :visibility-type+ old-preview-display :preview-display} metabase-field{new-database-type :database-typenew-base-type :base-typenew-field-comment :field-comment@@ -92,8 +96,11 @@new-db-generated? (not= old-database-is-generated new-database-is-generated)new-db-nullable? (not= old-database-is-nullable new-database-is-nullable)new-db-partitioned? (not= new-db-partitioned old-db-partitioned)- new-db-required? (not= old-db-required new-db-required)- new-visibility-type? (not= old-visibility-type new-visibility-type)+ new-db-required? (not= old-db-required new-db-required)+ new-visibility-type? (not= old-visibility-type new-visibility-type)+ ;; set preview_display=false for crufty fields (prevents FieldValues from being created)+ is-crufty? (crufty-field? database field-metadata)+ set-preview-display-false? (and is-crufty? old-preview-display);; calculate combined updatesupdates@@ -197,7 +204,9 @@new-db-required){:database_required new-db-required})(when new-visibility-type?- {:visibility_type new-visibility-type}))]+ {:visibility_type new-visibility-type})+ (when set-preview-display-false?+ {:preview_display false}))];; if any updates need to be done, do them and return 1 (because 1 Field was updated), otherwise return 0(if (and (seq updates)(pos? (t2/update! :model/Field (u/the-id metabase-field) updates)))diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 3396862cbaf..41b65597c85 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -252,15 +252,19 @@[field-or-field-id](if-not (map? field-or-field-id)(let [field-id (u/the-id field-or-field-id)]- (recur (or (t2/select-one ['Field :base_type :visibility_type :has_field_values] :id field-id)+ (recur (or (t2/select-one ['Field :base_type :visibility_type :has_field_values :preview_display] :id field-id)(throw (ex-info (tru "Field {0} does not exist." field-id){:field-id field-id, :status-code 404})))))(let [{base-type :base_typevisibility-type :visibility_type- has-field-values :has_field_values} field-or-field-id]+ has-field-values :has_field_values+ preview-display :preview_display} field-or-field-id](boolean(and- (not (contains? #{:retired :sensitive :hidden :details-only} (keyword visibility-type)))+ (not (contains? #{:retired :sensitive :hidden} (keyword visibility-type)))+ ;; preview_display is set to false by sync for fields that shouldn't have FieldValues+ ;; (e.g. long text fields, JSON columns, auto-cruft columns). Defaults to true if not provided.+ (not (false? preview-display))(not (isa? (keyword base-type) :type/field-values-unsupported))(not (= (keyword base-type) :type/*))(#{:list :auto-list} (keyword has-field-values)))))))diff --git a/test/metabase/sync/sync_metadata/fields/our_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/our_metadata_test.cljindex 1e250804b43..6c228ad374d 100644--- a/test/metabase/sync/sync_metadata/fields/our_metadata_test.clj+++ b/test/metabase/sync/sync_metadata/fields/our_metadata_test.clj@@ -22,7 +22,8 @@:pk? true:database-required false:database-is-auto-increment true- :json-unfolding false}+ :json-unfolding false+ :preview-display true}{:name "buyer":database-type "OBJECT":base-type :type/Dictionary@@ -31,6 +32,7 @@:database-required false:database-is-auto-increment false:json-unfolding false+ :preview-display true:nested-fields #{{:name "name":database-type "VARCHAR":base-type :type/Text@@ -38,7 +40,8 @@:pk? false:database-required false:json-unfolding false- :database-is-auto-increment false}+ :database-is-auto-increment false+ :preview-display true}{:name "cc":database-type "VARCHAR":base-type :type/Text@@ -46,7 +49,8 @@:pk? false:database-required false:json-unfolding false- :database-is-auto-increment false}}}+ :database-is-auto-increment false+ :preview-display true}}}{:name "ts":database-type "BIGINT":base-type :type/BigInteger@@ -55,7 +59,8 @@:pk? false:database-is-auto-increment false:json-unfolding false- :database-required false}+ :database-required false+ :preview-display true}{:name "toucan":database-type "OBJECT":base-type :type/Dictionary@@ -64,6 +69,7 @@:database-required false:database-is-auto-increment false:json-unfolding false+ :preview-display true:nested-fields #{{:name "name":database-type "VARCHAR":base-type :type/Text@@ -71,7 +77,8 @@:pk? false:database-required false:json-unfolding false- :database-is-auto-increment false}+ :database-is-auto-increment false+ :preview-display true}{:name "details":database-type "OBJECT":base-type :type/Dictionary@@ -80,6 +87,7 @@:database-required false:json-unfolding false:database-is-auto-increment false+ :preview-display true:nested-fields #{{:name "weight":database-type "DECIMAL":base-type :type/Decimal@@ -88,7 +96,8 @@:pk? false:database-required false:json-unfolding false- :database-is-auto-increment false}+ :database-is-auto-increment false+ :preview-display true}{:name "age":database-type "INT":base-type :type/Integer@@ -96,7 +105,8 @@:pk? false:database-required false:json-unfolding false- :database-is-auto-increment false}}}}}}+ :database-is-auto-increment false+ :preview-display true}}}}}}(let [transactions-table-id (u/the-id (t2/select-one-pk :model/Table :db_id (u/the-id db), :name "transactions"))remove-ids-and-nil-vals (partial walk/postwalk #(if-not (map? %)diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.cljindex 1b95ce28a5b..a0b9468f4ed 100644--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj@@ -14,7 +14,10 @@;; use alphabetical field_order by default because the default, database, will update the position(updates-that-will-be-performed! new-metadata-from-sync metadata-in-application-db {:field_order :alphabetical}))([new-metadata-from-sync metadata-in-application-db table]- (mt/with-temp [:model/Database db {}+ (updates-that-will-be-performed! new-metadata-from-sync metadata-in-application-db table {}))+ ([new-metadata-from-sync metadata-in-application-db table db-settings]+ (mt/with-temp [:model/Database db (cond-> {}+ (seq db-settings) (assoc :settings db-settings)):model/Table table (assoc table :db_id (u/the-id db))](let [update-operations (atom [])](with-redefs [t2/update! (fn [model id updates]@@ -220,6 +223,33 @@:database-is-auto-increment false:json-unfolding false})))))+(deftest crufty-field-sets-preview-display-false-test+ (testing "When a field becomes crufty (matches auto-cruft pattern), preview_display should be set to false"+ (testing "field with preview_display=true should have it set to false when crufty"+ (is (= [["Field" 1 {:visibility_type :details-only+ :preview_display false}]]+ (updates-that-will-be-performed!+ ;; new metadata from sync - field name matches auto-cruft pattern+ (merge default-metadata {:name "updated_at"})+ ;; existing field in application DB with preview_display=true+ (merge default-metadata {:name "updated_at"+ :id 1+ :visibility-type :normal+ :preview-display true})+ {:field_order :alphabetical}+ ;; database settings with auto-cruft pattern matching "updated_at"+ {:auto-cruft-columns ["updated_at"]}))))+ (testing "field already with preview_display=false should not be updated again"+ (is (= [["Field" 1 {:visibility_type :details-only}]]+ (updates-that-will-be-performed!+ (merge default-metadata {:name "updated_at"})+ (merge default-metadata {:name "updated_at"+ :id 1+ :visibility-type :normal+ :preview-display false})+ {:field_order :alphabetical}+ {:auto-cruft-columns ["updated_at"]}))))))+(deftest dont-overwrite-semantic-type-test(testing "We should not override non-nil `semantic_type`s"(is (= []diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.cljindex c12aaaa2a03..421e4a58f88 100644--- a/test/metabase/sync/sync_metadata/fields_test.clj+++ b/test/metabase/sync/sync_metadata/fields_test.clj@@ -468,6 +468,23 @@(mapv :visibility_type)frequencies))))))+(deftest auto-cruft-sets-preview-display-false-test+ (testing "When a field becomes crufty via auto-cruft settings, preview_display should be set to false (#10851)"+ (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery}]+ (sync-metadata/sync-db-metadata! db)+ (let [details-field (t2/select-one :model/Field :name "details"+ :table_id [:in (map :id (t2/select :model/Table :db_id (u/the-id db)))])]+ (is (=? {:visibility_type :normal+ :preview_display true}+ details-field)+ "before auto-cruft: field has default visibility and preview_display")+ (t2/update! :model/Database (u/the-id db) {:settings {:auto-cruft-columns ["details"]}})+ (sync-metadata/sync-db-metadata! (t2/select-one :model/Database (u/the-id db)))+ (is (=? {:visibility_type :details-only+ :preview_display false}+ (t2/select-one :model/Field :id (:id details-field)))+ "after auto-cruft: field should have visibility_type=details-only AND preview_display=false")))))+(deftest sync-fields-resilient-to-non-existence-test(testing "[[sync-fields/sync-fields-for-table!]] doesn't crash on a non-existent table (SEM-39)"(mt/test-drivers (mt/normal-drivers)diff --git a/test/metabase/warehouse_schema/models/field_values_test.clj b/test/metabase/warehouse_schema/models/field_values_test.cljindex aa68e3f89c3..2677e8eb34c 100644--- a/test/metabase/warehouse_schema/models/field_values_test.clj+++ b/test/metabase/warehouse_schema/models/field_values_test.clj@@ -94,7 +94,7 @@:base_type "type/Boolean"}true}- "retired/sensitive/hidden/details-only fields should always be excluded"+ "retired/sensitive/hidden fields should always be excluded"{{:base_type :type/Boolean:has_field_values :list:visibility_type :retired}@@ -118,16 +118,30 @@{:has_field_values :list:visibility_type :hidden:base_type :type/Text}- false+ false}- {:base_type :type/Boolean+ "details-only fields should be included (fixes #10851)"+ {{:base_type :type/Boolean:has_field_values :list:visibility_type :details-only}- false+ true{:has_field_values :list:visibility_type :details-only:base_type :type/Text}+ true}++ "fields with preview_display=false should be excluded"+ {{:base_type :type/Boolean+ :has_field_values :list+ :visibility_type :normal+ :preview_display false}+ false++ {:has_field_values :list+ :visibility_type :details-only+ :base_type :type/Text+ :preview_display false}false}"date/time based fields should always be excluded"