The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GHY-4748] Exclude tables hidden in user settings from sync (#83494)
Tyler Plude355147d6a9a4
diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.cljindex 716cf466e4b..31859d7f543 100644--- a/src/metabase/sync/db.clj+++ b/src/metabase/sync/db.clj@@ -1,7 +1,7 @@(ns metabase.sync.db- "Application database queries for the sync module. Table reads here ask for the rows sync itself wrote- (`{:user-settings? false}`): it reconciles them against what the warehouse reports, so it has to see a Table as it- recorded it, whatever a reader would be shown instead.+ "Application database queries for the sync module. Reads of what sync reconciles against the warehouse ask for the+ rows sync itself wrote (`{:user-settings? false}`), whatever a reader would be shown instead; choices of which+ Tables or Fields to act on read what users see.Every function here is a direct Toucan 2 call with no additional logic, so the rest of the module never talks to`toucan2.core` itself."@@ -20,9 +20,13 @@[metabase.warehouses.schema :as warehouses.schema][toucan2.core :as t2]))-(def ^:private sync-tables-clause- "Honey SQL clause selecting the Tables that take part in sync: active and not hidden."- [:and [:= :active true] [:= :visibility_type nil]])+(mu/defn- sync-tables-clause+ "Honey SQL clause matching the Tables aliased `table-alias` that take part in sync: active, with no+ `visibility_type` as users see it."+ [table-alias :- :keyword]+ [:and+ [:= (u/qualified-key table-alias :active) true]+ [:= (warehouse-schema-overlay/table-user-visibility-type table-alias) nil]]);;; ------------------------------------------------ Database ------------------------------------------------@@ -110,8 +114,8 @@table-name :- :string](t2/select-one-pk :model/Table :db_id database-id :name table-name :active true {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))-(mu/defn sync-tables-by-lower-name-and-schema- "The synced Tables of the Database with `database-id` whose lower-cased name and schema match."+(mu/defn active-tables-by-lower-name-and-schema+ "The active Tables of the Database with `database-id` whose lower-cased name and schema match, hidden ones included."[database-id :- ::lib.schema.id/databaselower-name :- :stringlower-schema :- [:maybe :string]]@@ -119,8 +123,8 @@:db_id database-id:%lower.name lower-name:%lower.schema lower-schema- {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]- :where sync-tables-clause}))+ :active true+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))(mu/defn tables-by-name"The `columns` of the Tables of the Database with `database-id` named one of `table-names`."@@ -166,20 +170,20 @@"The IDs of the synced Tables of the Database with `database-id`."[database-id :- ::lib.schema.id/database](t2/select-fn-vec :id :model/Table :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]- :where sync-tables-clause}))+ :where (sync-tables-clause :metabase_table)}))(mu/defn sync-table-schemas"The distinct `:schema` rows of the synced Tables of the Database with `database-id`."[database-id :- ::lib.schema.id/database](t2/query {:select-distinct [:schema]:from [:metabase_table]- :where [:and sync-tables-clause [:= :db_id database-id]]}))+ :where [:and (sync-tables-clause :metabase_table) [:= :db_id database-id]]}))(mu/defn sync-tables-count"The number of synced Tables in the Database with `database-id`."[database-id :- ::lib.schema.id/database](t2/count :model/Table :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]- :where sync-tables-clause}))+ :where (sync-tables-clause :metabase_table)}))(mu/defn sync-tables-reducible"Reducible synced Tables of the Database with `database-id` ordered by schema and name, optionally narrowed to@@ -190,7 +194,7 @@(t2/reducible-select :model/Table:db_id database-id{:from [(warehouse-schema-overlay/table-query {:user-settings? false})]- :where [:and sync-tables-clause+ :where [:and (sync-tables-clause :metabase_table)(when (seq schema-names) [:in :schema schema-names])(when (seq table-names) [:in :name table-names])]:order-by [[:schema :asc] [:name :asc]]}))@@ -200,13 +204,13 @@[database-id :- ::lib.schema.id/database](t2/reducible-select :model/Table{:select [:t.*]- :from [(warehouse-schema-overlay/table-query {:alias :t})]+ :from [(warehouse-schema-overlay/table-query {:user-settings? false :alias :t})]:left-join [[^:allow-subquery {:select [:table_id[[:min :last_analyzed] :earliest_last_analyzed]]:from [(warehouse-schema-overlay/field-query)]:group-by [:table_id]} :sub][:= :t.id :sub.table_id]]- :where [:and sync-tables-clause [:= :t.db_id database-id]]+ :where [:and (sync-tables-clause :t) [:= :t.db_id database-id]]:order-by [[:sub.earliest_last_analyzed :asc]]}))(mu/defn insert-table!@@ -496,7 +500,7 @@:table_id [:in ^:allow-subquery{:select [:id]:from [(t2/table-name :model/Table)]- :where [:and sync-tables-clause [:= :db_id database-id]]}]}+ :where [:and (sync-tables-clause :metabase_table) [:= :db_id database-id]]}]}{:last_analyzed :%now}))(defn- fk-field-id-subquerydiff --git a/src/metabase/sync/sync_metadata/fields.clj b/src/metabase/sync/sync_metadata/fields.cljindex c3816e7e321..d7171834625 100644--- a/src/metabase/sync/sync_metadata/fields.clj+++ b/src/metabase/sync/sync_metadata/fields.clj@@ -123,18 +123,23 @@[database :- i/DatabaseInstance](sync-util/with-error-handling (format "Error syncing Fields for Database ''%s''" (sync-util/name-for-logging database))(let [driver (driver.u/database->driver database)- schemas? (driver.u/supports? driver :schemas database)]+ schemas? (driver.u/supports? driver :schemas database)+ sync-table-id? (set (sync.db/sync-table-ids (:id database)))](letfn [(sync! [fields-metadata](transduce (comp(partition-by (juxt :table-name :table-schema))(map (fn [table-metadata](let [{:keys [table-name table-schema]} (first table-metadata)- table (->> (sync.db/sync-tables-by-lower-name-and-schema- (:id database)- (t2.util/lower-case-en table-name)- (some-> table-schema t2.util/lower-case-en))- (sort-by (select-best-matching-name table-schema table-name))- first)+ ;; match over hidden Tables too, so a hidden Table's columns are never+ ;; given to a visible Table whose name differs only in case+ table (let [best (->> (sync.db/active-tables-by-lower-name-and-schema+ (:id database)+ (t2.util/lower-case-en table-name)+ (some-> table-schema t2.util/lower-case-en))+ (sort-by (select-best-matching-name table-schema table-name))+ first)]+ (when (sync-table-id? (:id best))+ best))updated (if table(try;; TODO: decouple nested field columns sync from field sync. This will allowdiff --git a/src/metabase/warehouse_schema_overlay/core.clj b/src/metabase/warehouse_schema_overlay/core.cljindex c6f7fc270f6..7eb052dc29b 100644--- a/src/metabase/warehouse_schema_overlay/core.clj+++ b/src/metabase/warehouse_schema_overlay/core.clj@@ -145,6 +145,18 @@[:case [:= [:coalesce settings-column table-column] true] true :else false][:coalesce settings-column table-column]))))+(mu/defn table-user-visibility-type+ "Honey SQL expression for the `visibility_type` users see for the Table aliased `table-alias`: the user value+ when set, else the Table's. Reads only `table-alias`'s `id`, so it can filter a query over raw `metabase_table`."+ [table-alias :- :keyword]+ ;; A scalar subquery, not a join, so the caller's rows keep `metabase_table`'s shape. It re-reads the Table so the+ ;; LEFT JOIN always yields one row: a user-set NULL must not fall back to the Table's value.+ ^:allow-subquery+ {:select [[(table-user-settings-column :visibility_type :vt_t :vt_u)]]+ :from [[(t2/table-name :model/Table) :vt_t]]+ :left-join (table-user-settings-join :vt_t :vt_u)+ :where [:= :vt_t.id (u/qualified-key table-alias :id)]})+(mu/defn table-query :- [:tuple :any :keyword]"The source a query over Tables reads from: `metabase_table` merged with the user values, with the options of[[field-query]]."diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.cljindex b83b40e3c90..e049912c45b 100644--- a/test/metabase/app_db/schema_migrations_test.clj+++ b/test/metabase/app_db/schema_migrations_test.clj@@ -42,6 +42,7 @@[metabase.util.encryption :as encryption][metabase.util.encryption-test :as encryption-test][metabase.util.json :as json]+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -3456,6 +3457,51 @@(is (=? {:description_set true, :semantic_type_set false, :fk_target_field_id_set false}(t2/select-one :metabase_field_user_settings :field_id mixed-id))))))))+(deftest table-user-settings-migration-keeps-hidden-tables-hidden-test+ (testing "v64.2026-09-11T00:00:05-06: the backfill and the reset leave the visibility_type users see unchanged, with+ or without a settings row"+ (impl/test-migrations ["v64.2026-09-11T00:00:00" "v64.2026-09-11T00:00:06"] [migrate!]+ (let [db-id (t2/insert-returning-pk! :metabase_database {:name "Table User Settings Test DB"+ :engine "h2"+ :created_at :%now+ :updated_at :%now+ :details "{}"})+ insert-table! (fn [table-name published? visibility-type data-layer]+ (t2/insert-returning-pk! :metabase_table {:active true+ :db_id db-id+ :name table-name+ :is_published published?+ :visibility_type visibility-type+ :data_layer data-layer+ :created_at :%now+ :updated_at :%now}))+ hidden (insert-table! "hidden" true "hidden" "hidden")+ technical (insert-table! "technical" true "technical" "internal")+ cruft (insert-table! "cruft" true "cruft" "internal")+ visible (insert-table! "visible" true nil "internal")+ unpublished (insert-table! "unpublished" false "hidden" "hidden")+ user-visibility (fn [table-id]+ (t2/select-one-fn :v [:metabase_table+ [(warehouse-schema-overlay/table-user-visibility-type :metabase_table) :v]]+ {:where [:= :metabase_table.id table-id]}))]+ (migrate!)+ (testing "the backfill moves a published table's hidden or technical visibility into its settings row"+ (doseq [[table-id visibility-type] [[hidden "hidden"] [technical "technical"]]]+ (is (=? {:visibility_type nil :data_layer "internal"}+ (t2/select-one [:metabase_table :visibility_type :data_layer] :id table-id)))+ (is (=? {:visibility_type visibility-type :visibility_type_set true}+ (t2/select-one :metabase_table_user_settings :table_id table-id)))))+ (testing "sync's cruft stays on metabase_table and is not recorded as the user's"+ (is (= "cruft" (t2/select-one-fn :visibility_type :metabase_table :id cruft)))+ (is (=? {:visibility_type nil :visibility_type_set false}+ (t2/select-one :metabase_table_user_settings :table_id cruft))))+ (testing "an unpublished table gets no settings row and keeps its own visibility_type"+ (is (nil? (t2/select-one :metabase_table_user_settings :table_id unpublished)))+ (is (= "hidden" (t2/select-one-fn :visibility_type :metabase_table :id unpublished))))+ (testing "the visibility_type users see is unchanged by the migration"+ (is (= {hidden "hidden" technical "technical" cruft "cruft" visible nil unpublished "hidden"}+ (into {} (map (juxt identity user-visibility)) [hidden technical cruft visible unpublished]))))))))+(deftest glossary-entity-id-backfill-test(testing "v64.2026-09-11: glossary.entity_id is added, backfilled for existing rows, NOT NULL and unique"(impl/test-migrations ["v64.2026-09-11T12:00:00" "v64.2026-09-11T12:00:03"] [migrate!]diff --git a/test/metabase/sync/analyze_test.clj b/test/metabase/sync/analyze_test.cljindex d55a10c2c1a..3aaa766451b 100644--- a/test/metabase/sync/analyze_test.clj+++ b/test/metabase/sync/analyze_test.clj@@ -197,7 +197,7 @@(deftest dont-analyze-hidden-tables-test(testing "expect all the kinds of hidden tables to stay un-analyzed through transitions and repeated syncing"(letfn [(tests [sync!*]- (mt/with-temp [:model/Table table (assoc (fake-table) :visibility_type "hidden")+ (mt/with-temp [:model/Table table (fake-table):model/Field field (fake-field table)](letfn [(set-visibility! [visibility](set-table-visibility-type-via-api! table visibility)@@ -227,6 +227,15 @@(testing "\nsame test but with sync triggered programatically rather than via the API"(tests analyze-table!)))))+(deftest dont-analyze-data-layer-hidden-tables-test+ (testing "a table hidden through the Data Studio data layer stays un-analyzed"+ (mt/with-temp [:model/Table table (fake-table)+ :model/Field field (fake-field table)]+ (mt/user-http-request :crowberto :post 200 "data-studio/table/edit"+ {:table_ids [(u/the-id table)] :data_layer "hidden"})+ (analyze-table! table)+ (is (false? (fake-field-was-analyzed? field))))))+(deftest analyze-db!-return-value-test(testing "Returns values"(mt/with-temp [:model/Table table (fake-table)diff --git a/test/metabase/sync/field_values_test.clj b/test/metabase/sync/field_values_test.cljindex 4ca54865153..fbcdf607320 100644--- a/test/metabase/sync/field_values_test.clj+++ b/test/metabase/sync/field_values_test.clj@@ -80,6 +80,18 @@(is (= [1 2 3 4](venues-price-field-values))))))+(deftest sync-skips-field-values-of-user-hidden-table-test+ (testing "sync refreshes a table's FieldValues until a user hides it through PUT /api/table/:id"+ (data/with-temp-copy-of-db+ (letfn [(field-values-after-sync! []+ (field-values/get-or-create-full-field-values! (t2/select-one :model/Field (mt/id :venues :price)))+ (t2/update! :model/FieldValues :field_id (mt/id :venues :price) :type :full {:values [10 20 30 40]})+ (sync/update-field-values! (data/db))+ (venues-price-field-values))]+ (is (= [1 2 3 4] (field-values-after-sync!)))+ (mt/user-http-request :crowberto :put 200 (format "table/%d" (mt/id :venues)) {:visibility_type "hidden"})+ (is (= [10 20 30 40] (field-values-after-sync!)))))))+(deftest sync-should-properly-handle-last-used-at(try(testing "Test that syncing will skip updating inactive FieldValues"diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.cljindex 46f08c7f569..56c7d498701 100644--- a/test/metabase/sync/sync_metadata/fields_test.clj+++ b/test/metabase/sync/sync_metadata/fields_test.clj@@ -546,6 +546,27 @@{:from [(warehouse-schema-overlay/field-query)]})))"Second sync should preserve manually set :normal visibility_type"))))))))+(deftest hiding-a-case-variant-table-leaves-the-other-tables-fields-alone-test+ (testing "hiding one of two tables whose names differ only in case does not move its columns onto the other"+ (mt/test-driver :postgres+ (tx/drop-if-exists-and-create-db! driver/*driver* "hidden_case_variant_table_test")+ (let [details (mt/dbdef->connection-details :postgres :db {:database-name "hidden_case_variant_table_test"})+ spec (sql-jdbc.conn/connection-details->spec :postgres details)]+ (doseq [statement ["CREATE TABLE \"Foo\" (a INT, b INT);"+ "CREATE TABLE \"foo\" (x INT, y INT);"]]+ (jdbc/execute! spec [statement]))+ (doseq [[hidden visible expected] [["foo" "Foo" {"a" true, "b" true}]+ ["Foo" "foo" {"x" true, "y" true}]]]+ (testing (format "hiding %s" hidden)+ (mt/with-temp [:model/Database database {:engine :postgres, :details details}]+ (sync-metadata/sync-db-metadata! database)+ (let [table-id #(t2/select-one-pk :model/Table :db_id (u/the-id database) :name %)+ active-by-name #(t2/select-fn->fn :name :active :model/Field :table_id (table-id %))]+ (is (= expected (active-by-name visible)))+ (mt/user-http-request :crowberto :put 200 (format "table/%d" (table-id hidden)) {:visibility_type "hidden"})+ (sync-metadata/sync-db-metadata! database)+ (is (= expected (active-by-name visible)))))))))))+(deftest user-set-fks-are-preserved-by-sync-test(testing "Check that sync-table! doesn't remove user-set FKs during normal sync operations"(with-test-dbdiff --git a/test/metabase/sync/util_test.clj b/test/metabase/sync/util_test.cljindex 38b967f1d87..492fcaadbe3 100644--- a/test/metabase/sync/util_test.clj+++ b/test/metabase/sync/util_test.clj@@ -16,6 +16,7 @@[metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.test.util :as tu]+ [metabase.util.quick-task :as quick-task][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -373,3 +374,42 @@:sync db "test sync failure"(fn [] (throw (Exception. "sync boom"))))(is (< initial (mt/metric-value system :metabase-sync/failures {:driver "h2"}))))))))++;;; +----------------------------------------------------------------------------------------------------------------++;;; | Tables that take part in sync |+;;; +----------------------------------------------------------------------------------------------------------------+++(defn- in-sync-tables? [table-id]+ (contains? (into #{} (map :id) (sync-util/reducible-sync-tables (mt/id))) table-id))++(defn- set-visibility-type-via-api! [table-id visibility-type]+ (mt/user-http-request :crowberto :put 200 (format "table/%d" table-id) {:visibility_type visibility-type}))++(deftest reducible-sync-tables-follows-user-visibility-test+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [_task])]+ (testing "a visible table takes part in sync"+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]+ (is (in-sync-tables? table-id))))+ (testing "a table sync recorded as hidden is left out"+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES" :visibility_type :hidden}]+ (is (not (in-sync-tables? table-id)))))+ (testing "a table hidden through PUT /api/table/:id is left out"+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]+ (set-visibility-type-via-api! table-id "hidden")+ (is (not (in-sync-tables? table-id)))))+ (testing "a table hidden through the Data Studio data layer is left out"+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]+ (mt/user-http-request :crowberto :post 200 "data-studio/table/edit" {:table_ids [table-id] :data_layer "hidden"})+ (is (not (in-sync-tables? table-id)))))+ (testing "a hidden table takes part in sync again once the user un-hides it"+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]+ (set-visibility-type-via-api! table-id "hidden")+ (set-visibility-type-via-api! table-id nil)+ (is (in-sync-tables? table-id))))+ (testing "a cruft table takes part in sync once the user sets its visibility_type to NULL"+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES" :visibility_type :cruft}]+ (is (not (in-sync-tables? table-id)))+ (set-visibility-type-via-api! table-id nil)+ (is (= {:visibility_type nil :visibility_type_set true}+ (t2/select-one [:model/TableUserSettings :visibility_type :visibility_type_set] :table_id table-id)))+ (is (in-sync-tables? table-id))))))diff --git a/test/metabase/warehouse_schema_overlay/core_test.clj b/test/metabase/warehouse_schema_overlay/core_test.cljindex 812c4a31e32..d7b7c31f0f9 100644--- a/test/metabase/warehouse_schema_overlay/core_test.clj+++ b/test/metabase/warehouse_schema_overlay/core_test.clj@@ -1,9 +1,11 @@(ns metabase.warehouse-schema-overlay.core-test(:require[clojure.test :refer :all]+ [metabase.test :as mt][metabase.util.malli :as mu][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]- [metabase.warehouse-schema.schema]))+ [metabase.warehouse-schema.schema]+ [toucan2.core :as t2]))(comment metabase.warehouse-schema.schema/keep-me)@@ -16,3 +18,35 @@(testing "the spelled-out column list keeps up with the Field schema it cannot require"(is (= (into #{:id :unique_field_helper} (mu/map-schema-keys :metabase.warehouse-schema.schema/field.update))warehouse-schema-overlay/field-columns))))++(defn- visibility-types+ "`[table-user-visibility-type, visibility_type read through table-query]` for the Table `table-id`."+ [table-id]+ [(t2/select-one-fn :v [(t2/table-name :model/Table)+ [(warehouse-schema-overlay/table-user-visibility-type :metabase_table) :v]]+ {:where [:= :metabase_table.id table-id]})+ (some-> (t2/select-one-fn :visibility_type [:model/Table :visibility_type] :id table-id+ {:from [(warehouse-schema-overlay/table-query)]})+ name)])++(deftest table-user-visibility-type-test+ (testing "table-user-visibility-type matches the visibility_type table-query reads"+ (mt/with-temp [:model/Table {plain :id} {}+ :model/Table {cruft :id} {:visibility_type :cruft}+ :model/Table {put-hidden :id} {}+ :model/Table {data-studio :id} {}+ :model/Table {cruft-cleared :id} {:visibility_type :cruft}+ :model/Table {description-only :id} {:visibility_type :cruft}]+ (mt/user-http-request :crowberto :put 200 (format "table/%d" put-hidden) {:visibility_type "hidden"})+ (mt/user-http-request :crowberto :post 200 "data-studio/table/edit"+ {:table_ids [data-studio] :data_layer "hidden"})+ (t2/insert! :model/TableUserSettings {:table_id cruft-cleared :visibility_type nil})+ (t2/insert! :model/TableUserSettings {:table_id description-only :description "user description"})+ (doseq [[desc table-id expected] [["no settings row" plain nil]+ ["sync's cruft with no settings row" cruft "cruft"]+ ["hidden through PUT /api/table/:id" put-hidden "hidden"]+ ["data_layer hidden through Data Studio" data-studio "hidden"]+ ["an explicit user NULL over sync's cruft" cruft-cleared nil]+ ["a settings row with visibility_type unset over cruft" description-only "cruft"]]]+ (testing desc+ (is (= [expected expected] (visibility-types table-id))))))))