Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GHY-4748] Exclude tables hidden in user settings from sync (#83494)

Tyler Plude355147d6a9a4
diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.clj
index 716cf466e4b..31859d7f543 100644
--- a/src/metabase/sync/db.clj
+++ b/src/metabase/sync/db.clj
@@ -1,7 +1,7 @@
(ns metabase.sync.db
- "Application database queries for the sync module. Table reads here ask for the rows sync itself wrote
- (`{:user-settings? false}`): it reconciles them against what the warehouse reports, so it has to see a Table as it
- recorded it, whatever a reader would be shown instead.
+ "Application database queries for the sync module. Reads of what sync reconciles against the warehouse ask for the
+ rows sync itself wrote (`{:user-settings? false}`), whatever a reader would be shown instead; choices of which
+ Tables or Fields to act on read what users see.
Every function here is a direct Toucan 2 call with no additional logic, so the rest of the module never talks to
`toucan2.core` itself."
@@ -20,9 +20,13 @@
[metabase.warehouses.schema :as warehouses.schema]
[toucan2.core :as t2]))
-(def ^:private sync-tables-clause
- "Honey SQL clause selecting the Tables that take part in sync: active and not hidden."
- [:and [:= :active true] [:= :visibility_type nil]])
+(mu/defn- sync-tables-clause
+ "Honey SQL clause matching the Tables aliased `table-alias` that take part in sync: active, with no
+ `visibility_type` as users see it."
+ [table-alias :- :keyword]
+ [:and
+ [:= (u/qualified-key table-alias :active) true]
+ [:= (warehouse-schema-overlay/table-user-visibility-type table-alias) nil]])
;;; ------------------------------------------------ Database ------------------------------------------------
@@ -110,8 +114,8 @@
table-name :- :string]
(t2/select-one-pk :model/Table :db_id database-id :name table-name :active true {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
-(mu/defn sync-tables-by-lower-name-and-schema
- "The synced Tables of the Database with `database-id` whose lower-cased name and schema match."
+(mu/defn active-tables-by-lower-name-and-schema
+ "The active Tables of the Database with `database-id` whose lower-cased name and schema match, hidden ones included."
[database-id :- ::lib.schema.id/database
lower-name :- :string
lower-schema :- [:maybe :string]]
@@ -119,8 +123,8 @@
:db_id database-id
:%lower.name lower-name
:%lower.schema lower-schema
- {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
- :where sync-tables-clause}))
+ :active true
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn tables-by-name
"The `columns` of the Tables of the Database with `database-id` named one of `table-names`."
@@ -166,20 +170,20 @@
"The IDs of the synced Tables of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
(t2/select-fn-vec :id :model/Table :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
- :where sync-tables-clause}))
+ :where (sync-tables-clause :metabase_table)}))
(mu/defn sync-table-schemas
"The distinct `:schema` rows of the synced Tables of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
(t2/query {:select-distinct [:schema]
:from [:metabase_table]
- :where [:and sync-tables-clause [:= :db_id database-id]]}))
+ :where [:and (sync-tables-clause :metabase_table) [:= :db_id database-id]]}))
(mu/defn sync-tables-count
"The number of synced Tables in the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
(t2/count :model/Table :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
- :where sync-tables-clause}))
+ :where (sync-tables-clause :metabase_table)}))
(mu/defn sync-tables-reducible
"Reducible synced Tables of the Database with `database-id` ordered by schema and name, optionally narrowed to
@@ -190,7 +194,7 @@
(t2/reducible-select :model/Table
:db_id database-id
{:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
- :where [:and sync-tables-clause
+ :where [:and (sync-tables-clause :metabase_table)
(when (seq schema-names) [:in :schema schema-names])
(when (seq table-names) [:in :name table-names])]
:order-by [[:schema :asc] [:name :asc]]}))
@@ -200,13 +204,13 @@
[database-id :- ::lib.schema.id/database]
(t2/reducible-select :model/Table
{:select [:t.*]
- :from [(warehouse-schema-overlay/table-query {:alias :t})]
+ :from [(warehouse-schema-overlay/table-query {:user-settings? false :alias :t})]
:left-join [[^:allow-subquery {:select [:table_id
[[:min :last_analyzed] :earliest_last_analyzed]]
:from [(warehouse-schema-overlay/field-query)]
:group-by [:table_id]} :sub]
[:= :t.id :sub.table_id]]
- :where [:and sync-tables-clause [:= :t.db_id database-id]]
+ :where [:and (sync-tables-clause :t) [:= :t.db_id database-id]]
:order-by [[:sub.earliest_last_analyzed :asc]]}))
(mu/defn insert-table!
@@ -496,7 +500,7 @@
:table_id [:in ^:allow-subquery
{:select [:id]
:from [(t2/table-name :model/Table)]
- :where [:and sync-tables-clause [:= :db_id database-id]]}]}
+ :where [:and (sync-tables-clause :metabase_table) [:= :db_id database-id]]}]}
{:last_analyzed :%now}))
(defn- fk-field-id-subquery
diff --git a/src/metabase/sync/sync_metadata/fields.clj b/src/metabase/sync/sync_metadata/fields.clj
index c3816e7e321..d7171834625 100644
--- a/src/metabase/sync/sync_metadata/fields.clj
+++ b/src/metabase/sync/sync_metadata/fields.clj
@@ -123,18 +123,23 @@
[database :- i/DatabaseInstance]
(sync-util/with-error-handling (format "Error syncing Fields for Database ''%s''" (sync-util/name-for-logging database))
(let [driver (driver.u/database->driver database)
- schemas? (driver.u/supports? driver :schemas database)]
+ schemas? (driver.u/supports? driver :schemas database)
+ sync-table-id? (set (sync.db/sync-table-ids (:id database)))]
(letfn [(sync! [fields-metadata]
(transduce (comp
(partition-by (juxt :table-name :table-schema))
(map (fn [table-metadata]
(let [{:keys [table-name table-schema]} (first table-metadata)
- table (->> (sync.db/sync-tables-by-lower-name-and-schema
- (:id database)
- (t2.util/lower-case-en table-name)
- (some-> table-schema t2.util/lower-case-en))
- (sort-by (select-best-matching-name table-schema table-name))
- first)
+ ;; match over hidden Tables too, so a hidden Table's columns are never
+ ;; given to a visible Table whose name differs only in case
+ table (let [best (->> (sync.db/active-tables-by-lower-name-and-schema
+ (:id database)
+ (t2.util/lower-case-en table-name)
+ (some-> table-schema t2.util/lower-case-en))
+ (sort-by (select-best-matching-name table-schema table-name))
+ first)]
+ (when (sync-table-id? (:id best))
+ best))
updated (if table
(try
;; TODO: decouple nested field columns sync from field sync. This will allow
diff --git a/src/metabase/warehouse_schema_overlay/core.clj b/src/metabase/warehouse_schema_overlay/core.clj
index c6f7fc270f6..7eb052dc29b 100644
--- a/src/metabase/warehouse_schema_overlay/core.clj
+++ b/src/metabase/warehouse_schema_overlay/core.clj
@@ -145,6 +145,18 @@
[:case [:= [:coalesce settings-column table-column] true] true :else false]
[:coalesce settings-column table-column]))))
+(mu/defn table-user-visibility-type
+ "Honey SQL expression for the `visibility_type` users see for the Table aliased `table-alias`: the user value
+ when set, else the Table's. Reads only `table-alias`'s `id`, so it can filter a query over raw `metabase_table`."
+ [table-alias :- :keyword]
+ ;; A scalar subquery, not a join, so the caller's rows keep `metabase_table`'s shape. It re-reads the Table so the
+ ;; LEFT JOIN always yields one row: a user-set NULL must not fall back to the Table's value.
+ ^:allow-subquery
+ {:select [[(table-user-settings-column :visibility_type :vt_t :vt_u)]]
+ :from [[(t2/table-name :model/Table) :vt_t]]
+ :left-join (table-user-settings-join :vt_t :vt_u)
+ :where [:= :vt_t.id (u/qualified-key table-alias :id)]})
+
(mu/defn table-query :- [:tuple :any :keyword]
"The source a query over Tables reads from: `metabase_table` merged with the user values, with the options of
[[field-query]]."
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index b83b40e3c90..e049912c45b 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -42,6 +42,7 @@
[metabase.util.encryption :as encryption]
[metabase.util.encryption-test :as encryption-test]
[metabase.util.json :as json]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -3456,6 +3457,51 @@
(is (=? {:description_set true, :semantic_type_set false, :fk_target_field_id_set false}
(t2/select-one :metabase_field_user_settings :field_id mixed-id))))))))
+(deftest table-user-settings-migration-keeps-hidden-tables-hidden-test
+ (testing "v64.2026-09-11T00:00:05-06: the backfill and the reset leave the visibility_type users see unchanged, with
+ or without a settings row"
+ (impl/test-migrations ["v64.2026-09-11T00:00:00" "v64.2026-09-11T00:00:06"] [migrate!]
+ (let [db-id (t2/insert-returning-pk! :metabase_database {:name "Table User Settings Test DB"
+ :engine "h2"
+ :created_at :%now
+ :updated_at :%now
+ :details "{}"})
+ insert-table! (fn [table-name published? visibility-type data-layer]
+ (t2/insert-returning-pk! :metabase_table {:active true
+ :db_id db-id
+ :name table-name
+ :is_published published?
+ :visibility_type visibility-type
+ :data_layer data-layer
+ :created_at :%now
+ :updated_at :%now}))
+ hidden (insert-table! "hidden" true "hidden" "hidden")
+ technical (insert-table! "technical" true "technical" "internal")
+ cruft (insert-table! "cruft" true "cruft" "internal")
+ visible (insert-table! "visible" true nil "internal")
+ unpublished (insert-table! "unpublished" false "hidden" "hidden")
+ user-visibility (fn [table-id]
+ (t2/select-one-fn :v [:metabase_table
+ [(warehouse-schema-overlay/table-user-visibility-type :metabase_table) :v]]
+ {:where [:= :metabase_table.id table-id]}))]
+ (migrate!)
+ (testing "the backfill moves a published table's hidden or technical visibility into its settings row"
+ (doseq [[table-id visibility-type] [[hidden "hidden"] [technical "technical"]]]
+ (is (=? {:visibility_type nil :data_layer "internal"}
+ (t2/select-one [:metabase_table :visibility_type :data_layer] :id table-id)))
+ (is (=? {:visibility_type visibility-type :visibility_type_set true}
+ (t2/select-one :metabase_table_user_settings :table_id table-id)))))
+ (testing "sync's cruft stays on metabase_table and is not recorded as the user's"
+ (is (= "cruft" (t2/select-one-fn :visibility_type :metabase_table :id cruft)))
+ (is (=? {:visibility_type nil :visibility_type_set false}
+ (t2/select-one :metabase_table_user_settings :table_id cruft))))
+ (testing "an unpublished table gets no settings row and keeps its own visibility_type"
+ (is (nil? (t2/select-one :metabase_table_user_settings :table_id unpublished)))
+ (is (= "hidden" (t2/select-one-fn :visibility_type :metabase_table :id unpublished))))
+ (testing "the visibility_type users see is unchanged by the migration"
+ (is (= {hidden "hidden" technical "technical" cruft "cruft" visible nil unpublished "hidden"}
+ (into {} (map (juxt identity user-visibility)) [hidden technical cruft visible unpublished]))))))))
+
(deftest glossary-entity-id-backfill-test
(testing "v64.2026-09-11: glossary.entity_id is added, backfilled for existing rows, NOT NULL and unique"
(impl/test-migrations ["v64.2026-09-11T12:00:00" "v64.2026-09-11T12:00:03"] [migrate!]
diff --git a/test/metabase/sync/analyze_test.clj b/test/metabase/sync/analyze_test.clj
index d55a10c2c1a..3aaa766451b 100644
--- a/test/metabase/sync/analyze_test.clj
+++ b/test/metabase/sync/analyze_test.clj
@@ -197,7 +197,7 @@
(deftest dont-analyze-hidden-tables-test
(testing "expect all the kinds of hidden tables to stay un-analyzed through transitions and repeated syncing"
(letfn [(tests [sync!*]
- (mt/with-temp [:model/Table table (assoc (fake-table) :visibility_type "hidden")
+ (mt/with-temp [:model/Table table (fake-table)
:model/Field field (fake-field table)]
(letfn [(set-visibility! [visibility]
(set-table-visibility-type-via-api! table visibility)
@@ -227,6 +227,15 @@
(testing "\nsame test but with sync triggered programatically rather than via the API"
(tests analyze-table!)))))
+(deftest dont-analyze-data-layer-hidden-tables-test
+ (testing "a table hidden through the Data Studio data layer stays un-analyzed"
+ (mt/with-temp [:model/Table table (fake-table)
+ :model/Field field (fake-field table)]
+ (mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
+ {:table_ids [(u/the-id table)] :data_layer "hidden"})
+ (analyze-table! table)
+ (is (false? (fake-field-was-analyzed? field))))))
+
(deftest analyze-db!-return-value-test
(testing "Returns values"
(mt/with-temp [:model/Table table (fake-table)
diff --git a/test/metabase/sync/field_values_test.clj b/test/metabase/sync/field_values_test.clj
index 4ca54865153..fbcdf607320 100644
--- a/test/metabase/sync/field_values_test.clj
+++ b/test/metabase/sync/field_values_test.clj
@@ -80,6 +80,18 @@
(is (= [1 2 3 4]
(venues-price-field-values))))))
+(deftest sync-skips-field-values-of-user-hidden-table-test
+ (testing "sync refreshes a table's FieldValues until a user hides it through PUT /api/table/:id"
+ (data/with-temp-copy-of-db
+ (letfn [(field-values-after-sync! []
+ (field-values/get-or-create-full-field-values! (t2/select-one :model/Field (mt/id :venues :price)))
+ (t2/update! :model/FieldValues :field_id (mt/id :venues :price) :type :full {:values [10 20 30 40]})
+ (sync/update-field-values! (data/db))
+ (venues-price-field-values))]
+ (is (= [1 2 3 4] (field-values-after-sync!)))
+ (mt/user-http-request :crowberto :put 200 (format "table/%d" (mt/id :venues)) {:visibility_type "hidden"})
+ (is (= [10 20 30 40] (field-values-after-sync!)))))))
+
(deftest sync-should-properly-handle-last-used-at
(try
(testing "Test that syncing will skip updating inactive FieldValues"
diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.clj
index 46f08c7f569..56c7d498701 100644
--- a/test/metabase/sync/sync_metadata/fields_test.clj
+++ b/test/metabase/sync/sync_metadata/fields_test.clj
@@ -546,6 +546,27 @@
{:from [(warehouse-schema-overlay/field-query)]})))
"Second sync should preserve manually set :normal visibility_type"))))))))
+(deftest hiding-a-case-variant-table-leaves-the-other-tables-fields-alone-test
+ (testing "hiding one of two tables whose names differ only in case does not move its columns onto the other"
+ (mt/test-driver :postgres
+ (tx/drop-if-exists-and-create-db! driver/*driver* "hidden_case_variant_table_test")
+ (let [details (mt/dbdef->connection-details :postgres :db {:database-name "hidden_case_variant_table_test"})
+ spec (sql-jdbc.conn/connection-details->spec :postgres details)]
+ (doseq [statement ["CREATE TABLE \"Foo\" (a INT, b INT);"
+ "CREATE TABLE \"foo\" (x INT, y INT);"]]
+ (jdbc/execute! spec [statement]))
+ (doseq [[hidden visible expected] [["foo" "Foo" {"a" true, "b" true}]
+ ["Foo" "foo" {"x" true, "y" true}]]]
+ (testing (format "hiding %s" hidden)
+ (mt/with-temp [:model/Database database {:engine :postgres, :details details}]
+ (sync-metadata/sync-db-metadata! database)
+ (let [table-id #(t2/select-one-pk :model/Table :db_id (u/the-id database) :name %)
+ active-by-name #(t2/select-fn->fn :name :active :model/Field :table_id (table-id %))]
+ (is (= expected (active-by-name visible)))
+ (mt/user-http-request :crowberto :put 200 (format "table/%d" (table-id hidden)) {:visibility_type "hidden"})
+ (sync-metadata/sync-db-metadata! database)
+ (is (= expected (active-by-name visible)))))))))))
+
(deftest user-set-fks-are-preserved-by-sync-test
(testing "Check that sync-table! doesn't remove user-set FKs during normal sync operations"
(with-test-db
diff --git a/test/metabase/sync/util_test.clj b/test/metabase/sync/util_test.clj
index 38b967f1d87..492fcaadbe3 100644
--- a/test/metabase/sync/util_test.clj
+++ b/test/metabase/sync/util_test.clj
@@ -16,6 +16,7 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.test.util :as tu]
+ [metabase.util.quick-task :as quick-task]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -373,3 +374,42 @@
:sync db "test sync failure"
(fn [] (throw (Exception. "sync boom"))))
(is (< initial (mt/metric-value system :metabase-sync/failures {:driver "h2"}))))))))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Tables that take part in sync |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(defn- in-sync-tables? [table-id]
+ (contains? (into #{} (map :id) (sync-util/reducible-sync-tables (mt/id))) table-id))
+
+(defn- set-visibility-type-via-api! [table-id visibility-type]
+ (mt/user-http-request :crowberto :put 200 (format "table/%d" table-id) {:visibility_type visibility-type}))
+
+(deftest reducible-sync-tables-follows-user-visibility-test
+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [_task])]
+ (testing "a visible table takes part in sync"
+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]
+ (is (in-sync-tables? table-id))))
+ (testing "a table sync recorded as hidden is left out"
+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES" :visibility_type :hidden}]
+ (is (not (in-sync-tables? table-id)))))
+ (testing "a table hidden through PUT /api/table/:id is left out"
+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]
+ (set-visibility-type-via-api! table-id "hidden")
+ (is (not (in-sync-tables? table-id)))))
+ (testing "a table hidden through the Data Studio data layer is left out"
+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]
+ (mt/user-http-request :crowberto :post 200 "data-studio/table/edit" {:table_ids [table-id] :data_layer "hidden"})
+ (is (not (in-sync-tables? table-id)))))
+ (testing "a hidden table takes part in sync again once the user un-hides it"
+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES"}]
+ (set-visibility-type-via-api! table-id "hidden")
+ (set-visibility-type-via-api! table-id nil)
+ (is (in-sync-tables? table-id))))
+ (testing "a cruft table takes part in sync once the user sets its visibility_type to NULL"
+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :name "VENUES" :visibility_type :cruft}]
+ (is (not (in-sync-tables? table-id)))
+ (set-visibility-type-via-api! table-id nil)
+ (is (= {:visibility_type nil :visibility_type_set true}
+ (t2/select-one [:model/TableUserSettings :visibility_type :visibility_type_set] :table_id table-id)))
+ (is (in-sync-tables? table-id))))))
diff --git a/test/metabase/warehouse_schema_overlay/core_test.clj b/test/metabase/warehouse_schema_overlay/core_test.clj
index 812c4a31e32..d7b7c31f0f9 100644
--- a/test/metabase/warehouse_schema_overlay/core_test.clj
+++ b/test/metabase/warehouse_schema_overlay/core_test.clj
@@ -1,9 +1,11 @@
(ns metabase.warehouse-schema-overlay.core-test
(:require
[clojure.test :refer :all]
+ [metabase.test :as mt]
[metabase.util.malli :as mu]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
- [metabase.warehouse-schema.schema]))
+ [metabase.warehouse-schema.schema]
+ [toucan2.core :as t2]))
(comment metabase.warehouse-schema.schema/keep-me)
@@ -16,3 +18,35 @@
(testing "the spelled-out column list keeps up with the Field schema it cannot require"
(is (= (into #{:id :unique_field_helper} (mu/map-schema-keys :metabase.warehouse-schema.schema/field.update))
warehouse-schema-overlay/field-columns))))
+
+(defn- visibility-types
+ "`[table-user-visibility-type, visibility_type read through table-query]` for the Table `table-id`."
+ [table-id]
+ [(t2/select-one-fn :v [(t2/table-name :model/Table)
+ [(warehouse-schema-overlay/table-user-visibility-type :metabase_table) :v]]
+ {:where [:= :metabase_table.id table-id]})
+ (some-> (t2/select-one-fn :visibility_type [:model/Table :visibility_type] :id table-id
+ {:from [(warehouse-schema-overlay/table-query)]})
+ name)])
+
+(deftest table-user-visibility-type-test
+ (testing "table-user-visibility-type matches the visibility_type table-query reads"
+ (mt/with-temp [:model/Table {plain :id} {}
+ :model/Table {cruft :id} {:visibility_type :cruft}
+ :model/Table {put-hidden :id} {}
+ :model/Table {data-studio :id} {}
+ :model/Table {cruft-cleared :id} {:visibility_type :cruft}
+ :model/Table {description-only :id} {:visibility_type :cruft}]
+ (mt/user-http-request :crowberto :put 200 (format "table/%d" put-hidden) {:visibility_type "hidden"})
+ (mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
+ {:table_ids [data-studio] :data_layer "hidden"})
+ (t2/insert! :model/TableUserSettings {:table_id cruft-cleared :visibility_type nil})
+ (t2/insert! :model/TableUserSettings {:table_id description-only :description "user description"})
+ (doseq [[desc table-id expected] [["no settings row" plain nil]
+ ["sync's cruft with no settings row" cruft "cruft"]
+ ["hidden through PUT /api/table/:id" put-hidden "hidden"]
+ ["data_layer hidden through Data Studio" data-studio "hidden"]
+ ["an explicit user NULL over sync's cruft" cruft-cleared nil]
+ ["a settings row with visibility_type unset over cruft" description-only "cruft"]]]
+ (testing desc
+ (is (= [expected expected] (visibility-types table-id))))))))