The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Track published tables in remote sync only through their user settings (#84564)
Alexander Polyankin0abf90cddde8
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.cljindex 357584c20d1..1060c8895e0 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj@@ -2,6 +2,7 @@"Application database queries for the remote-sync module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself."(:require+ [medley.core :as m][metabase-enterprise.remote-sync.schema :as remote-sync.schema][metabase.collections.schema :as collections.schema][metabase.lib.schema.id :as lib.schema.id]@@ -13,7 +14,7 @@(def ^:private ConditionKey"The column keys used in the `:conditions` / `:cascade-filter` / `:removal-conditions` of a remote-sync modelspec (see `metabase-enterprise.remote-sync.spec`)."- [:enum :exploration_id :built_in_type :active :entity_id :collection_id :model_id :archived :archived_at])+ [:enum :exploration_id :built_in_type :entity_id :collection_id :model_id :archived :archived_at])(def ^:private Conditions"A map of column to value (possibly nil) or Toucan 2 operator-vector value, or nil for none."@@ -35,14 +36,6 @@Collection."[:maybe [:or ms/PositiveInt [:= -1]]])-(def ^:private Path- "A `{:db_name :schema :table_name :field_name}` path used to locate a Table or Field."- [:map {:closed true}- [:db_name :string]- [:schema {:optional true} [:maybe :string]]- [:table_name :string]- [:field_name {:optional true} :string]])-(mu/defn eligible-children"The instances of `model-key` whose `fk` column equals `model-id`, additionally matching `conditions` (a map ofcolumn to value or Toucan 2 operator-vector value, or nil for none)."@@ -52,6 +45,13 @@conditions :- Conditions](apply t2/select model-key fk model-id (mapcat identity conditions)))+(mu/defn row-exists? :- :boolean+ "Whether a `model` row has `column` equal to `id`."+ [model :- :keyword+ column :- :keyword+ id :- ms/PositiveInt]+ (t2/exists? model column id))+(mu/defn ids-where"The IDs of the instances of `model-key` matching `conditions` (a map of column to value or Toucan 2operator-vector value, or nil for every instance)."@@ -217,15 +217,20 @@(t2/delete! model :id [:in ids]))(defn- tracking-select-parts- "The SELECT/FROM/JOIN joining a `model-key` instance (Field, Segment, or Measure) to its Table for sync tracking,- plus the `alias` its own table is joined under (so callers can address its `:id` and `:entity_id` columns).+ "The SELECT/FROM/JOIN of a `model-key` instance (TableUserSettings, FieldUserSettings, Dimension, Segment, or+ Measure) with its Table for sync tracking, plus the `alias` the row its id refers to is under (so callers can+ address its `:id` and `:entity_id` columns). The user settings are keyed by the id of their Table or Field.Segment and Measure share alias \"s\" — both are looked up the same way by [[tracking-details-by-entity-ids]]."[model-key](case model-key- :model/Field {:alias "f"- :select [:f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]- :from [[:metabase_field :f]]- :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :f.table_id :t.id]]}+ :model/TableUserSettings {:alias "t"+ :select [:t.name [:t.id :table_id] :t.collection_id [:t.name :table_name]]+ :from [(warehouse-schema-overlay/table-query {:alias :t})]}+ (:model/FieldUserSettings :model/Dimension)+ {:alias "f"+ :select [:f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]+ :from [[:metabase_field :f]]+ :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :f.table_id :t.id]]}:model/Segment {:alias "s":select [:s.name :s.table_id [:t.collection_id :collection_id] [:t.name :table_name]]:from [[:segment :s]]@@ -236,12 +241,13 @@:join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :s.table_id :t.id]]}))(mu/defn tracking-details-by-id- "The name, table id, collection id, and table name of the `model-key` (Field, Segment, or Measure) instance with- `model-id`, or nil."+ "The name, table id, collection id, and table name of the `model-key` (TableUserSettings, FieldUserSettings,+ Dimension, Segment, or Measure) instance with `model-id`, or nil."[model-key :- :keywordmodel-id :- ms/PositiveInt](let [{:keys [alias select from join]} (tracking-select-parts model-key)]- (first (t2/query {:select select :from from :join join :where [:= (keyword alias "id") model-id]}))))+ (first (t2/query (m/assoc-some {:select select :from from :where [:= (keyword alias "id") model-id]}+ :join join)))))(mu/defn tracking-details-by-entity-ids"The `:id`, name, table id, collection id, and table name of the `model-key` (Segment or Measure) instances with@@ -279,58 +285,11 @@entity-ids :- [:set :string]](t2/select-pks-vec model :entity_id [:in entity-ids]))-(defn- path-expr- "Matches the Tables (aliased `t` in a Database aliased `db`) at `paths`, and their Fields (aliased `f`) when- `has-field?`."- [paths has-field?]- (into [:or]- (for [path paths]- (let [{:keys [db_name schema table_name field_name]} path]- (cond-> [:and- [:= :db.name db_name]- (if schema [:= :t.schema schema] [:is :t.schema nil])- [:= :t.name table_name]]- (and has-field? field_name)- (conj [:= :f.name field_name]))))))--(mu/defn tables-at-paths- "The `:id`, `:name`, and `:collection_id` rows of the Tables at `paths` (`{:db_name :schema :table_name}`)."- [paths :- [:sequential Path]]- (t2/query {:select [:t.id :t.name :t.collection_id]- :from [[:metabase_table :t]]- :join [[:metabase_database :db] [:= :db.id :t.db_id]]- :where (path-expr paths false)}))--(mu/defn fields-at-paths- "The `:id`, `:name`, `:table_id`, `:collection_id`, and `:table_name` rows of the Fields at `paths`- (`{:db_name :schema :table_name :field_name}`)."- [paths :- [:sequential Path]]- (t2/query {:select [:f.id :f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]- :from [[:metabase_field :f]]- :join [[:metabase_table :t] [:= :t.id :f.table_id]- [:metabase_database :db] [:= :db.id :t.db_id]]- :where (path-expr paths true)}))-(mu/defn card-types"The `:id`, `:type`, and `:display` of the Cards with `card-ids`."[card-ids :- [:sequential ::lib.schema.id/card]](t2/select [:model/Card :id :type :display] :id [:in card-ids]))-(mu/defn table-user-settings-exist? :- :boolean- "Whether the Table with `table-id` has a TableUserSettings row."- [table-id :- ::lib.schema.id/table]- (t2/exists? :model/TableUserSettings :table_id table-id))--(mu/defn field-user-settings-exist? :- :boolean- "Whether the Field with `field-id` has a FieldUserSettings row."- [field-id :- ::lib.schema.id/field]- (t2/exists? :model/FieldUserSettings :field_id field-id))--(mu/defn dimension-exists-for-field? :- :boolean- "Whether the Field with `field-id` has a Dimension."- [field-id :- ::lib.schema.id/field]- (t2/exists? :model/Dimension :field_id field-id))-(mu/defn published-table-ids :- [:set ::lib.schema.id/table]"The ids of the Tables published in the Collections with `collection-ids`."[collection-ids :- [:sequential ::lib.schema.id/collection]]@@ -383,7 +342,7 @@(mu/defn fields-tracking-details"The `:id`, name, table id, collection id, and table name of the Fields with `field-ids`."[field-ids :- [:sequential ::lib.schema.id/field]]- (let [{:keys [select from join]} (tracking-select-parts :model/Field)]+ (let [{:keys [select from join]} (tracking-select-parts :model/FieldUserSettings)](t2/query {:select (into [:f.id] select) :from from :join join :where [:in :f.id field-ids]})))(mu/defn snippetsdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.cljindex 28928149361..972821626eb 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj@@ -11,7 +11,7 @@Tracked model types:- Card, Dashboard, Document, NativeQuerySnippet, Timeline, Collection- - Table and TableUserSettings (when published in a remote-synced collection)+ - TableUserSettings, keyed by its Table (when the Table is published in a remote-synced collection)- FieldUserSettings and Dimension, keyed by their Field (when its Table is published in a remote-synced collection)- Segment, Measure (when belonging to a published table in a remote-synced collection)- Transform, TransformTag, transforms-namespace Collections (when remote-sync-transforms setting is enabled)@@ -200,8 +200,9 @@"Re-checks `model-id`'s eligibility against current DB state. The eligibility that got us here was computedfrom an event payload, which a concurrent change may have invalidated in the meantime."[model-spec model-id]- (boolean (when-let [instance (remote-sync.db/instance (:model-key model-spec) model-id)]- (spec/check-eligibility model-spec instance))))+ (boolean (when-let [instance (remote-sync.db/instance (spec/source-model model-spec) model-id)]+ (and (spec/check-eligibility model-spec instance)+ (spec/row-exists? model-spec model-id)))))(defn- create-or-update-sync-object-from-spec!"Creates or updates a RemoteSyncObject entry using a spec for field hydration.@@ -268,22 +269,32 @@(when-let [ak (:archived-key child-spec)]{ak false})))+(defn- user-settings-child-ids+ "The ids of the Fields of the Table with `table-id` that have the `child-spec` user settings."+ [child-spec table-id]+ (case (:model-key child-spec)+ :model/FieldUserSettings (remote-sync.db/field-ids-with-user-settings [table-id])+ :model/Dimension (remote-sync.db/field-ids-with-dimensions [table-id])))+(defn- cascade-to-children!"When a parent model becomes eligible/ineligible, cascade to its children.For eligible: query child entities using :parent-fk and derived filter, check eligibility, create RSOs.For ineligible: query existing RSOs by model_table_id and mark as removed."[model-spec model-id status eligible?]- (doseq [child-spec (spec/children-specs (:model-key model-spec))]- (let [fk (:parent-fk child-spec)- filter (cascade-filter child-spec)]- (if eligible?- ;; Eligible branch: query actual entities and create RSOs for eligible children- (doseq [child (remote-sync.db/eligible-children (:model-key child-spec) fk model-id filter)]- (when (spec/check-eligibility child-spec child)- (create-or-update-sync-object-from-spec! child-spec (:id child) status)))- ;; Ineligible branch: mark existing child RSOs as removed- (doseq [child-rso (remote-sync.db/active-child-rsos (:model-type child-spec) model-id)]- (create-or-update-sync-object-from-spec! child-spec (:model_id child-rso) "removed"))))))+ (doseq [child-spec (spec/children-specs (:model-key model-spec))+ child-id (cond+ (and eligible? (= :user-settings (:identity child-spec)))+ (user-settings-child-ids child-spec model-id)++ eligible?+ (for [child (remote-sync.db/eligible-children (:model-key child-spec) (:parent-fk child-spec)+ model-id (cascade-filter child-spec))+ :when (spec/check-eligibility child-spec child)]+ (:id child))++ :else+ (map :model_id (remote-sync.db/active-child-rsos (:model-type child-spec) model-id)))]+ (create-or-update-sync-object-from-spec! child-spec child-id (if eligible? status "removed"))))(defn- handle-model-event-from-spec"Generic event handler that uses a spec for all configuration.@@ -291,7 +302,8 @@[model-spec topic {:keys [object]}](let [model-type (:model-type model-spec)model-id (:id object)- eligible? (spec/check-eligibility model-spec object)+ eligible? (and (spec/check-eligibility model-spec object)+ (spec/row-exists? model-spec model-id))existing-entry (remote-sync.db/rso model-type model-id)status (spec/determine-status model-spec topic object)](cond@@ -331,7 +343,7 @@;;; --------------------------------- Spec-based Event Registration (Non-Collection) ------------------------------------(doseq [[_model-key model-spec] (dissoc spec/remote-sync-specs :model/Collection :model/Field :model/Table)]+(doseq [[_model-key model-spec] (dissoc spec/remote-sync-specs :model/Collection)](register-events-for-spec! model-spec));;; ----------------------------------------- Collection Event Handler -------------------------------------------------@@ -421,58 +433,3 @@(do(log/infof "Collection %s no longer needs syncing, marking as removed" (:id object))(create-or-update-remote-sync-object-entry! "Collection" (:id object) "removed" hydrate-collection-details)))))--;;; ------------------------------------------- User Settings Tracking ---------------------------------------------------(def ^:private table-spec (get spec/remote-sync-specs :model/Table))-(def ^:private field-spec (get spec/remote-sync-specs :model/Field))--(defn- hydrate-table-user-settings-details- "The RemoteSyncObject details of the Table with `table-id`, as its own `:table_id`/`:table_name`."- [table-id]- (let [details (spec/hydrate-model-details table-spec table-id)]- (assoc details :table_id (:id details) :table_name (:name details))))--(defn- hydrate-field-details- "The RemoteSyncObject details of the Field with `field-id`."- [field-id]- (spec/hydrate-model-details field-spec field-id))--(defn- sync-user-settings!- "Track `model-type` `id` as updated when `eligible?` and `exists?` holds for it, else mark it removed when tracked."- [model-type id eligible? exists? hydrate-details]- (cond- (and eligible? (exists? id))- (create-or-update-remote-sync-object-entry! model-type id "update" hydrate-details)-- (remote-sync.db/rso-exists? model-type id)- (create-or-update-remote-sync-object-entry! model-type id "removed" hydrate-details)))--(events/derive! :event/field-update ::field-update-event)-(events/derive! ::field-update-event :metabase/event)--(methodical/defmethod events/publish-event! ::field-update-event- [_topic {:keys [object]}]- (let [eligible? (spec/check-eligibility field-spec object)]- (sync-user-settings! "FieldUserSettings" (:id object) eligible?- remote-sync.db/field-user-settings-exist? hydrate-field-details)- (sync-user-settings! "Dimension" (:id object) eligible?- remote-sync.db/dimension-exists-for-field? hydrate-field-details)))--(defn- handle-table-event!- "The generic Table handler plus the Table's TableUserSettings; one handler, since a second primary method on the- same events would displace the generic one."- [topic {:keys [object] :as event}]- (handle-model-event-from-spec table-spec topic event)- (sync-user-settings! "TableUserSettings" (:id object) (spec/check-eligibility table-spec object)- remote-sync.db/table-user-settings-exist? hydrate-table-user-settings-details))--(let [event-kws (spec/event-keywords table-spec)- parent-kw (:parent event-kws)]- (events/derive! parent-kw :metabase/event)- (doseq [[_event-type event-kw] (dissoc event-kws :parent)]- (events/derive! event-kw parent-kw))- (methodical/add-primary-method!- #'events/publish-event!- parent-kw- (fn [topic event] (handle-table-event! topic event))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 3087fedb117..b44bd721369 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -459,7 +459,7 @@(serialization/entity-file-path? path))(defn- pulled-change-count- "Total number of entities applied by a pull, across entity-id- and path-identified models in `imported-data`."+ "Total number of entities applied by a pull, across the entity-id models and the user settings in `imported-data`."[imported-data](transduce (map count) + 0 (concat (vals (:by-entity-id imported-data))(vals (:by-path imported-data)))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 4a931ce058b..2057b44e4a4 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -32,14 +32,17 @@Each spec contains:- :model-type - String name for RemoteSyncObject model_type column- :model-key - Toucan2 model keyword (e.g., :model/Card)- - :identity - Identity strategy: :entity-id, :path, or :hybrid- - :path-keys - For :path or :hybrid identity: vector of path components [:database :schema :table :field]- - :parent-model - For :parent-table eligibility: the parent model key to check eligibility against- (e.g., :model/Table for Field, Segment, Measure)- - :parent-fk - For child models: the FK column pointing to the parent (e.g., :table_id)+ - :identity - Identity strategy: :entity-id, :hybrid, or :user-settings (a Table's or Field's user+ settings, keyed by the Table or Field id and loaded and tracked by their own import code)+ - :path-keys - For :hybrid identity: vector of path components [:database :schema :table]+ - :source-model - Optional model whose instances the events carry and eligibility is checked on, when it is not+ :model-key (e.g., :model/Field for FieldUserSettings)+ - :row-key - Optional column of `model-key` holding the tracked id; the model is only tracked for an id+ with such a row (e.g., :field_id for FieldUserSettings)+ - :parent-model - The parent model key (e.g., :model/TableUserSettings for FieldUserSettings, Segment, Measure)+ - :parent-fk - For child models that cascade from their parent: the FK pointing to it (e.g., :table_id)- :cascade-filter - Optional map of additional filter conditions for cascade queries.Only needed when the filter differs from {archived-key false}.- E.g., Field needs {:active true} since it has no :archived-key.- :delete-after - Optional vector of model keys that must be deleted AFTER this model.Used to handle FK constraints during import cleanup. For example, if Cardhas :delete-after [:model/Collection], Card will be deleted before Collection.@@ -220,35 +223,56 @@:export-scope :root-collections ; query for root-level remote-synced + transforms-namespace collections:enabled? true}- :model/Table- {:model-type "Table"- :model-key :model/Table- :identity :path- :path-keys [:database :schema :table]+ :model/TableUserSettings+ {:model-type "TableUserSettings"+ :model-key :model/TableUserSettings+ :source-model :model/Table+ :identity :user-settings+ :row-key :table_id:events {:prefix :event/table:types [:create :update :delete :publish :unpublish]}:eligibility {:type :published-table}- :archived-key :archived_at ; tables use archived_at, not archived- :tracking {:select-fields [:name :collection_id]+ :archived-key :archived_at+ :tracking {:hydrate-query? true:field-mappings {:model_name :name:model_collection_id :collection_id- :model_table_id :id- :model_table_name :name}} ; self-reference+ :model_table_id :table_id+ :model_table_name :table_name}}:removal {:statuses #{"removed"}}:enabled? true}- :model/Field- {:model-type "Field"- :model-key :model/Field- :identity :path- :path-keys [:database :schema :table :field]- :parent-model :model/Table+ :model/FieldUserSettings+ {:model-type "FieldUserSettings"+ :model-key :model/FieldUserSettings+ :source-model :model/Field+ :identity :user-settings+ :row-key :field_id+ :parent-model :model/TableUserSettings:parent-fk :table_id- :cascade-filter {:active true}:events {:prefix :event/field:types [:create :update :delete]}:eligibility {:type :parent-table}- :archived-key nil ; fields don't have archived+ :archived-key nil+ :tracking {:hydrate-query? true+ :field-mappings {:model_name :name+ :model_collection_id :collection_id+ :model_table_id :table_id+ :model_table_name :table_name}}+ :removal {:statuses #{"removed"}}+ :enabled? true}++ :model/Dimension+ {:model-type "Dimension"+ :model-key :model/Dimension+ :source-model :model/Field+ :identity :user-settings+ :row-key :field_id+ :parent-model :model/TableUserSettings+ :parent-fk :table_id+ :events {:prefix :event/field+ :types [:create :update :delete]}+ :eligibility {:type :parent-table}+ :archived-key nil:tracking {:hydrate-query? true:field-mappings {:model_name :name:model_collection_id :collection_id@@ -262,7 +286,7 @@:model-key :model/Segment:identity :hybrid ; entity_id but needs table context for path:path-keys [:database :schema :table]- :parent-model :model/Table+ :parent-model :model/TableUserSettings:events {:prefix :event/segment:types [:create :update :delete]}:eligibility {:type :parent-table}@@ -280,7 +304,7 @@:model-key :model/Measure:identity :hybrid ; entity_id but needs table context for path:path-keys [:database :schema :table]- :parent-model :model/Table+ :parent-model :model/TableUserSettings:events {:prefix :event/measure:types [:create :update :delete]}:eligibility {:type :parent-table}@@ -889,16 +913,31 @@false))))-(defmethod check-eligibility-by-type :published-table- [_ {:keys [is_published collection_id]}]+(defn- published-in-remote-synced-collection?+ "Whether `table` is published in a remote-synced Collection."+ [{:keys [is_published collection_id]}](and is_published(collections/remote-synced-collection? collection_id)))+(defmethod check-eligibility-by-type :published-table+ [_ table]+ (published-in-remote-synced-collection? table))+(defmethod check-eligibility-by-type :parent-table- [{:keys [parent-model]} {:keys [table_id]}]+ [_ {:keys [table_id]}](when table_id- (when-let [table (remote-sync.db/instance parent-model table_id)]- (check-eligibility (spec-for-model-key parent-model) table))))+ (some-> (remote-sync.db/instance :model/Table table_id) published-in-remote-synced-collection?)))++(defn source-model+ "The model whose instances the spec's events carry and its eligibility is checked on."+ [spec]+ (or (:source-model spec) (:model-key spec)))++(defn row-exists?+ "Whether the spec's model has a row for `id`, which always holds for a spec without a `:row-key`."+ [{:keys [model-key row-key]} id]+ (or (nil? row-key)+ (remote-sync.db/row-exists? model-key row-key id)))(defmethod check-eligibility-by-type :setting[{:keys [eligibility]} _object]@@ -1060,14 +1099,9 @@id)id)))-(defmethod extract-identity-from-serdes-path :path+(defmethod extract-identity-from-serdes-path :user-settings[_ serdes-path _ingest-one]- (let [path-map (into {} (map (fn [elem] [(keyword (u/lower-case-en (:model elem))) (:id elem)]) serdes-path))]- (cond-> {}- (contains? path-map :database) (assoc :db_name (:database path-map))- (contains? path-map :schema) (assoc :schema (:schema path-map))- (contains? path-map :table) (assoc :table_name (:table path-map))- (contains? path-map :field) (assoc :field_name (:field path-map)))))+ serdes-path)(defmethod extract-identity-from-serdes-path :default[_ _ _]@@ -1076,7 +1110,7 @@(defn extract-imported-entities"Processes serdes paths from an import and extracts entity identities grouped by how they should be looked up.Returns a map with :by-entity-id containing entity_ids grouped by model type, and :by-path containing- path lookup maps for models like Table and Field that use path-based identity.+ the serdes paths of the user settings models, which are only counted.`ingest-one` (path -> ingested entity) is for the pre-load conflict check: a `:natural-key-paths?` path whoseid matches no local row yet resolves to the row the loader will match on the entity's natural key."@@ -1094,7 +1128,7 @@(:entity-id :hybrid)(update-in acc [:by-entity-id model-type] (fnil conj #{}) identity-data)- :path+ :user-settings(update-in acc [:by-path (:model-key spec)] (fnil conj []) identity-data)acc)@@ -1121,14 +1155,13 @@(defn event-keywords"Returns a map of event keywords for a spec: {:parent :create :update :delete}."- [{:keys [events]}]+ [{:keys [model-type events]}](let [{:keys [prefix types]} eventsns-str (namespace prefix)base (name prefix)](merge- ;; Parent keyword uses the event prefix name (e.g., :card -> card-change-event){:parent (keyword "metabase-enterprise.remote-sync.events"- (str base "-change-event"))}+ (str (u/->kebab-case-en model-type) "-change-event"))}(into {}(for [event-type types][event-type (keyword ns-str (str base "-" (name event-type)))])))))@@ -1153,7 +1186,6 @@Parameters:- spec: The model spec- data: For :entity-id/:hybrid - a collection of entity_ids- For :path - a collection of path maps {:db_name :schema :table_name [:field_name]}- timestamp: The sync timestamp"{:arglists '([spec data timestamp])}(fn [spec _data _timestamp] (:identity spec)))@@ -1201,39 +1233,6 @@:status_changed_at timestamp}mapped-fields))))))))-(defmethod query-entities-for-sync :path- [{:keys [model-type model-key]} paths timestamp]- (when (seq paths)- (case model-key- :model/Table- (->> (remote-sync.db/tables-at-paths paths)- (map (fn [{:keys [id name collection_id]}]- {:model_type model-type- :model_id id- :model_name name- :model_collection_id collection_id- :model_display nil- :model_table_id id- :model_table_name name- :status "synced"- :status_changed_at timestamp})))-- :model/Field- (->> (remote-sync.db/fields-at-paths paths)- (map (fn [{:keys [id name table_id table_name collection_id]}]- {:model_type model-type- :model_id id- :model_name name- :model_collection_id collection_id- :model_display nil- :model_table_id table_id- :model_table_name table_name- :status "synced"- :status_changed_at timestamp})))-- ;; Default for unknown path models- nil)))-(defmethod query-entities-for-sync :default[_ _ _]nil)@@ -1242,15 +1241,10 @@"Builds RemoteSyncObject entries for all imported entities based on their specs. Iterates over enabledspecs and queries the database to hydrate the fields needed for each sync object. Returns a sequenceof maps ready for insertion into the RemoteSyncObject table."- [timestamp {:keys [by-entity-id by-path]}]+ [timestamp {:keys [by-entity-id]}](into []- (for [[model-key spec] (enabled-specs)- :let [identity-type (:identity spec)- model-type (:model-type spec)- data (case identity-type- :entity-id (get by-entity-id model-type)- :path (get by-path model-key)- :hybrid (get by-entity-id model-type))]+ (for [[_model-key spec] (enabled-specs)+ :let [data (get by-entity-id (:model-type spec))]:when (seq data)entity (query-entities-for-sync spec data timestamp)]entity)))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.cljindex 138c8f29d0f..907c667f029 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj@@ -896,10 +896,10 @@(deftest ^:parallel snippet-event-derivation-test(testing "snippet events properly derive from :metabase/event"- (is (events/isa? ::remote-sync.events/snippet-change-event :metabase/event))- (is (events/isa? :event/snippet-create ::remote-sync.events/snippet-change-event))- (is (events/isa? :event/snippet-update ::remote-sync.events/snippet-change-event))- (is (events/isa? :event/snippet-delete ::remote-sync.events/snippet-change-event))))+ (is (events/isa? ::remote-sync.events/native-query-snippet-change-event :metabase/event))+ (is (events/isa? :event/snippet-create ::remote-sync.events/native-query-snippet-change-event))+ (is (events/isa? :event/snippet-update ::remote-sync.events/native-query-snippet-change-event))+ (is (events/isa? :event/snippet-delete ::remote-sync.events/native-query-snippet-change-event))))(deftest ^:parallel collection-event-derivation-test(testing "collection events properly derive from :metabase/event"@@ -1075,18 +1075,25 @@;;; Table Event Tests(deftest table-update-event-creates-entry-for-published-table-test- (testing "table-update event creates remote sync object entry for published table in remote-synced collection"+ (testing "table-update event creates remote sync object entry for published table in remote-synced collection,+ only once the table has a TableUserSettings row"(mt/with-temporary-setting-values [remote-sync-type :read-write](mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}:model/Table table {:name "Test Table":is_published true:collection_id (:id remote-sync-collection)}](t2/delete! :model/RemoteSyncObject)+ (events/publish-event! :event/table-update+ {:object table :user-id (mt/user->id :rasta)})+ (is (not (t2/exists? :model/RemoteSyncObject)))+ (t2/insert! :model/TableUserSettings {:table_id (:id table)+ :is_published true+ :collection_id (:id remote-sync-collection)})(events/publish-event! :event/table-update{:object table :user-id (mt/user->id :rasta)})(let [entries (t2/select :model/RemoteSyncObject)](is (= 1 (count entries)))- (is (=? {:model_type "Table"+ (is (=? {:model_type "TableUserSettings":model_id (:id table):status "update"}(first entries))))))))@@ -1122,13 +1129,16 @@:model/Table table {:name "Test Table":is_published true:collection_id (:id remote-sync-collection)}]- (track-as-synced! "Table" table)+ (t2/insert! :model/TableUserSettings {:table_id (:id table)+ :is_published true+ :collection_id (:id remote-sync-collection)})+ (track-as-synced! "TableUserSettings" table)(events/publish-event! :event/table-update{:object (assoc table :archived_at (t/offset-date-time)):user-id (mt/user->id :rasta)})(let [entries (t2/select :model/RemoteSyncObject)](is (= 1 (count entries)))- (is (=? {:model_type "Table"+ (is (=? {:model_type "TableUserSettings":model_id (:id table):status "delete"}(first entries))))))))@@ -1141,20 +1151,20 @@:collection_id (:id remote-sync-collection)}](t2/delete! :model/RemoteSyncObject);; First, create an entry with status "synced" directly- (t2/insert! :model/RemoteSyncObject {:model_type "Table"+ (t2/insert! :model/RemoteSyncObject {:model_type "TableUserSettings":model_id (:id table):model_name "Test Table":model_table_id (:id table):model_table_name "Test Table":status "synced":status_changed_at (t/offset-date-time)})- (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))](is (= "synced" (:status initial-entry)));; Now unpublish the table(events/publish-event! :event/table-update{:object (assoc table :is_published false):user-id (mt/user->id :rasta)})- (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))](is (= "removed" (:status update-entry))))))))(deftest table-moved-to-normal-collection-marks-as-removed-test@@ -1166,28 +1176,58 @@:collection_id (:id remote-sync-collection)}](t2/delete! :model/RemoteSyncObject);; First, create an entry with status "synced" directly- (t2/insert! :model/RemoteSyncObject {:model_type "Table"+ (t2/insert! :model/RemoteSyncObject {:model_type "TableUserSettings":model_id (:id table):model_name "Test Table":model_table_id (:id table):model_table_name "Test Table":status "synced":status_changed_at (t/offset-date-time)})- (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))](is (= "synced" (:status initial-entry)));; Now move to normal collection(events/publish-event! :event/table-update{:object (assoc table :collection_id (:id normal-collection)):user-id (mt/user->id :rasta)})- (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))](is (= "removed" (:status update-entry))))))))+(deftest table-publish-tracks-only-user-settings-test+ (testing "publishing a table tracks its TableUserSettings and the FieldUserSettings and Dimensions of its fields,+ never the Table or its Fields, and unpublishing marks them removed"+ (mt/with-temporary-setting-values [remote-sync-type :read-write]+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "Remote-Sync" :type "library-data"}+ :model/Table table {:name "T"}+ :model/Field curated {:name "curated" :table_id (:id table) :base_type :type/Text}+ :model/Field remapped {:name "remapped" :table_id (:id table) :base_type :type/Integer}+ :model/Field _plain {:name "plain" :table_id (:id table) :base_type :type/Text}+ :model/Dimension _ {:field_id (:id remapped) :name "Remapped" :type :internal}]+ (t2/insert! :model/FieldUserSettings {:field_id (:id curated) :description "curated"})+ (t2/insert! :model/TableUserSettings {:table_id (:id table) :is_published true :collection_id (:id coll)})+ (t2/delete! :model/RemoteSyncObject)+ (let [tracked (fn [] (into #{} (map (juxt :model_type :model_id :status)) (t2/select :model/RemoteSyncObject)))]+ (events/publish-event! :event/table-publish+ {:object (assoc table :is_published true :collection_id (:id coll))+ :user-id (mt/user->id :rasta)})+ (is (= #{["TableUserSettings" (:id table) "update"]+ ["FieldUserSettings" (:id curated) "update"]+ ["Dimension" (:id remapped) "update"]}+ (tracked)))+ (t2/update! :model/TableUserSettings (:id table) {:is_published false :collection_id nil})+ (events/publish-event! :event/table-unpublish+ {:object (assoc table :is_published false :collection_id nil)+ :user-id (mt/user->id :rasta)})+ (is (= #{["TableUserSettings" (:id table) "removed"]+ ["FieldUserSettings" (:id curated) "removed"]+ ["Dimension" (:id remapped) "removed"]}+ (tracked))))))))+(deftest ^:parallel table-event-derivation-test(testing "table events properly derive from :metabase/event"- (is (events/isa? ::remote-sync.events/table-change-event :metabase/event))- (is (events/isa? :event/table-create ::remote-sync.events/table-change-event))- (is (events/isa? :event/table-update ::remote-sync.events/table-change-event))- (is (events/isa? :event/table-delete ::remote-sync.events/table-change-event))))+ (is (events/isa? ::remote-sync.events/table-user-settings-change-event :metabase/event))+ (is (events/isa? :event/table-create ::remote-sync.events/table-user-settings-change-event))+ (is (events/isa? :event/table-update ::remote-sync.events/table-user-settings-change-event))+ (is (events/isa? :event/table-delete ::remote-sync.events/table-user-settings-change-event))));;; Segment Event Testsdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex bac1af0a951..089c96b9abb 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -812,7 +812,7 @@:location "/"}](t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}- {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])+ {:model_type "TableUserSettings" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])(let [initial-files {"main" {"databases/test_db/tables/test_table/test_table.yaml"(test-helpers/generate-table-yaml "test-table" "test-db")}}mock-source (test-helpers/create-mock-source :initial-files initial-files)@@ -854,7 +854,7 @@"Removed segment files should be deleted after export")))))))))(deftest export!-updates-removed-table-entries-to-synced-test- (testing "export! updates RemoteSyncObject entries for 'removed' Tables/Fields/Segments to 'synced' status after successful export"+ (testing "export! updates 'removed' Segment entries to 'synced' and drops 'removed' TableUserSettings/FieldUserSettings entries after successful export"(mt/with-model-cleanup [:model/RemoteSyncObject :model/Segment](mt/with-temporary-setting-values [remote-sync-type :read-write](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]@@ -874,23 +874,23 @@:entity_id "test-segment-xxxxxxxx"}](t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}- {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}- {:model_type "Field" :model_id field-id :model_name "test-field" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}+ {:model_type "TableUserSettings" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}+ {:model_type "FieldUserSettings" :model_id field-id :model_name "test-field" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}{:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])(is (= 4 (t2/count :model/RemoteSyncObject)))(let [mock-source (test-helpers/create-mock-source)result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")](is (= :success (:status result)))- (is (= 4 (t2/count :model/RemoteSyncObject))- "All RemoteSyncObject entries should remain after export")- (let [table-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id table-id)- field-entry (t2/select-one :model/RemoteSyncObject :model_type "Field" :model_id field-id)+ (is (= 2 (t2/count :model/RemoteSyncObject))+ "Only the Collection and Segment entries should remain after export")+ (let [table-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id table-id)+ field-entry (t2/select-one :model/RemoteSyncObject :model_type "FieldUserSettings" :model_id field-id)segment-entry (t2/select-one :model/RemoteSyncObject :model_type "Segment" :model_id segment-id)coll-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)]- (is (= "synced" (:status table-entry))- "Table entry should be updated to synced after export")- (is (= "synced" (:status field-entry))- "Field entry should be updated to synced after export")+ (is (nil? table-entry)+ "TableUserSettings entry should be dropped after export")+ (is (nil? field-entry)+ "FieldUserSettings entry should be dropped after export")(is (= "synced" (:status segment-entry))"Segment entry should be updated to synced after export")(is (= "synced" (:status coll-entry))@@ -910,7 +910,7 @@:location "/"}](t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}- {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])+ {:model_type "TableUserSettings" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])(let [initial-files {"main" {"databases/test_db/schemas/public/tables/test_table/test_table.yaml"(test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}mock-source (test-helpers/create-mock-source :initial-files initial-files)@@ -997,58 +997,28 @@;; Import Table/Field/Segment tracking tests-(deftest import!-tracks-tables-in-remote-sync-object-test- (testing "import! creates RemoteSyncObject entries for imported Tables"+(deftest import!-does-not-track-tables-or-fields-test+ (testing "import! of Table and Field files creates no RemoteSyncObject entries for them, which are not synced"(mt/with-model-cleanup [:model/RemoteSyncObject](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Database {db-id :id} {:name "test-db"}- :model/Table {table-id :id} {:name "test-table" :db_id db-id}- :model/Collection {coll-id :id}- {:name "Test Collection"- :is_remote_synced true- :entity_id "test-collection-1xxxx"- :location "/"}]- (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"- (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")- "databases/test_db/tables/test_table/test_table.yaml"- (test-helpers/generate-table-yaml "test-table" "test-db")}}- mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result)))- (is (t2/exists? :model/RemoteSyncObject- :model_type "Collection"- :model_id coll-id- :status "synced")- "Collection should be tracked in RemoteSyncObject")- (is (t2/exists? :model/RemoteSyncObject- :model_type "Table"- :model_id table-id- :status "synced")- "Table should be tracked in RemoteSyncObject")))))))--(deftest import!-tracks-fields-in-remote-sync-object-test- (testing "import! creates RemoteSyncObject entries for imported Fields"- (mt/with-model-cleanup [:model/RemoteSyncObject]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}- :model/Table {table-id :id} {:name "test-table" :db_id db-id}- :model/Field {field-id :id} {:name "test-field" :table_id table-id :base_type :type/Text}- :model/Collection _ {:name "Test Collection"- :is_remote_synced true- :entity_id "test-collection-1xxxx"- :location "/"}]+ :model/Table {table-id :id} {:name "test-table" :db_id db-id :schema "PUBLIC"}+ :model/Field _ {:name "test-field" :table_id table-id :base_type :type/Text}+ :model/Collection {coll-id :id} {:name "Test Collection"+ :is_remote_synced true+ :entity_id "test-collection-1xxxx"+ :location "/"}](let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")- "databases/test_db/tables/test_table/fields/test_field.yaml"- (test-helpers/generate-field-yaml "test-field" "test-table" "test-db")}}+ "databases/test_db/schemas/PUBLIC/tables/test_table/test_table.yaml"+ (test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")+ "databases/test_db/schemas/PUBLIC/tables/test_table/fields/test_field.yaml"+ (test-helpers/generate-field-yaml "test-field" "test-table" "test-db" :schema "PUBLIC")}}mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- (is (t2/exists? :model/RemoteSyncObject- :model_type "Field"- :model_id field-id- :status "synced")- "Field should be tracked in RemoteSyncObject")))))))+ (is (= [["Collection" coll-id "synced"]]+ (t2/select-fn-vec (juxt :model_type :model_id :status) :model/RemoteSyncObject)))))))))(deftest import!-tracks-segments-in-remote-sync-object-test(testing "import! creates RemoteSyncObject entries for imported Segments"@@ -1086,29 +1056,6 @@:status "synced")"Segment should be tracked in RemoteSyncObject")))))))-(deftest import!-tracks-tables-with-schema-in-remote-sync-object-test- (testing "import! creates RemoteSyncObject entries for imported Tables with schema"- (mt/with-model-cleanup [:model/RemoteSyncObject]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}- :model/Table {table-id :id} {:name "test-table" :db_id db-id :schema "PUBLIC"}- :model/Collection _ {:name "Test Collection"- :is_remote_synced true- :entity_id "test-collection-1xxxx"- :location "/"}]- (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"- (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")- "databases/test_db/schemas/PUBLIC/tables/test_table/test_table.yaml"- (test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}- mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result)))- (is (t2/exists? :model/RemoteSyncObject- :model_type "Table"- :model_id table-id- :status "synced")- "Table with schema should be tracked in RemoteSyncObject")))))))-(deftest import!-includes-actions-attached-to-models-test(testing "import! successfully imports Actions attached to Models in synced collections"(mt/with-model-cleanup [:model/RemoteSyncObject :model/Action]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.cljindex abd8603642d..6c0f548d04b 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj@@ -27,16 +27,16 @@(deftest all-specs-have-valid-identity-test(testing "Every spec has a valid identity type"- (let [valid-identity-types #{:entity-id :path :hybrid}]+ (let [valid-identity-types #{:entity-id :hybrid :user-settings}](doseq [[model-key spec] spec/remote-sync-specs](testing (str "Spec for " model-key)(is (contains? valid-identity-types (:identity spec))(str "Invalid identity type: " (:identity spec))))))))(deftest path-based-specs-have-path-keys-test- (testing "Specs with :path or :hybrid identity have :path-keys"+ (testing "Specs with :hybrid identity have :path-keys"(doseq [[model-key spec] spec/remote-sync-specs- :when (#{:path :hybrid} (:identity spec))]+ :when (= :hybrid (:identity spec))](testing (str "Spec for " model-key)(is (vector? (:path-keys spec))"path-keys should be a vector")@@ -96,11 +96,9 @@(when-let [cf (:cascade-filter spec)](is (map? cf)":cascade-filter should be a map when present")))))- (testing "children-specs derives the correct children for Table"- (let [children (spec/children-specs :model/Table)]- (is (= 1 (count children)))- (is (= #{:model/Field}- (into #{} (map :model-key) children)))))+ (testing "children-specs derives the cascading children of TableUserSettings"+ (is (= #{:model/FieldUserSettings :model/Dimension}+ (into #{} (map :model-key) (spec/children-specs :model/TableUserSettings)))))(testing "children-specs returns empty for models with no children"(is (empty? (spec/children-specs :model/Card)))))@@ -110,7 +108,9 @@(testing "spec-for-model-type returns correct spec"(is (= :model/Card (:model-key (spec/spec-for-model-type "Card"))))(is (= :model/Dashboard (:model-key (spec/spec-for-model-type "Dashboard"))))- (is (= :model/Table (:model-key (spec/spec-for-model-type "Table"))))+ (is (= :model/TableUserSettings (:model-key (spec/spec-for-model-type "TableUserSettings"))))+ (is (nil? (spec/spec-for-model-type "Table")))+ (is (nil? (spec/spec-for-model-type "Field")))(is (= :model/Measure (:model-key (spec/spec-for-model-type "Measure"))))(is (nil? (spec/spec-for-model-type "NonExistent")))))@@ -127,8 +127,9 @@(is (set? types))(is (contains? types "Card"))(is (contains? types "Dashboard"))- (is (contains? types "Table"))- (is (contains? types "Field"))+ (is (contains? types "TableUserSettings"))+ (is (contains? types "FieldUserSettings"))+ (is (contains? types "Dimension"))(is (contains? types "Segment"))(is (contains? types "Measure"))(is (contains? types "Transform"))@@ -137,24 +138,22 @@(is (contains? types "Glossary"))(is (contains? types "Action"))(is (contains? types "DataApp"))- (is (= 17 (count types))))))+ (is (= 18 (count types))))))(deftest specs-by-identity-type-test(testing "specs-by-identity-type filters correctly"(mt/with-temporary-setting-values [remote-sync-transforms false](let [entity-id-specs (spec/specs-by-identity-type :entity-id)- path-specs (spec/specs-by-identity-type :path)+ user-settings-specs (spec/specs-by-identity-type :user-settings)hybrid-specs (spec/specs-by-identity-type :hybrid)](testing "entity-id specs"(is (contains? entity-id-specs :model/Card))(is (contains? entity-id-specs :model/Dashboard))(is (contains? entity-id-specs :model/Collection))- (is (not (contains? entity-id-specs :model/Table)))- (is (not (contains? entity-id-specs :model/Field))))- (testing "path specs"- (is (contains? path-specs :model/Table))- (is (contains? path-specs :model/Field))- (is (not (contains? path-specs :model/Card))))+ (is (not (contains? entity-id-specs :model/TableUserSettings))))+ (testing "user settings specs"+ (is (= #{:model/TableUserSettings :model/FieldUserSettings :model/Dimension}+ (set (keys user-settings-specs)))))(testing "hybrid specs"(is (contains? hybrid-specs :model/Segment))(is (contains? hybrid-specs :model/Measure))diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsxindex 6883a1f179b..e92a82d34b8 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx@@ -348,7 +348,7 @@ describe("AllChangesView", () => {const tableEntity = createMockRemoteSyncEntity({id: 100,name: "Orders Table",- model: "table",+ model: "tableusersettings",collection_id: 1,sync_status: "update",});@@ -415,7 +415,7 @@ describe("AllChangesView", () => {const fieldEntity = createMockRemoteSyncEntity({id: 400,name: "email",- model: "field",+ model: "fieldusersettings",collection_id: 1,sync_status: "update",table_id: 100,@@ -442,6 +442,54 @@ describe("AllChangesView", () => {});});+ describe("table user settings", () => {+ it("should link a table's user settings and its fields' user settings and dimensions to Data Studio", async () => {+ const tableEntity = createMockRemoteSyncEntity({+ id: 100,+ name: "Orders Table",+ model: "tableusersettings",+ collection_id: 1,+ sync_status: "update",+ table_id: 100,+ table_name: "Orders Table",+ });+ const fieldEntity = createMockRemoteSyncEntity({+ id: 400,+ name: "email",+ model: "fieldusersettings",+ collection_id: 1,+ sync_status: "update",+ table_id: 100,+ table_name: "Orders Table",+ });+ const dimensionEntity = createMockRemoteSyncEntity({+ id: 401,+ name: "user_id",+ model: "dimension",+ collection_id: 1,+ sync_status: "update",+ table_id: 100,+ table_name: "Orders Table",+ });++ setup({+ entities: [tableEntity, fieldEntity, dimensionEntity],+ });++ expect(+ await screen.findByRole("link", { name: /Orders Table/ }),+ ).toHaveAttribute("href", "/data-studio/library/tables/100");+ expect(screen.getByRole("link", { name: /email/ })).toHaveAttribute(+ "href",+ "/data-studio/library/tables/100/fields/400",+ );+ expect(screen.getByRole("link", { name: /user_id/ })).toHaveAttribute(+ "href",+ "/data-studio/library/tables/100/fields/401",+ );+ });+ });+describe("snippet collections", () => {it("should display snippets in snippet collections with hierarchy", async () => {const snippetCollection = createMockCollection({diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsxindex 30dca428dcb..74ccae6f1d1 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx@@ -19,6 +19,14 @@ function getEntityUrl(entity: RemoteSyncEntity): string | null {if (entity.model === "transformtest") {return null;}+ if (entity.model === "tableusersettings") {+ return Urls.dataStudioTable(entity.id);+ }+ if (entity.model === "fieldusersettings" || entity.model === "dimension") {+ return entity.table_id != null+ ? Urls.dataStudioTableFields(entity.table_id, entity.id)+ : null;+ }if (entity.model === "action") {if (entity.card_id === undefined) {return null;@@ -33,7 +41,10 @@ function getEntityUrl(entity: RemoteSyncEntity): string | null {export const EntityLink = ({ entity }: EntityLinkProps) => {const getIcon = useGetIcon();const entityIcon = useMemo((): IconData => {- if (entity.model === "field") {+ if (entity.model === "tableusersettings") {+ return { name: "table" };+ }+ if (entity.model === "fieldusersettings" || entity.model === "dimension") {return { name: "field" };}diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.tsindex d0930da4eaf..9d93302aeb1 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts@@ -116,7 +116,13 @@ const displayGroupSpecs: DisplayGroupSpec[] = [},{id: "tables",- models: new Set(["table", "field", "segment", "measure"]),+ models: new Set([+ "tableusersettings",+ "fieldusersettings",+ "dimension",+ "segment",+ "measure",+ ]),icon: "synced_collection",priority: 50,},@@ -341,7 +347,8 @@ const getSpecById = (id: string): DisplayGroupSpec | undefined => {* Models that are children of tables (get their collection from a parent table)*/const TABLE_CHILD_MODELS: Set<RemoteSyncEntityModel> = new Set([- "field",+ "fieldusersettings",+ "dimension","segment","measure",]);@@ -403,7 +410,7 @@ const sortEntitiesByStatus = (* dirty tables and orphan children (children whose parent table is not dirty).*/const buildTableGroups = (items: RemoteSyncEntity[]): TableGroup[] => {- const tables = items.filter((item) => item.model === "table");+ const tables = items.filter((item) => item.model === "tableusersettings");const tableChildren = items.filter((item) => isTableChildModel(item.model));const childrenByTableId = _.groupBy(tableChildren, (e) => e.table_id || 0);const tableIds = new Set(tables.map((tbl) => tbl.id));@@ -446,7 +453,7 @@ const filterOtherItems = (return false;}return (- item.model !== "table" &&+ item.model !== "tableusersettings" &&!isTableChildModel(item.model) &&!(item.model === "collection" && item.id === Number(collectionId)));diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.tsindex a65bedbd9c4..0f4ae3758a6 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts@@ -225,8 +225,12 @@ describe("remote_sync utils", () => {});describe("isTableChildModel", () => {- it("should return true for field model", () => {- expect(isTableChildModel("field")).toBe(true);+ it("should return true for field user settings model", () => {+ expect(isTableChildModel("fieldusersettings")).toBe(true);+ });++ it("should return true for dimension model", () => {+ expect(isTableChildModel("dimension")).toBe(true);});it("should return true for segment model", () => {@@ -241,8 +245,8 @@ describe("remote_sync utils", () => {expect(isTableChildModel("card")).toBe(false);});- it("should return false for table model", () => {- expect(isTableChildModel("table")).toBe(false);+ it("should return false for table user settings model", () => {+ expect(isTableChildModel("tableusersettings")).toBe(false);});it("should return false for dashboard model", () => {diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.tsindex 5bb1e1438d6..fe418e2096c 100644--- a/frontend/src/metabase-types/api/remote-sync.ts+++ b/frontend/src/metabase-types/api/remote-sync.ts@@ -12,8 +12,9 @@ export type RemoteSyncEntityModel =| "collection"| "document"| "nativequerysnippet"- | "table"- | "field"+ | "tableusersettings"+ | "fieldusersettings"+ | "dimension"| "segment"| "measure"| "transform"diff --git a/resources/migrations/065/20261016_remote_sync_object_user_settings.yaml b/resources/migrations/065/20261016_remote_sync_object_user_settings.yamlnew file mode 100644index 00000000000..0284b97a93a--- /dev/null+++ b/resources/migrations/065/20261016_remote_sync_object_user_settings.yaml@@ -0,0 +1,42 @@+## Remote sync now tracks a published table through its TableUserSettings, FieldUserSettings, and Dimensions+## rather than its Table and Fields. A Table or Field row whose table or field has a user settings row, and no row of+## that user settings yet, becomes that user settings row, keeping its pending status so unpushed changes are still+## pushed; the rest are deleted after.++databaseChangeLog:+ - changeSet:+ id: v65.2026-10-16T00:00:00+ author: ranquild+ comment: Turn the RemoteSyncObject rows of Tables with user settings into TableUserSettings rows+ changes:+ - sql:+ sql: >-+ UPDATE remote_sync_object SET model_type = 'TableUserSettings'+ WHERE model_type = 'Table'+ AND EXISTS (SELECT 1 FROM metabase_table_user_settings tus WHERE tus.table_id = remote_sync_object.model_id)+ AND NOT EXISTS (SELECT 1 FROM (SELECT model_id FROM remote_sync_object WHERE model_type = 'TableUserSettings') rso+ WHERE rso.model_id = remote_sync_object.model_id)+ rollback: []++ - changeSet:+ id: v65.2026-10-16T00:00:01+ author: ranquild+ comment: Turn the RemoteSyncObject rows of Fields with user settings into FieldUserSettings rows+ changes:+ - sql:+ sql: >-+ UPDATE remote_sync_object SET model_type = 'FieldUserSettings'+ WHERE model_type = 'Field'+ AND EXISTS (SELECT 1 FROM metabase_field_user_settings fus WHERE fus.field_id = remote_sync_object.model_id)+ AND NOT EXISTS (SELECT 1 FROM (SELECT model_id FROM remote_sync_object WHERE model_type = 'FieldUserSettings') rso+ WHERE rso.model_id = remote_sync_object.model_id)+ rollback: []++ - changeSet:+ id: v65.2026-10-16T00:00:02+ author: ranquild+ comment: Delete the remaining RemoteSyncObject rows of Tables and Fields+ changes:+ - sql:+ sql: DELETE FROM remote_sync_object WHERE model_type IN ('Table', 'Field')+ rollback: []diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.cljindex 56685ac5187..51b5f59a8c9 100644--- a/test/metabase/app_db/schema_migrations_test.clj+++ b/test/metabase/app_db/schema_migrations_test.clj@@ -3954,3 +3954,64 @@(is (t2/exists? :data_app :id app-id))(is (t2/exists? :permissions_group :id ordinary-group))(is (empty? (t2/select :data_app_group_assignment)))))))++(deftest remote-sync-object-user-settings-test+ (testing "v65.2026-10-16T00:00:00-02: the RemoteSyncObject rows of Tables and Fields with user settings become their+ user settings' rows, keeping their status, unless those are already tracked, and the rest are deleted"+ (impl/test-migrations ["v65.2026-10-16T00:00:00" "v65.2026-10-16T00:00:02"] [migrate!]+ (let [db-id (t2/insert-returning-pk! :metabase_database {:name "Remote Sync User Settings Test DB"+ :engine "h2"+ :created_at :%now+ :updated_at :%now+ :details "{}"})+ table! (fn [table-name]+ (t2/insert-returning-pk! :metabase_table {:active true+ :db_id db-id+ :name table-name+ :created_at :%now+ :updated_at :%now}))+ field! (fn [table-id field-name]+ (t2/insert-returning-pk! :metabase_field {:table_id table-id+ :name field-name+ :display_name field-name+ :base_type "type/Text"+ :database_type "TEXT"+ :position 0+ :database_position 0+ :active true+ :preview_display true+ :created_at :%now+ :updated_at :%now}))+ rso! (fn [model-type model-id status]+ (t2/insert-returning-pk! :remote_sync_object {:model_type model-type+ :model_id model-id+ :model_name model-type+ :status status+ :status_changed_at :%now+ :file_path (str model-type "/" model-id ".yaml")}))+ published (table! "published")+ unpublished (table! "unpublished")+ tracked (table! "tracked")+ edited (field! published "edited")+ unedited (field! published "unedited")+ tracked-fld (field! tracked "tracked")]+ (t2/insert! :metabase_table_user_settings [{:table_id published :created_at :%now :updated_at :%now}+ {:table_id tracked :created_at :%now :updated_at :%now}])+ (t2/insert! :metabase_field_user_settings [{:field_id edited :created_at :%now :updated_at :%now}+ {:field_id tracked-fld :created_at :%now :updated_at :%now}])+ (let [table-rso (rso! "Table" published "create")+ field-rso (rso! "Field" edited "update")+ dropped [(rso! "Table" unpublished "update") (rso! "Field" unedited "update")+ (rso! "Table" tracked "update") (rso! "Field" tracked-fld "update")]+ kept [(rso! "TableUserSettings" tracked "update") (rso! "FieldUserSettings" tracked-fld "update")]+ card-rso (rso! "Card" 1 "update")]+ (migrate!)+ (is (=? {:model_type "TableUserSettings" :model_id published :status "create"}+ (t2/select-one :remote_sync_object :id table-rso)))+ (is (=? {:model_type "FieldUserSettings" :model_id edited :status "update"}+ (t2/select-one :remote_sync_object :id field-rso)))+ (is (not (t2/exists? :remote_sync_object :id [:in dropped])))+ (is (= (set kept)+ (set (concat (t2/select-pks-vec :remote_sync_object :model_type "TableUserSettings" :model_id tracked)+ (t2/select-pks-vec :remote_sync_object :model_type "FieldUserSettings" :model_id tracked-fld)))))+ (is (t2/exists? :remote_sync_object :id card-rso)))))))