Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Track published tables in remote sync only through their user settings (#84564)

Alexander Polyankin0abf90cddde8
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index 357584c20d1..1060c8895e0 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the remote-sync module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself."
(:require
+ [medley.core :as m]
[metabase-enterprise.remote-sync.schema :as remote-sync.schema]
[metabase.collections.schema :as collections.schema]
[metabase.lib.schema.id :as lib.schema.id]
@@ -13,7 +14,7 @@
(def ^:private ConditionKey
"The column keys used in the `:conditions` / `:cascade-filter` / `:removal-conditions` of a remote-sync model
spec (see `metabase-enterprise.remote-sync.spec`)."
- [:enum :exploration_id :built_in_type :active :entity_id :collection_id :model_id :archived :archived_at])
+ [:enum :exploration_id :built_in_type :entity_id :collection_id :model_id :archived :archived_at])
(def ^:private Conditions
"A map of column to value (possibly nil) or Toucan 2 operator-vector value, or nil for none."
@@ -35,14 +36,6 @@
Collection."
[:maybe [:or ms/PositiveInt [:= -1]]])
-(def ^:private Path
- "A `{:db_name :schema :table_name :field_name}` path used to locate a Table or Field."
- [:map {:closed true}
- [:db_name :string]
- [:schema {:optional true} [:maybe :string]]
- [:table_name :string]
- [:field_name {:optional true} :string]])
-
(mu/defn eligible-children
"The instances of `model-key` whose `fk` column equals `model-id`, additionally matching `conditions` (a map of
column to value or Toucan 2 operator-vector value, or nil for none)."
@@ -52,6 +45,13 @@
conditions :- Conditions]
(apply t2/select model-key fk model-id (mapcat identity conditions)))
+(mu/defn row-exists? :- :boolean
+ "Whether a `model` row has `column` equal to `id`."
+ [model :- :keyword
+ column :- :keyword
+ id :- ms/PositiveInt]
+ (t2/exists? model column id))
+
(mu/defn ids-where
"The IDs of the instances of `model-key` matching `conditions` (a map of column to value or Toucan 2
operator-vector value, or nil for every instance)."
@@ -217,15 +217,20 @@
(t2/delete! model :id [:in ids]))
(defn- tracking-select-parts
- "The SELECT/FROM/JOIN joining a `model-key` instance (Field, Segment, or Measure) to its Table for sync tracking,
- plus the `alias` its own table is joined under (so callers can address its `:id` and `:entity_id` columns).
+ "The SELECT/FROM/JOIN of a `model-key` instance (TableUserSettings, FieldUserSettings, Dimension, Segment, or
+ Measure) with its Table for sync tracking, plus the `alias` the row its id refers to is under (so callers can
+ address its `:id` and `:entity_id` columns). The user settings are keyed by the id of their Table or Field.
Segment and Measure share alias \"s\" — both are looked up the same way by [[tracking-details-by-entity-ids]]."
[model-key]
(case model-key
- :model/Field {:alias "f"
- :select [:f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
- :from [[:metabase_field :f]]
- :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :f.table_id :t.id]]}
+ :model/TableUserSettings {:alias "t"
+ :select [:t.name [:t.id :table_id] :t.collection_id [:t.name :table_name]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t})]}
+ (:model/FieldUserSettings :model/Dimension)
+ {:alias "f"
+ :select [:f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
+ :from [[:metabase_field :f]]
+ :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :f.table_id :t.id]]}
:model/Segment {:alias "s"
:select [:s.name :s.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
:from [[:segment :s]]
@@ -236,12 +241,13 @@
:join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :s.table_id :t.id]]}))
(mu/defn tracking-details-by-id
- "The name, table id, collection id, and table name of the `model-key` (Field, Segment, or Measure) instance with
- `model-id`, or nil."
+ "The name, table id, collection id, and table name of the `model-key` (TableUserSettings, FieldUserSettings,
+ Dimension, Segment, or Measure) instance with `model-id`, or nil."
[model-key :- :keyword
model-id :- ms/PositiveInt]
(let [{:keys [alias select from join]} (tracking-select-parts model-key)]
- (first (t2/query {:select select :from from :join join :where [:= (keyword alias "id") model-id]}))))
+ (first (t2/query (m/assoc-some {:select select :from from :where [:= (keyword alias "id") model-id]}
+ :join join)))))
(mu/defn tracking-details-by-entity-ids
"The `:id`, name, table id, collection id, and table name of the `model-key` (Segment or Measure) instances with
@@ -279,58 +285,11 @@
entity-ids :- [:set :string]]
(t2/select-pks-vec model :entity_id [:in entity-ids]))
-(defn- path-expr
- "Matches the Tables (aliased `t` in a Database aliased `db`) at `paths`, and their Fields (aliased `f`) when
- `has-field?`."
- [paths has-field?]
- (into [:or]
- (for [path paths]
- (let [{:keys [db_name schema table_name field_name]} path]
- (cond-> [:and
- [:= :db.name db_name]
- (if schema [:= :t.schema schema] [:is :t.schema nil])
- [:= :t.name table_name]]
- (and has-field? field_name)
- (conj [:= :f.name field_name]))))))
-
-(mu/defn tables-at-paths
- "The `:id`, `:name`, and `:collection_id` rows of the Tables at `paths` (`{:db_name :schema :table_name}`)."
- [paths :- [:sequential Path]]
- (t2/query {:select [:t.id :t.name :t.collection_id]
- :from [[:metabase_table :t]]
- :join [[:metabase_database :db] [:= :db.id :t.db_id]]
- :where (path-expr paths false)}))
-
-(mu/defn fields-at-paths
- "The `:id`, `:name`, `:table_id`, `:collection_id`, and `:table_name` rows of the Fields at `paths`
- (`{:db_name :schema :table_name :field_name}`)."
- [paths :- [:sequential Path]]
- (t2/query {:select [:f.id :f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
- :from [[:metabase_field :f]]
- :join [[:metabase_table :t] [:= :t.id :f.table_id]
- [:metabase_database :db] [:= :db.id :t.db_id]]
- :where (path-expr paths true)}))
-
(mu/defn card-types
"The `:id`, `:type`, and `:display` of the Cards with `card-ids`."
[card-ids :- [:sequential ::lib.schema.id/card]]
(t2/select [:model/Card :id :type :display] :id [:in card-ids]))
-(mu/defn table-user-settings-exist? :- :boolean
- "Whether the Table with `table-id` has a TableUserSettings row."
- [table-id :- ::lib.schema.id/table]
- (t2/exists? :model/TableUserSettings :table_id table-id))
-
-(mu/defn field-user-settings-exist? :- :boolean
- "Whether the Field with `field-id` has a FieldUserSettings row."
- [field-id :- ::lib.schema.id/field]
- (t2/exists? :model/FieldUserSettings :field_id field-id))
-
-(mu/defn dimension-exists-for-field? :- :boolean
- "Whether the Field with `field-id` has a Dimension."
- [field-id :- ::lib.schema.id/field]
- (t2/exists? :model/Dimension :field_id field-id))
-
(mu/defn published-table-ids :- [:set ::lib.schema.id/table]
"The ids of the Tables published in the Collections with `collection-ids`."
[collection-ids :- [:sequential ::lib.schema.id/collection]]
@@ -383,7 +342,7 @@
(mu/defn fields-tracking-details
"The `:id`, name, table id, collection id, and table name of the Fields with `field-ids`."
[field-ids :- [:sequential ::lib.schema.id/field]]
- (let [{:keys [select from join]} (tracking-select-parts :model/Field)]
+ (let [{:keys [select from join]} (tracking-select-parts :model/FieldUserSettings)]
(t2/query {:select (into [:f.id] select) :from from :join join :where [:in :f.id field-ids]})))
(mu/defn snippets
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
index 28928149361..972821626eb 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
@@ -11,7 +11,7 @@
Tracked model types:
- Card, Dashboard, Document, NativeQuerySnippet, Timeline, Collection
- - Table and TableUserSettings (when published in a remote-synced collection)
+ - TableUserSettings, keyed by its Table (when the Table is published in a remote-synced collection)
- FieldUserSettings and Dimension, keyed by their Field (when its Table is published in a remote-synced collection)
- Segment, Measure (when belonging to a published table in a remote-synced collection)
- Transform, TransformTag, transforms-namespace Collections (when remote-sync-transforms setting is enabled)
@@ -200,8 +200,9 @@
"Re-checks `model-id`'s eligibility against current DB state. The eligibility that got us here was computed
from an event payload, which a concurrent change may have invalidated in the meantime."
[model-spec model-id]
- (boolean (when-let [instance (remote-sync.db/instance (:model-key model-spec) model-id)]
- (spec/check-eligibility model-spec instance))))
+ (boolean (when-let [instance (remote-sync.db/instance (spec/source-model model-spec) model-id)]
+ (and (spec/check-eligibility model-spec instance)
+ (spec/row-exists? model-spec model-id)))))
(defn- create-or-update-sync-object-from-spec!
"Creates or updates a RemoteSyncObject entry using a spec for field hydration.
@@ -268,22 +269,32 @@
(when-let [ak (:archived-key child-spec)]
{ak false})))
+(defn- user-settings-child-ids
+ "The ids of the Fields of the Table with `table-id` that have the `child-spec` user settings."
+ [child-spec table-id]
+ (case (:model-key child-spec)
+ :model/FieldUserSettings (remote-sync.db/field-ids-with-user-settings [table-id])
+ :model/Dimension (remote-sync.db/field-ids-with-dimensions [table-id])))
+
(defn- cascade-to-children!
"When a parent model becomes eligible/ineligible, cascade to its children.
For eligible: query child entities using :parent-fk and derived filter, check eligibility, create RSOs.
For ineligible: query existing RSOs by model_table_id and mark as removed."
[model-spec model-id status eligible?]
- (doseq [child-spec (spec/children-specs (:model-key model-spec))]
- (let [fk (:parent-fk child-spec)
- filter (cascade-filter child-spec)]
- (if eligible?
- ;; Eligible branch: query actual entities and create RSOs for eligible children
- (doseq [child (remote-sync.db/eligible-children (:model-key child-spec) fk model-id filter)]
- (when (spec/check-eligibility child-spec child)
- (create-or-update-sync-object-from-spec! child-spec (:id child) status)))
- ;; Ineligible branch: mark existing child RSOs as removed
- (doseq [child-rso (remote-sync.db/active-child-rsos (:model-type child-spec) model-id)]
- (create-or-update-sync-object-from-spec! child-spec (:model_id child-rso) "removed"))))))
+ (doseq [child-spec (spec/children-specs (:model-key model-spec))
+ child-id (cond
+ (and eligible? (= :user-settings (:identity child-spec)))
+ (user-settings-child-ids child-spec model-id)
+
+ eligible?
+ (for [child (remote-sync.db/eligible-children (:model-key child-spec) (:parent-fk child-spec)
+ model-id (cascade-filter child-spec))
+ :when (spec/check-eligibility child-spec child)]
+ (:id child))
+
+ :else
+ (map :model_id (remote-sync.db/active-child-rsos (:model-type child-spec) model-id)))]
+ (create-or-update-sync-object-from-spec! child-spec child-id (if eligible? status "removed"))))
(defn- handle-model-event-from-spec
"Generic event handler that uses a spec for all configuration.
@@ -291,7 +302,8 @@
[model-spec topic {:keys [object]}]
(let [model-type (:model-type model-spec)
model-id (:id object)
- eligible? (spec/check-eligibility model-spec object)
+ eligible? (and (spec/check-eligibility model-spec object)
+ (spec/row-exists? model-spec model-id))
existing-entry (remote-sync.db/rso model-type model-id)
status (spec/determine-status model-spec topic object)]
(cond
@@ -331,7 +343,7 @@
;;; --------------------------------- Spec-based Event Registration (Non-Collection) -----------------------------------
-(doseq [[_model-key model-spec] (dissoc spec/remote-sync-specs :model/Collection :model/Field :model/Table)]
+(doseq [[_model-key model-spec] (dissoc spec/remote-sync-specs :model/Collection)]
(register-events-for-spec! model-spec))
;;; ----------------------------------------- Collection Event Handler -------------------------------------------------
@@ -421,58 +433,3 @@
(do
(log/infof "Collection %s no longer needs syncing, marking as removed" (:id object))
(create-or-update-remote-sync-object-entry! "Collection" (:id object) "removed" hydrate-collection-details)))))
-
-;;; ------------------------------------------- User Settings Tracking -------------------------------------------------
-
-(def ^:private table-spec (get spec/remote-sync-specs :model/Table))
-(def ^:private field-spec (get spec/remote-sync-specs :model/Field))
-
-(defn- hydrate-table-user-settings-details
- "The RemoteSyncObject details of the Table with `table-id`, as its own `:table_id`/`:table_name`."
- [table-id]
- (let [details (spec/hydrate-model-details table-spec table-id)]
- (assoc details :table_id (:id details) :table_name (:name details))))
-
-(defn- hydrate-field-details
- "The RemoteSyncObject details of the Field with `field-id`."
- [field-id]
- (spec/hydrate-model-details field-spec field-id))
-
-(defn- sync-user-settings!
- "Track `model-type` `id` as updated when `eligible?` and `exists?` holds for it, else mark it removed when tracked."
- [model-type id eligible? exists? hydrate-details]
- (cond
- (and eligible? (exists? id))
- (create-or-update-remote-sync-object-entry! model-type id "update" hydrate-details)
-
- (remote-sync.db/rso-exists? model-type id)
- (create-or-update-remote-sync-object-entry! model-type id "removed" hydrate-details)))
-
-(events/derive! :event/field-update ::field-update-event)
-(events/derive! ::field-update-event :metabase/event)
-
-(methodical/defmethod events/publish-event! ::field-update-event
- [_topic {:keys [object]}]
- (let [eligible? (spec/check-eligibility field-spec object)]
- (sync-user-settings! "FieldUserSettings" (:id object) eligible?
- remote-sync.db/field-user-settings-exist? hydrate-field-details)
- (sync-user-settings! "Dimension" (:id object) eligible?
- remote-sync.db/dimension-exists-for-field? hydrate-field-details)))
-
-(defn- handle-table-event!
- "The generic Table handler plus the Table's TableUserSettings; one handler, since a second primary method on the
- same events would displace the generic one."
- [topic {:keys [object] :as event}]
- (handle-model-event-from-spec table-spec topic event)
- (sync-user-settings! "TableUserSettings" (:id object) (spec/check-eligibility table-spec object)
- remote-sync.db/table-user-settings-exist? hydrate-table-user-settings-details))
-
-(let [event-kws (spec/event-keywords table-spec)
- parent-kw (:parent event-kws)]
- (events/derive! parent-kw :metabase/event)
- (doseq [[_event-type event-kw] (dissoc event-kws :parent)]
- (events/derive! event-kw parent-kw))
- (methodical/add-primary-method!
- #'events/publish-event!
- parent-kw
- (fn [topic event] (handle-table-event! topic event))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 3087fedb117..b44bd721369 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -459,7 +459,7 @@
(serialization/entity-file-path? path))
(defn- pulled-change-count
- "Total number of entities applied by a pull, across entity-id- and path-identified models in `imported-data`."
+ "Total number of entities applied by a pull, across the entity-id models and the user settings in `imported-data`."
[imported-data]
(transduce (map count) + 0 (concat (vals (:by-entity-id imported-data))
(vals (:by-path imported-data)))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 4a931ce058b..2057b44e4a4 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -32,14 +32,17 @@
Each spec contains:
- :model-type - String name for RemoteSyncObject model_type column
- :model-key - Toucan2 model keyword (e.g., :model/Card)
- - :identity - Identity strategy: :entity-id, :path, or :hybrid
- - :path-keys - For :path or :hybrid identity: vector of path components [:database :schema :table :field]
- - :parent-model - For :parent-table eligibility: the parent model key to check eligibility against
- (e.g., :model/Table for Field, Segment, Measure)
- - :parent-fk - For child models: the FK column pointing to the parent (e.g., :table_id)
+ - :identity - Identity strategy: :entity-id, :hybrid, or :user-settings (a Table's or Field's user
+ settings, keyed by the Table or Field id and loaded and tracked by their own import code)
+ - :path-keys - For :hybrid identity: vector of path components [:database :schema :table]
+ - :source-model - Optional model whose instances the events carry and eligibility is checked on, when it is not
+ :model-key (e.g., :model/Field for FieldUserSettings)
+ - :row-key - Optional column of `model-key` holding the tracked id; the model is only tracked for an id
+ with such a row (e.g., :field_id for FieldUserSettings)
+ - :parent-model - The parent model key (e.g., :model/TableUserSettings for FieldUserSettings, Segment, Measure)
+ - :parent-fk - For child models that cascade from their parent: the FK pointing to it (e.g., :table_id)
- :cascade-filter - Optional map of additional filter conditions for cascade queries.
Only needed when the filter differs from {archived-key false}.
- E.g., Field needs {:active true} since it has no :archived-key.
- :delete-after - Optional vector of model keys that must be deleted AFTER this model.
Used to handle FK constraints during import cleanup. For example, if Card
has :delete-after [:model/Collection], Card will be deleted before Collection.
@@ -220,35 +223,56 @@
:export-scope :root-collections ; query for root-level remote-synced + transforms-namespace collections
:enabled? true}
- :model/Table
- {:model-type "Table"
- :model-key :model/Table
- :identity :path
- :path-keys [:database :schema :table]
+ :model/TableUserSettings
+ {:model-type "TableUserSettings"
+ :model-key :model/TableUserSettings
+ :source-model :model/Table
+ :identity :user-settings
+ :row-key :table_id
:events {:prefix :event/table
:types [:create :update :delete :publish :unpublish]}
:eligibility {:type :published-table}
- :archived-key :archived_at ; tables use archived_at, not archived
- :tracking {:select-fields [:name :collection_id]
+ :archived-key :archived_at
+ :tracking {:hydrate-query? true
:field-mappings {:model_name :name
:model_collection_id :collection_id
- :model_table_id :id
- :model_table_name :name}} ; self-reference
+ :model_table_id :table_id
+ :model_table_name :table_name}}
:removal {:statuses #{"removed"}}
:enabled? true}
- :model/Field
- {:model-type "Field"
- :model-key :model/Field
- :identity :path
- :path-keys [:database :schema :table :field]
- :parent-model :model/Table
+ :model/FieldUserSettings
+ {:model-type "FieldUserSettings"
+ :model-key :model/FieldUserSettings
+ :source-model :model/Field
+ :identity :user-settings
+ :row-key :field_id
+ :parent-model :model/TableUserSettings
:parent-fk :table_id
- :cascade-filter {:active true}
:events {:prefix :event/field
:types [:create :update :delete]}
:eligibility {:type :parent-table}
- :archived-key nil ; fields don't have archived
+ :archived-key nil
+ :tracking {:hydrate-query? true
+ :field-mappings {:model_name :name
+ :model_collection_id :collection_id
+ :model_table_id :table_id
+ :model_table_name :table_name}}
+ :removal {:statuses #{"removed"}}
+ :enabled? true}
+
+ :model/Dimension
+ {:model-type "Dimension"
+ :model-key :model/Dimension
+ :source-model :model/Field
+ :identity :user-settings
+ :row-key :field_id
+ :parent-model :model/TableUserSettings
+ :parent-fk :table_id
+ :events {:prefix :event/field
+ :types [:create :update :delete]}
+ :eligibility {:type :parent-table}
+ :archived-key nil
:tracking {:hydrate-query? true
:field-mappings {:model_name :name
:model_collection_id :collection_id
@@ -262,7 +286,7 @@
:model-key :model/Segment
:identity :hybrid ; entity_id but needs table context for path
:path-keys [:database :schema :table]
- :parent-model :model/Table
+ :parent-model :model/TableUserSettings
:events {:prefix :event/segment
:types [:create :update :delete]}
:eligibility {:type :parent-table}
@@ -280,7 +304,7 @@
:model-key :model/Measure
:identity :hybrid ; entity_id but needs table context for path
:path-keys [:database :schema :table]
- :parent-model :model/Table
+ :parent-model :model/TableUserSettings
:events {:prefix :event/measure
:types [:create :update :delete]}
:eligibility {:type :parent-table}
@@ -889,16 +913,31 @@
false))))
-(defmethod check-eligibility-by-type :published-table
- [_ {:keys [is_published collection_id]}]
+(defn- published-in-remote-synced-collection?
+ "Whether `table` is published in a remote-synced Collection."
+ [{:keys [is_published collection_id]}]
(and is_published
(collections/remote-synced-collection? collection_id)))
+(defmethod check-eligibility-by-type :published-table
+ [_ table]
+ (published-in-remote-synced-collection? table))
+
(defmethod check-eligibility-by-type :parent-table
- [{:keys [parent-model]} {:keys [table_id]}]
+ [_ {:keys [table_id]}]
(when table_id
- (when-let [table (remote-sync.db/instance parent-model table_id)]
- (check-eligibility (spec-for-model-key parent-model) table))))
+ (some-> (remote-sync.db/instance :model/Table table_id) published-in-remote-synced-collection?)))
+
+(defn source-model
+ "The model whose instances the spec's events carry and its eligibility is checked on."
+ [spec]
+ (or (:source-model spec) (:model-key spec)))
+
+(defn row-exists?
+ "Whether the spec's model has a row for `id`, which always holds for a spec without a `:row-key`."
+ [{:keys [model-key row-key]} id]
+ (or (nil? row-key)
+ (remote-sync.db/row-exists? model-key row-key id)))
(defmethod check-eligibility-by-type :setting
[{:keys [eligibility]} _object]
@@ -1060,14 +1099,9 @@
id)
id)))
-(defmethod extract-identity-from-serdes-path :path
+(defmethod extract-identity-from-serdes-path :user-settings
[_ serdes-path _ingest-one]
- (let [path-map (into {} (map (fn [elem] [(keyword (u/lower-case-en (:model elem))) (:id elem)]) serdes-path))]
- (cond-> {}
- (contains? path-map :database) (assoc :db_name (:database path-map))
- (contains? path-map :schema) (assoc :schema (:schema path-map))
- (contains? path-map :table) (assoc :table_name (:table path-map))
- (contains? path-map :field) (assoc :field_name (:field path-map)))))
+ serdes-path)
(defmethod extract-identity-from-serdes-path :default
[_ _ _]
@@ -1076,7 +1110,7 @@
(defn extract-imported-entities
"Processes serdes paths from an import and extracts entity identities grouped by how they should be looked up.
Returns a map with :by-entity-id containing entity_ids grouped by model type, and :by-path containing
- path lookup maps for models like Table and Field that use path-based identity.
+ the serdes paths of the user settings models, which are only counted.
`ingest-one` (path -> ingested entity) is for the pre-load conflict check: a `:natural-key-paths?` path whose
id matches no local row yet resolves to the row the loader will match on the entity's natural key."
@@ -1094,7 +1128,7 @@
(:entity-id :hybrid)
(update-in acc [:by-entity-id model-type] (fnil conj #{}) identity-data)
- :path
+ :user-settings
(update-in acc [:by-path (:model-key spec)] (fnil conj []) identity-data)
acc)
@@ -1121,14 +1155,13 @@
(defn event-keywords
"Returns a map of event keywords for a spec: {:parent :create :update :delete}."
- [{:keys [events]}]
+ [{:keys [model-type events]}]
(let [{:keys [prefix types]} events
ns-str (namespace prefix)
base (name prefix)]
(merge
- ;; Parent keyword uses the event prefix name (e.g., :card -> card-change-event)
{:parent (keyword "metabase-enterprise.remote-sync.events"
- (str base "-change-event"))}
+ (str (u/->kebab-case-en model-type) "-change-event"))}
(into {}
(for [event-type types]
[event-type (keyword ns-str (str base "-" (name event-type)))])))))
@@ -1153,7 +1186,6 @@
Parameters:
- spec: The model spec
- data: For :entity-id/:hybrid - a collection of entity_ids
- For :path - a collection of path maps {:db_name :schema :table_name [:field_name]}
- timestamp: The sync timestamp"
{:arglists '([spec data timestamp])}
(fn [spec _data _timestamp] (:identity spec)))
@@ -1201,39 +1233,6 @@
:status_changed_at timestamp}
mapped-fields))))))))
-(defmethod query-entities-for-sync :path
- [{:keys [model-type model-key]} paths timestamp]
- (when (seq paths)
- (case model-key
- :model/Table
- (->> (remote-sync.db/tables-at-paths paths)
- (map (fn [{:keys [id name collection_id]}]
- {:model_type model-type
- :model_id id
- :model_name name
- :model_collection_id collection_id
- :model_display nil
- :model_table_id id
- :model_table_name name
- :status "synced"
- :status_changed_at timestamp})))
-
- :model/Field
- (->> (remote-sync.db/fields-at-paths paths)
- (map (fn [{:keys [id name table_id table_name collection_id]}]
- {:model_type model-type
- :model_id id
- :model_name name
- :model_collection_id collection_id
- :model_display nil
- :model_table_id table_id
- :model_table_name table_name
- :status "synced"
- :status_changed_at timestamp})))
-
- ;; Default for unknown path models
- nil)))
-
(defmethod query-entities-for-sync :default
[_ _ _]
nil)
@@ -1242,15 +1241,10 @@
"Builds RemoteSyncObject entries for all imported entities based on their specs. Iterates over enabled
specs and queries the database to hydrate the fields needed for each sync object. Returns a sequence
of maps ready for insertion into the RemoteSyncObject table."
- [timestamp {:keys [by-entity-id by-path]}]
+ [timestamp {:keys [by-entity-id]}]
(into []
- (for [[model-key spec] (enabled-specs)
- :let [identity-type (:identity spec)
- model-type (:model-type spec)
- data (case identity-type
- :entity-id (get by-entity-id model-type)
- :path (get by-path model-key)
- :hybrid (get by-entity-id model-type))]
+ (for [[_model-key spec] (enabled-specs)
+ :let [data (get by-entity-id (:model-type spec))]
:when (seq data)
entity (query-entities-for-sync spec data timestamp)]
entity)))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
index 138c8f29d0f..907c667f029 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
@@ -896,10 +896,10 @@
(deftest ^:parallel snippet-event-derivation-test
(testing "snippet events properly derive from :metabase/event"
- (is (events/isa? ::remote-sync.events/snippet-change-event :metabase/event))
- (is (events/isa? :event/snippet-create ::remote-sync.events/snippet-change-event))
- (is (events/isa? :event/snippet-update ::remote-sync.events/snippet-change-event))
- (is (events/isa? :event/snippet-delete ::remote-sync.events/snippet-change-event))))
+ (is (events/isa? ::remote-sync.events/native-query-snippet-change-event :metabase/event))
+ (is (events/isa? :event/snippet-create ::remote-sync.events/native-query-snippet-change-event))
+ (is (events/isa? :event/snippet-update ::remote-sync.events/native-query-snippet-change-event))
+ (is (events/isa? :event/snippet-delete ::remote-sync.events/native-query-snippet-change-event))))
(deftest ^:parallel collection-event-derivation-test
(testing "collection events properly derive from :metabase/event"
@@ -1075,18 +1075,25 @@
;;; Table Event Tests
(deftest table-update-event-creates-entry-for-published-table-test
- (testing "table-update event creates remote sync object entry for published table in remote-synced collection"
+ (testing "table-update event creates remote sync object entry for published table in remote-synced collection,
+ only once the table has a TableUserSettings row"
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
:model/Table table {:name "Test Table"
:is_published true
:collection_id (:id remote-sync-collection)}]
(t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/table-update
+ {:object table :user-id (mt/user->id :rasta)})
+ (is (not (t2/exists? :model/RemoteSyncObject)))
+ (t2/insert! :model/TableUserSettings {:table_id (:id table)
+ :is_published true
+ :collection_id (:id remote-sync-collection)})
(events/publish-event! :event/table-update
{:object table :user-id (mt/user->id :rasta)})
(let [entries (t2/select :model/RemoteSyncObject)]
(is (= 1 (count entries)))
- (is (=? {:model_type "Table"
+ (is (=? {:model_type "TableUserSettings"
:model_id (:id table)
:status "update"}
(first entries))))))))
@@ -1122,13 +1129,16 @@
:model/Table table {:name "Test Table"
:is_published true
:collection_id (:id remote-sync-collection)}]
- (track-as-synced! "Table" table)
+ (t2/insert! :model/TableUserSettings {:table_id (:id table)
+ :is_published true
+ :collection_id (:id remote-sync-collection)})
+ (track-as-synced! "TableUserSettings" table)
(events/publish-event! :event/table-update
{:object (assoc table :archived_at (t/offset-date-time))
:user-id (mt/user->id :rasta)})
(let [entries (t2/select :model/RemoteSyncObject)]
(is (= 1 (count entries)))
- (is (=? {:model_type "Table"
+ (is (=? {:model_type "TableUserSettings"
:model_id (:id table)
:status "delete"}
(first entries))))))))
@@ -1141,20 +1151,20 @@
:collection_id (:id remote-sync-collection)}]
(t2/delete! :model/RemoteSyncObject)
;; First, create an entry with status "synced" directly
- (t2/insert! :model/RemoteSyncObject {:model_type "Table"
+ (t2/insert! :model/RemoteSyncObject {:model_type "TableUserSettings"
:model_id (:id table)
:model_name "Test Table"
:model_table_id (:id table)
:model_table_name "Test Table"
:status "synced"
:status_changed_at (t/offset-date-time)})
- (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))]
(is (= "synced" (:status initial-entry)))
;; Now unpublish the table
(events/publish-event! :event/table-update
{:object (assoc table :is_published false)
:user-id (mt/user->id :rasta)})
- (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))]
(is (= "removed" (:status update-entry))))))))
(deftest table-moved-to-normal-collection-marks-as-removed-test
@@ -1166,28 +1176,58 @@
:collection_id (:id remote-sync-collection)}]
(t2/delete! :model/RemoteSyncObject)
;; First, create an entry with status "synced" directly
- (t2/insert! :model/RemoteSyncObject {:model_type "Table"
+ (t2/insert! :model/RemoteSyncObject {:model_type "TableUserSettings"
:model_id (:id table)
:model_name "Test Table"
:model_table_id (:id table)
:model_table_name "Test Table"
:status "synced"
:status_changed_at (t/offset-date-time)})
- (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))]
(is (= "synced" (:status initial-entry)))
;; Now move to normal collection
(events/publish-event! :event/table-update
{:object (assoc table :collection_id (:id normal-collection))
:user-id (mt/user->id :rasta)})
- (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table))]
(is (= "removed" (:status update-entry))))))))
+(deftest table-publish-tracks-only-user-settings-test
+ (testing "publishing a table tracks its TableUserSettings and the FieldUserSettings and Dimensions of its fields,
+ never the Table or its Fields, and unpublishing marks them removed"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "Remote-Sync" :type "library-data"}
+ :model/Table table {:name "T"}
+ :model/Field curated {:name "curated" :table_id (:id table) :base_type :type/Text}
+ :model/Field remapped {:name "remapped" :table_id (:id table) :base_type :type/Integer}
+ :model/Field _plain {:name "plain" :table_id (:id table) :base_type :type/Text}
+ :model/Dimension _ {:field_id (:id remapped) :name "Remapped" :type :internal}]
+ (t2/insert! :model/FieldUserSettings {:field_id (:id curated) :description "curated"})
+ (t2/insert! :model/TableUserSettings {:table_id (:id table) :is_published true :collection_id (:id coll)})
+ (t2/delete! :model/RemoteSyncObject)
+ (let [tracked (fn [] (into #{} (map (juxt :model_type :model_id :status)) (t2/select :model/RemoteSyncObject)))]
+ (events/publish-event! :event/table-publish
+ {:object (assoc table :is_published true :collection_id (:id coll))
+ :user-id (mt/user->id :rasta)})
+ (is (= #{["TableUserSettings" (:id table) "update"]
+ ["FieldUserSettings" (:id curated) "update"]
+ ["Dimension" (:id remapped) "update"]}
+ (tracked)))
+ (t2/update! :model/TableUserSettings (:id table) {:is_published false :collection_id nil})
+ (events/publish-event! :event/table-unpublish
+ {:object (assoc table :is_published false :collection_id nil)
+ :user-id (mt/user->id :rasta)})
+ (is (= #{["TableUserSettings" (:id table) "removed"]
+ ["FieldUserSettings" (:id curated) "removed"]
+ ["Dimension" (:id remapped) "removed"]}
+ (tracked))))))))
+
(deftest ^:parallel table-event-derivation-test
(testing "table events properly derive from :metabase/event"
- (is (events/isa? ::remote-sync.events/table-change-event :metabase/event))
- (is (events/isa? :event/table-create ::remote-sync.events/table-change-event))
- (is (events/isa? :event/table-update ::remote-sync.events/table-change-event))
- (is (events/isa? :event/table-delete ::remote-sync.events/table-change-event))))
+ (is (events/isa? ::remote-sync.events/table-user-settings-change-event :metabase/event))
+ (is (events/isa? :event/table-create ::remote-sync.events/table-user-settings-change-event))
+ (is (events/isa? :event/table-update ::remote-sync.events/table-user-settings-change-event))
+ (is (events/isa? :event/table-delete ::remote-sync.events/table-user-settings-change-event))))
;;; Segment Event Tests
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index bac1af0a951..089c96b9abb 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -812,7 +812,7 @@
:location "/"}]
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
- {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
+ {:model_type "TableUserSettings" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
(let [initial-files {"main" {"databases/test_db/tables/test_table/test_table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
@@ -854,7 +854,7 @@
"Removed segment files should be deleted after export")))))))))
(deftest export!-updates-removed-table-entries-to-synced-test
- (testing "export! updates RemoteSyncObject entries for 'removed' Tables/Fields/Segments to 'synced' status after successful export"
+ (testing "export! updates 'removed' Segment entries to 'synced' and drops 'removed' TableUserSettings/FieldUserSettings entries after successful export"
(mt/with-model-cleanup [:model/RemoteSyncObject :model/Segment]
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
@@ -874,23 +874,23 @@
:entity_id "test-segment-xxxxxxxx"}]
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
- {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
- {:model_type "Field" :model_id field-id :model_name "test-field" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
+ {:model_type "TableUserSettings" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
+ {:model_type "FieldUserSettings" :model_id field-id :model_name "test-field" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
{:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
(is (= 4 (t2/count :model/RemoteSyncObject)))
(let [mock-source (test-helpers/create-mock-source)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
- (is (= 4 (t2/count :model/RemoteSyncObject))
- "All RemoteSyncObject entries should remain after export")
- (let [table-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id table-id)
- field-entry (t2/select-one :model/RemoteSyncObject :model_type "Field" :model_id field-id)
+ (is (= 2 (t2/count :model/RemoteSyncObject))
+ "Only the Collection and Segment entries should remain after export")
+ (let [table-entry (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id table-id)
+ field-entry (t2/select-one :model/RemoteSyncObject :model_type "FieldUserSettings" :model_id field-id)
segment-entry (t2/select-one :model/RemoteSyncObject :model_type "Segment" :model_id segment-id)
coll-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)]
- (is (= "synced" (:status table-entry))
- "Table entry should be updated to synced after export")
- (is (= "synced" (:status field-entry))
- "Field entry should be updated to synced after export")
+ (is (nil? table-entry)
+ "TableUserSettings entry should be dropped after export")
+ (is (nil? field-entry)
+ "FieldUserSettings entry should be dropped after export")
(is (= "synced" (:status segment-entry))
"Segment entry should be updated to synced after export")
(is (= "synced" (:status coll-entry))
@@ -910,7 +910,7 @@
:location "/"}]
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
- {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
+ {:model_type "TableUserSettings" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
(let [initial-files {"main" {"databases/test_db/schemas/public/tables/test_table/test_table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
@@ -997,58 +997,28 @@
;; Import Table/Field/Segment tracking tests
-(deftest import!-tracks-tables-in-remote-sync-object-test
- (testing "import! creates RemoteSyncObject entries for imported Tables"
+(deftest import!-does-not-track-tables-or-fields-test
+ (testing "import! of Table and Field files creates no RemoteSyncObject entries for them, which are not synced"
(mt/with-model-cleanup [:model/RemoteSyncObject]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
- :model/Table {table-id :id} {:name "test-table" :db_id db-id}
- :model/Collection {coll-id :id}
- {:name "Test Collection"
- :is_remote_synced true
- :entity_id "test-collection-1xxxx"
- :location "/"}]
- (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
- (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test_db/tables/test_table/test_table.yaml"
- (test-helpers/generate-table-yaml "test-table" "test-db")}}
- mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result)))
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id coll-id
- :status "synced")
- "Collection should be tracked in RemoteSyncObject")
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Table"
- :model_id table-id
- :status "synced")
- "Table should be tracked in RemoteSyncObject")))))))
-
-(deftest import!-tracks-fields-in-remote-sync-object-test
- (testing "import! creates RemoteSyncObject entries for imported Fields"
- (mt/with-model-cleanup [:model/RemoteSyncObject]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
- (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
- :model/Table {table-id :id} {:name "test-table" :db_id db-id}
- :model/Field {field-id :id} {:name "test-field" :table_id table-id :base_type :type/Text}
- :model/Collection _ {:name "Test Collection"
- :is_remote_synced true
- :entity_id "test-collection-1xxxx"
- :location "/"}]
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id :schema "PUBLIC"}
+ :model/Field _ {:name "test-field" :table_id table-id :base_type :type/Text}
+ :model/Collection {coll-id :id} {:name "Test Collection"
+ :is_remote_synced true
+ :entity_id "test-collection-1xxxx"
+ :location "/"}]
(let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test_db/tables/test_table/fields/test_field.yaml"
- (test-helpers/generate-field-yaml "test-field" "test-table" "test-db")}}
+ "databases/test_db/schemas/PUBLIC/tables/test_table/test_table.yaml"
+ (test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")
+ "databases/test_db/schemas/PUBLIC/tables/test_table/fields/test_field.yaml"
+ (test-helpers/generate-field-yaml "test-field" "test-table" "test-db" :schema "PUBLIC")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Field"
- :model_id field-id
- :status "synced")
- "Field should be tracked in RemoteSyncObject")))))))
+ (is (= [["Collection" coll-id "synced"]]
+ (t2/select-fn-vec (juxt :model_type :model_id :status) :model/RemoteSyncObject)))))))))
(deftest import!-tracks-segments-in-remote-sync-object-test
(testing "import! creates RemoteSyncObject entries for imported Segments"
@@ -1086,29 +1056,6 @@
:status "synced")
"Segment should be tracked in RemoteSyncObject")))))))
-(deftest import!-tracks-tables-with-schema-in-remote-sync-object-test
- (testing "import! creates RemoteSyncObject entries for imported Tables with schema"
- (mt/with-model-cleanup [:model/RemoteSyncObject]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
- (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
- :model/Table {table-id :id} {:name "test-table" :db_id db-id :schema "PUBLIC"}
- :model/Collection _ {:name "Test Collection"
- :is_remote_synced true
- :entity_id "test-collection-1xxxx"
- :location "/"}]
- (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
- (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test_db/schemas/PUBLIC/tables/test_table/test_table.yaml"
- (test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}
- mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result)))
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Table"
- :model_id table-id
- :status "synced")
- "Table with schema should be tracked in RemoteSyncObject")))))))
-
(deftest import!-includes-actions-attached-to-models-test
(testing "import! successfully imports Actions attached to Models in synced collections"
(mt/with-model-cleanup [:model/RemoteSyncObject :model/Action]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index abd8603642d..6c0f548d04b 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -27,16 +27,16 @@
(deftest all-specs-have-valid-identity-test
(testing "Every spec has a valid identity type"
- (let [valid-identity-types #{:entity-id :path :hybrid}]
+ (let [valid-identity-types #{:entity-id :hybrid :user-settings}]
(doseq [[model-key spec] spec/remote-sync-specs]
(testing (str "Spec for " model-key)
(is (contains? valid-identity-types (:identity spec))
(str "Invalid identity type: " (:identity spec))))))))
(deftest path-based-specs-have-path-keys-test
- (testing "Specs with :path or :hybrid identity have :path-keys"
+ (testing "Specs with :hybrid identity have :path-keys"
(doseq [[model-key spec] spec/remote-sync-specs
- :when (#{:path :hybrid} (:identity spec))]
+ :when (= :hybrid (:identity spec))]
(testing (str "Spec for " model-key)
(is (vector? (:path-keys spec))
"path-keys should be a vector")
@@ -96,11 +96,9 @@
(when-let [cf (:cascade-filter spec)]
(is (map? cf)
":cascade-filter should be a map when present")))))
- (testing "children-specs derives the correct children for Table"
- (let [children (spec/children-specs :model/Table)]
- (is (= 1 (count children)))
- (is (= #{:model/Field}
- (into #{} (map :model-key) children)))))
+ (testing "children-specs derives the cascading children of TableUserSettings"
+ (is (= #{:model/FieldUserSettings :model/Dimension}
+ (into #{} (map :model-key) (spec/children-specs :model/TableUserSettings)))))
(testing "children-specs returns empty for models with no children"
(is (empty? (spec/children-specs :model/Card)))))
@@ -110,7 +108,9 @@
(testing "spec-for-model-type returns correct spec"
(is (= :model/Card (:model-key (spec/spec-for-model-type "Card"))))
(is (= :model/Dashboard (:model-key (spec/spec-for-model-type "Dashboard"))))
- (is (= :model/Table (:model-key (spec/spec-for-model-type "Table"))))
+ (is (= :model/TableUserSettings (:model-key (spec/spec-for-model-type "TableUserSettings"))))
+ (is (nil? (spec/spec-for-model-type "Table")))
+ (is (nil? (spec/spec-for-model-type "Field")))
(is (= :model/Measure (:model-key (spec/spec-for-model-type "Measure"))))
(is (nil? (spec/spec-for-model-type "NonExistent")))))
@@ -127,8 +127,9 @@
(is (set? types))
(is (contains? types "Card"))
(is (contains? types "Dashboard"))
- (is (contains? types "Table"))
- (is (contains? types "Field"))
+ (is (contains? types "TableUserSettings"))
+ (is (contains? types "FieldUserSettings"))
+ (is (contains? types "Dimension"))
(is (contains? types "Segment"))
(is (contains? types "Measure"))
(is (contains? types "Transform"))
@@ -137,24 +138,22 @@
(is (contains? types "Glossary"))
(is (contains? types "Action"))
(is (contains? types "DataApp"))
- (is (= 17 (count types))))))
+ (is (= 18 (count types))))))
(deftest specs-by-identity-type-test
(testing "specs-by-identity-type filters correctly"
(mt/with-temporary-setting-values [remote-sync-transforms false]
(let [entity-id-specs (spec/specs-by-identity-type :entity-id)
- path-specs (spec/specs-by-identity-type :path)
+ user-settings-specs (spec/specs-by-identity-type :user-settings)
hybrid-specs (spec/specs-by-identity-type :hybrid)]
(testing "entity-id specs"
(is (contains? entity-id-specs :model/Card))
(is (contains? entity-id-specs :model/Dashboard))
(is (contains? entity-id-specs :model/Collection))
- (is (not (contains? entity-id-specs :model/Table)))
- (is (not (contains? entity-id-specs :model/Field))))
- (testing "path specs"
- (is (contains? path-specs :model/Table))
- (is (contains? path-specs :model/Field))
- (is (not (contains? path-specs :model/Card))))
+ (is (not (contains? entity-id-specs :model/TableUserSettings))))
+ (testing "user settings specs"
+ (is (= #{:model/TableUserSettings :model/FieldUserSettings :model/Dimension}
+ (set (keys user-settings-specs)))))
(testing "hybrid specs"
(is (contains? hybrid-specs :model/Segment))
(is (contains? hybrid-specs :model/Measure))
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
index 6883a1f179b..e92a82d34b8 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
@@ -348,7 +348,7 @@ describe("AllChangesView", () => {
const tableEntity = createMockRemoteSyncEntity({
id: 100,
name: "Orders Table",
- model: "table",
+ model: "tableusersettings",
collection_id: 1,
sync_status: "update",
});
@@ -415,7 +415,7 @@ describe("AllChangesView", () => {
const fieldEntity = createMockRemoteSyncEntity({
id: 400,
name: "email",
- model: "field",
+ model: "fieldusersettings",
collection_id: 1,
sync_status: "update",
table_id: 100,
@@ -442,6 +442,54 @@ describe("AllChangesView", () => {
});
});
+ describe("table user settings", () => {
+ it("should link a table's user settings and its fields' user settings and dimensions to Data Studio", async () => {
+ const tableEntity = createMockRemoteSyncEntity({
+ id: 100,
+ name: "Orders Table",
+ model: "tableusersettings",
+ collection_id: 1,
+ sync_status: "update",
+ table_id: 100,
+ table_name: "Orders Table",
+ });
+ const fieldEntity = createMockRemoteSyncEntity({
+ id: 400,
+ name: "email",
+ model: "fieldusersettings",
+ collection_id: 1,
+ sync_status: "update",
+ table_id: 100,
+ table_name: "Orders Table",
+ });
+ const dimensionEntity = createMockRemoteSyncEntity({
+ id: 401,
+ name: "user_id",
+ model: "dimension",
+ collection_id: 1,
+ sync_status: "update",
+ table_id: 100,
+ table_name: "Orders Table",
+ });
+
+ setup({
+ entities: [tableEntity, fieldEntity, dimensionEntity],
+ });
+
+ expect(
+ await screen.findByRole("link", { name: /Orders Table/ }),
+ ).toHaveAttribute("href", "/data-studio/library/tables/100");
+ expect(screen.getByRole("link", { name: /email/ })).toHaveAttribute(
+ "href",
+ "/data-studio/library/tables/100/fields/400",
+ );
+ expect(screen.getByRole("link", { name: /user_id/ })).toHaveAttribute(
+ "href",
+ "/data-studio/library/tables/100/fields/401",
+ );
+ });
+ });
+
describe("snippet collections", () => {
it("should display snippets in snippet collections with hierarchy", async () => {
const snippetCollection = createMockCollection({
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx
index 30dca428dcb..74ccae6f1d1 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx
@@ -19,6 +19,14 @@ function getEntityUrl(entity: RemoteSyncEntity): string | null {
if (entity.model === "transformtest") {
return null;
}
+ if (entity.model === "tableusersettings") {
+ return Urls.dataStudioTable(entity.id);
+ }
+ if (entity.model === "fieldusersettings" || entity.model === "dimension") {
+ return entity.table_id != null
+ ? Urls.dataStudioTableFields(entity.table_id, entity.id)
+ : null;
+ }
if (entity.model === "action") {
if (entity.card_id === undefined) {
return null;
@@ -33,7 +41,10 @@ function getEntityUrl(entity: RemoteSyncEntity): string | null {
export const EntityLink = ({ entity }: EntityLinkProps) => {
const getIcon = useGetIcon();
const entityIcon = useMemo((): IconData => {
- if (entity.model === "field") {
+ if (entity.model === "tableusersettings") {
+ return { name: "table" };
+ }
+ if (entity.model === "fieldusersettings" || entity.model === "dimension") {
return { name: "field" };
}
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts
index d0930da4eaf..9d93302aeb1 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts
@@ -116,7 +116,13 @@ const displayGroupSpecs: DisplayGroupSpec[] = [
},
{
id: "tables",
- models: new Set(["table", "field", "segment", "measure"]),
+ models: new Set([
+ "tableusersettings",
+ "fieldusersettings",
+ "dimension",
+ "segment",
+ "measure",
+ ]),
icon: "synced_collection",
priority: 50,
},
@@ -341,7 +347,8 @@ const getSpecById = (id: string): DisplayGroupSpec | undefined => {
* Models that are children of tables (get their collection from a parent table)
*/
const TABLE_CHILD_MODELS: Set<RemoteSyncEntityModel> = new Set([
- "field",
+ "fieldusersettings",
+ "dimension",
"segment",
"measure",
]);
@@ -403,7 +410,7 @@ const sortEntitiesByStatus = (
* dirty tables and orphan children (children whose parent table is not dirty).
*/
const buildTableGroups = (items: RemoteSyncEntity[]): TableGroup[] => {
- const tables = items.filter((item) => item.model === "table");
+ const tables = items.filter((item) => item.model === "tableusersettings");
const tableChildren = items.filter((item) => isTableChildModel(item.model));
const childrenByTableId = _.groupBy(tableChildren, (e) => e.table_id || 0);
const tableIds = new Set(tables.map((tbl) => tbl.id));
@@ -446,7 +453,7 @@ const filterOtherItems = (
return false;
}
return (
- item.model !== "table" &&
+ item.model !== "tableusersettings" &&
!isTableChildModel(item.model) &&
!(item.model === "collection" && item.id === Number(collectionId))
);
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts
index a65bedbd9c4..0f4ae3758a6 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts
@@ -225,8 +225,12 @@ describe("remote_sync utils", () => {
});
describe("isTableChildModel", () => {
- it("should return true for field model", () => {
- expect(isTableChildModel("field")).toBe(true);
+ it("should return true for field user settings model", () => {
+ expect(isTableChildModel("fieldusersettings")).toBe(true);
+ });
+
+ it("should return true for dimension model", () => {
+ expect(isTableChildModel("dimension")).toBe(true);
});
it("should return true for segment model", () => {
@@ -241,8 +245,8 @@ describe("remote_sync utils", () => {
expect(isTableChildModel("card")).toBe(false);
});
- it("should return false for table model", () => {
- expect(isTableChildModel("table")).toBe(false);
+ it("should return false for table user settings model", () => {
+ expect(isTableChildModel("tableusersettings")).toBe(false);
});
it("should return false for dashboard model", () => {
diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.ts
index 5bb1e1438d6..fe418e2096c 100644
--- a/frontend/src/metabase-types/api/remote-sync.ts
+++ b/frontend/src/metabase-types/api/remote-sync.ts
@@ -12,8 +12,9 @@ export type RemoteSyncEntityModel =
| "collection"
| "document"
| "nativequerysnippet"
- | "table"
- | "field"
+ | "tableusersettings"
+ | "fieldusersettings"
+ | "dimension"
| "segment"
| "measure"
| "transform"
diff --git a/resources/migrations/065/20261016_remote_sync_object_user_settings.yaml b/resources/migrations/065/20261016_remote_sync_object_user_settings.yaml
new file mode 100644
index 00000000000..0284b97a93a
--- /dev/null
+++ b/resources/migrations/065/20261016_remote_sync_object_user_settings.yaml
@@ -0,0 +1,42 @@
+## Remote sync now tracks a published table through its TableUserSettings, FieldUserSettings, and Dimensions
+## rather than its Table and Fields. A Table or Field row whose table or field has a user settings row, and no row of
+## that user settings yet, becomes that user settings row, keeping its pending status so unpushed changes are still
+## pushed; the rest are deleted after.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.2026-10-16T00:00:00
+ author: ranquild
+ comment: Turn the RemoteSyncObject rows of Tables with user settings into TableUserSettings rows
+ changes:
+ - sql:
+ sql: >-
+ UPDATE remote_sync_object SET model_type = 'TableUserSettings'
+ WHERE model_type = 'Table'
+ AND EXISTS (SELECT 1 FROM metabase_table_user_settings tus WHERE tus.table_id = remote_sync_object.model_id)
+ AND NOT EXISTS (SELECT 1 FROM (SELECT model_id FROM remote_sync_object WHERE model_type = 'TableUserSettings') rso
+ WHERE rso.model_id = remote_sync_object.model_id)
+ rollback: []
+
+ - changeSet:
+ id: v65.2026-10-16T00:00:01
+ author: ranquild
+ comment: Turn the RemoteSyncObject rows of Fields with user settings into FieldUserSettings rows
+ changes:
+ - sql:
+ sql: >-
+ UPDATE remote_sync_object SET model_type = 'FieldUserSettings'
+ WHERE model_type = 'Field'
+ AND EXISTS (SELECT 1 FROM metabase_field_user_settings fus WHERE fus.field_id = remote_sync_object.model_id)
+ AND NOT EXISTS (SELECT 1 FROM (SELECT model_id FROM remote_sync_object WHERE model_type = 'FieldUserSettings') rso
+ WHERE rso.model_id = remote_sync_object.model_id)
+ rollback: []
+
+ - changeSet:
+ id: v65.2026-10-16T00:00:02
+ author: ranquild
+ comment: Delete the remaining RemoteSyncObject rows of Tables and Fields
+ changes:
+ - sql:
+ sql: DELETE FROM remote_sync_object WHERE model_type IN ('Table', 'Field')
+ rollback: []
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index 56685ac5187..51b5f59a8c9 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -3954,3 +3954,64 @@
(is (t2/exists? :data_app :id app-id))
(is (t2/exists? :permissions_group :id ordinary-group))
(is (empty? (t2/select :data_app_group_assignment)))))))
+
+(deftest remote-sync-object-user-settings-test
+ (testing "v65.2026-10-16T00:00:00-02: the RemoteSyncObject rows of Tables and Fields with user settings become their
+ user settings' rows, keeping their status, unless those are already tracked, and the rest are deleted"
+ (impl/test-migrations ["v65.2026-10-16T00:00:00" "v65.2026-10-16T00:00:02"] [migrate!]
+ (let [db-id (t2/insert-returning-pk! :metabase_database {:name "Remote Sync User Settings Test DB"
+ :engine "h2"
+ :created_at :%now
+ :updated_at :%now
+ :details "{}"})
+ table! (fn [table-name]
+ (t2/insert-returning-pk! :metabase_table {:active true
+ :db_id db-id
+ :name table-name
+ :created_at :%now
+ :updated_at :%now}))
+ field! (fn [table-id field-name]
+ (t2/insert-returning-pk! :metabase_field {:table_id table-id
+ :name field-name
+ :display_name field-name
+ :base_type "type/Text"
+ :database_type "TEXT"
+ :position 0
+ :database_position 0
+ :active true
+ :preview_display true
+ :created_at :%now
+ :updated_at :%now}))
+ rso! (fn [model-type model-id status]
+ (t2/insert-returning-pk! :remote_sync_object {:model_type model-type
+ :model_id model-id
+ :model_name model-type
+ :status status
+ :status_changed_at :%now
+ :file_path (str model-type "/" model-id ".yaml")}))
+ published (table! "published")
+ unpublished (table! "unpublished")
+ tracked (table! "tracked")
+ edited (field! published "edited")
+ unedited (field! published "unedited")
+ tracked-fld (field! tracked "tracked")]
+ (t2/insert! :metabase_table_user_settings [{:table_id published :created_at :%now :updated_at :%now}
+ {:table_id tracked :created_at :%now :updated_at :%now}])
+ (t2/insert! :metabase_field_user_settings [{:field_id edited :created_at :%now :updated_at :%now}
+ {:field_id tracked-fld :created_at :%now :updated_at :%now}])
+ (let [table-rso (rso! "Table" published "create")
+ field-rso (rso! "Field" edited "update")
+ dropped [(rso! "Table" unpublished "update") (rso! "Field" unedited "update")
+ (rso! "Table" tracked "update") (rso! "Field" tracked-fld "update")]
+ kept [(rso! "TableUserSettings" tracked "update") (rso! "FieldUserSettings" tracked-fld "update")]
+ card-rso (rso! "Card" 1 "update")]
+ (migrate!)
+ (is (=? {:model_type "TableUserSettings" :model_id published :status "create"}
+ (t2/select-one :remote_sync_object :id table-rso)))
+ (is (=? {:model_type "FieldUserSettings" :model_id edited :status "update"}
+ (t2/select-one :remote_sync_object :id field-rso)))
+ (is (not (t2/exists? :remote_sync_object :id [:in dropped])))
+ (is (= (set kept)
+ (set (concat (t2/select-pks-vec :remote_sync_object :model_type "TableUserSettings" :model_id tracked)
+ (t2/select-pks-vec :remote_sync_object :model_type "FieldUserSettings" :model_id tracked-fld)))))
+ (is (t2/exists? :remote_sync_object :id card-rso)))))))