Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

feat: add table/field/segment support to remote-sync (#67327)

Edward Paget30cd48cc7c3c
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 48676a2c7db..e58e8b61ff7 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1283,6 +1283,7 @@
lib-be
models
permissions
+ remote-sync
search
util
xrays}}
@@ -1635,6 +1636,7 @@
permissions
premium-features
query-processor
+ remote-sync
search
util
warehouses}}
@@ -1645,6 +1647,7 @@
:uses #{analytics
api
app-db
+ config
database-routing
driver
events
@@ -1778,6 +1781,7 @@
premium-features
query-permissions
query-processor
+ remote-sync
util
warehouses
enterprise/impersonation}}
diff --git a/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.ts b/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.ts
index edca1dad546..8303178b5ab 100644
--- a/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.ts
@@ -35,7 +35,7 @@ describe("bulk table operations", () => {
);
cy.intercept(
"GET",
- `/api/database/${WRITABLE_DB_ID}/schema/public?include_hidden=true&include_editable_data_model=true`,
+ `/api/database/${WRITABLE_DB_ID}/schema/public?include_hidden=true`,
).as("getSchema");
cy.intercept("POST", "/api/ee/data-studio/table/publish-tables").as(
"publishTables",
diff --git a/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.ts b/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.ts
index 517456353aa..d3c59b71899 100644
--- a/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.ts
@@ -36,7 +36,7 @@ describe("scenarios > data studio > datamodel", () => {
cy.signInAsAdmin();
H.activateToken("bleeding-edge");
- cy.intercept("GET", "/api/database?*").as("databases");
+ cy.intercept("GET", "/api/database").as("databases");
cy.intercept("GET", "/api/database/*/schemas?*").as("schemas");
cy.intercept("GET", "/api/table/*/query_metadata*").as("metadata");
cy.intercept("GET", "/api/database/*/schema/*").as("schema");
diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj
index fa78afd67f1..e3042982c9e 100644
--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj
+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj
@@ -3,22 +3,28 @@
[metabase.api.common :as api]
[metabase.permissions.core :as perms]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
+ [metabase.remote-sync.core :as remote-sync]
[metabase.util :as u]
[metabase.warehouses.models.database :as database]
[toucan2.core :as t2]))
(defenterprise current-user-can-write-field?
- "Enterprise version. Returns a boolean whether the current user can write the given field."
+ "Enterprise version. Returns a boolean whether the current user can write the given field.
+ Checks both that the user has manage-table-metadata permission and that the parent table
+ is editable (not in a remote-synced collection in read-only mode)."
:feature :advanced-permissions
[instance]
- (let [db-id (or (get-in instance [:table :db_id])
- (database/table-id->database-id (:table_id instance)))]
- (perms/user-has-permission-for-table?
- api/*current-user-id*
- :perms/manage-table-metadata
- :yes
- db-id
- (:table_id instance))))
+ (let [table (or (:table instance)
+ (t2/select-one :model/Table :id (:table_id instance)))]
+ (and (remote-sync/table-editable? table)
+ (let [db-id (or (:db_id table)
+ (database/table-id->database-id (:table_id instance)))]
+ (perms/user-has-permission-for-table?
+ api/*current-user-id*
+ :perms/manage-table-metadata
+ :yes
+ db-id
+ (:table_id instance))))))
(defenterprise current-user-can-manage-schema-metadata?
"Enterprise version. Returns a boolean whether the current user has permission to edit table metadata for any tables
@@ -43,15 +49,17 @@
db-id))
(defenterprise current-user-can-write-table?
- "Enterprise version."
+ "Enterprise version. Checks both that the user has manage-table-metadata permission and
+ that the table is editable (not in a remote-synced collection in read-only mode)."
:feature :advanced-permissions
[table]
- (perms/user-has-permission-for-table?
- api/*current-user-id*
- :perms/manage-table-metadata
- :yes
- (:db_id table)
- (:id table)))
+ (and (remote-sync/table-editable? table)
+ (perms/user-has-permission-for-table?
+ api/*current-user-id*
+ :perms/manage-table-metadata
+ :yes
+ (:db_id table)
+ (:id table))))
(defn with-advanced-permissions
"Adds to `user` a set of boolean flags indicating whether or not current user has access to advanced permissions.
diff --git a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj
index 14b4cc7d447..8bf1d2cfc29 100644
--- a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj
@@ -201,7 +201,12 @@
set-map (select-keys body set-ks)]
(when (seq set-map)
(t2/update! :model/Table [:in table-ids] set-map)
- (maybe-sync-unhidden-tables! existing-tables set-map))
+ (maybe-sync-unhidden-tables! existing-tables set-map)
+ ;; Publish update events for remote sync tracking
+ (let [updated-tables (t2/select :model/Table :id [:in table-ids])]
+ (doseq [table updated-tables]
+ (events/publish-event! :event/table-update {:object table
+ :user-id api/*current-user-id*}))))
{}))
(api.macros/defendpoint :post "/selection" :- ::bulk-table-selection-info
@@ -242,11 +247,19 @@
upstream-ids (all-upstream-table-ids where)
update-where (if (seq upstream-ids)
[:or where [:in :id upstream-ids]]
- where)]
+ where)
+ ;; Get table IDs before update for event publishing
+ table-ids-to-update (t2/select-pks-set :model/Table {:where update-where})]
(t2/query {:update (t2/table-name :model/Table)
:set {:collection_id (:id target-collection)
:is_published true}
:where update-where})
+ ;; Publish update events for remote sync tracking
+ (when (seq table-ids-to-update)
+ (let [updated-tables (t2/select :model/Table :id [:in table-ids-to-update])]
+ (doseq [table updated-tables]
+ (events/publish-event! :event/table-update {:object table
+ :user-id api/*current-user-id*}))))
{:target_collection target-collection}))
(api.macros/defendpoint :post "/unpublish-tables" :- :nil
@@ -259,11 +272,19 @@
downstream-ids (all-downstream-table-ids where)
update-where (if (seq downstream-ids)
[:or where [:in :id downstream-ids]]
- where)]
+ where)
+ ;; Get table IDs before update for event publishing
+ table-ids-to-update (t2/select-pks-set :model/Table {:where update-where})]
(t2/query {:update (t2/table-name :model/Table)
:set {:collection_id nil
:is_published false}
:where update-where})
+ ;; Publish update events for remote sync tracking
+ (when (seq table-ids-to-update)
+ (let [updated-tables (t2/select :model/Table :id [:in table-ids-to-update])]
+ (doseq [table updated-tables]
+ (events/publish-event! :event/table-update {:object table
+ :user-id api/*current-user-id*}))))
nil))
(defn- sync-schema-async!
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj
index 9dfa4a74e86..8fae4bc6a11 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj
@@ -32,6 +32,27 @@
(or (= (settings/remote-sync-type) :read-write)
(not (collections/remote-synced-collection? collection))))
+(defenterprise table-editable?
+ "Determines if a table's metadata should be editable.
+
+ Takes a table to check for editability.
+
+ Returns true if the table is editable, false otherwise. Returns false if:
+ - remote-sync-type is :read-only AND
+ - table is published AND
+ - table is in a remote-synced collection
+
+ Always returns true on OSS.
+
+ If the table has a pre-hydrated :collection key, uses that to avoid an extra query."
+ :feature :none
+ [table]
+ (or (= (settings/remote-sync-type) :read-write)
+ (not (:is_published table))
+ ;; Use pre-hydrated :collection if available, otherwise fall back to :collection_id
+ (not (collections/remote-synced-collection? (or (:collection table)
+ (:collection_id table))))))
+
(mu/defn bulk-set-remote-sync :- :nil
"Sets remote sync to true/false on one or collections in a single transaction. Checks that the remote sync state
afterwards is consistent in terms of dependency rules. Collections are provided as a map of collection-id -> sync state."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
index aaf67aab587..2af78510ca6 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
@@ -1,11 +1,20 @@
(ns metabase-enterprise.remote-sync.events
- "Event system for remote sync operations and model change tracking.
-
- Provides event publishing and handling for remote sync operations,
- allowing components to react to remote sync state changes.
-
- Also tracks changes to models (cards, dashboards, documents, collections)
- that are part of remote-synced collections."
+ "Event handlers for tracking model changes in remote-synced collections.
+
+ Listens to Metabase model events (create/update/delete) and maintains the
+ RemoteSyncObject table with the current sync status of each tracked model.
+ This enables the remote sync system to know which objects have changed
+ since the last sync operation.
+
+ Handlers are defined using the `defmodel-change-handler` macro which:
+ - Derives event hierarchies for create/update/delete events
+ - Checks if the model is in a remote-synced collection
+ - Creates or updates RemoteSyncObject entries with denormalized model details
+
+ Tracked model types:
+ - Card, Dashboard, Document, NativeQuerySnippet, Timeline, Collection
+ - Table (when published in a remote-synced collection)
+ - Field, Segment (when belonging to a published table in a remote-synced collection)"
(:require
[java-time.api :as t]
[metabase.collections.core :as collections]
@@ -14,7 +23,7 @@
[methodical.core :as methodical]
[toucan2.core :as t2]))
- ;; Helper functions for model change tracking
+;; Helper functions for model change tracking
(defn- model-in-remote-synced-collection?
"Checks if a model (card, dashboard, document) is in a remote-synced collection. Takes a model instance with a
@@ -23,51 +32,28 @@
(boolean
(collections/remote-synced-collection? collection_id)))
-(defmulti ^:private remote-sync-model-details
- "Returns a map of relevant fields for the model."
- {:arglists '([model-type model-id])}
- (fn [model-type _model-id] model-type))
-
-(defmethod remote-sync-model-details "Card"
- [_model-type model-id]
- (t2/select-one [:model/Card :name :collection_id :display] :id model-id))
-
-(defmethod remote-sync-model-details "Dashboard"
- [_model-type model-id]
- (t2/select-one [:model/Dashboard :name :collection_id] :id model-id))
-
-(defmethod remote-sync-model-details "Document"
- [_model-type model-id]
- (t2/select-one [:model/Document :name :collection_id] :id model-id))
-
-(defmethod remote-sync-model-details "NativeQuerySnippet"
- [_model-type model-id]
- (t2/select-one [:model/NativeQuerySnippet :name :id] :id model-id))
-
-(defmethod remote-sync-model-details "Timeline"
- [_model-type model-id]
- (t2/select-one [:model/Timeline :name :collection_id] :id model-id))
-
-(defmethod remote-sync-model-details "Collection"
- [_model-type model-id]
- (t2/select-one [:model/Collection :name [:id :collection_id]] :id model-id))
-
(defn- create-or-update-remote-sync-object-entry!
- "Creates or updates a remote sync object entry for a model change. Takes a model-type (type of model: 'Card',
- 'Dashboard', 'Document', 'Collection'), a model-id (ID of the affected model), a status (status of the sync:
- 'create', 'update', 'removed', 'delete', 'error', 'synced'), and an optional user-id (ID of the user making
- the change). Returns the created or updated remote sync object entry."
- [model-type model-id status & [_user-id]]
+ "Creates or updates a remote sync object entry for a model change.
+
+ Parameters:
+ - model-type: Type of model ('Card', 'Dashboard', 'Document', 'Collection', etc.)
+ - model-id: ID of the affected model
+ - status: Sync status ('create', 'update', 'removed', 'delete', 'error', 'synced')
+ - hydrate-details-fn: Function that takes model-id and returns a map with :name, :collection_id,
+ and optionally :display, :table_id, :table_name"
+ [model-type model-id status hydrate-details-fn]
(let [existing (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)]
(cond
(not existing)
- (let [model-details (remote-sync-model-details model-type model-id)]
+ (let [model-details (hydrate-details-fn model-id)]
(t2/insert! :model/RemoteSyncObject
{:model_type model-type
:model_id model-id
:model_name (:name model-details)
:model_collection_id (:collection_id model-details)
:model_display (some-> model-details :display name)
+ :model_table_id (:table_id model-details)
+ :model_table_name (:table_name model-details)
:status status
:status_changed_at (t/offset-date-time)}))
@@ -83,146 +69,144 @@
;; If the entry was created, the status should remain create until synced
(not (= "create" (:status existing)))
- (let [model (remote-sync-model-details model-type model-id)]
+ (let [model-details (hydrate-details-fn model-id)]
(t2/update! :model/RemoteSyncObject (:id existing)
{:status status
:status_changed_at (t/offset-date-time)
- :model_name (:name model)
- :model_collection_id (:collection_id model)
- :model_display (some-> model :display name)})))))
+ :model_name (:name model-details)
+ :model_collection_id (:collection_id model-details)
+ :model_display (some-> model-details :display name)
+ :model_table_id (:table_id model-details)
+ :model_table_name (:table_name model-details)})))))
;; Model change tracking event handlers
-;; Card events
-(derive ::card-change-event :metabase/event)
-(derive :event/card-create ::card-change-event)
-(derive :event/card-update ::card-change-event)
-(derive :event/card-delete ::card-change-event)
-
-(methodical/defmethod events/publish-event! ::card-change-event
- [topic event]
- (let [{:keys [object user-id]} event
- in-remote-synced? (model-in-remote-synced-collection? object)
- existing-entry (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id object))
- status (if (:archived object)
- "delete"
- (case topic
- :event/card-create "create"
- :event/card-update "update"
- :event/card-delete "delete"))]
- (cond
- ;; Card is in a remote-synced collection - create or update entry
- in-remote-synced?
- (do
- (log/infof "Creating remote sync object entry for card %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "Card" (:id object) status user-id))
-
- ;; Card was tracked but moved out of remote-synced collection - mark as removed
- (and existing-entry (not in-remote-synced?))
- (do
- (log/infof "Card %s moved out of remote-synced collection, marking as removed" (:id object))
- (create-or-update-remote-sync-object-entry! "Card" (:id object) "removed" user-id)))))
-
-;; Dashboard events
-(derive ::dashboard-change-event :metabase/event)
-(derive :event/dashboard-create ::dashboard-change-event)
-(derive :event/dashboard-update ::dashboard-change-event)
-(derive :event/dashboard-delete ::dashboard-change-event)
-
-(methodical/defmethod events/publish-event! ::dashboard-change-event
- [topic event]
- (let [{:keys [object user-id]} event
- in-remote-synced? (model-in-remote-synced-collection? object)
- existing-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id (:id object))
- status (if (:archived object)
- "delete"
- (case topic
- :event/dashboard-create "create"
- :event/dashboard-update "update"
- :event/dashboard-delete "delete"))]
- (cond
- ;; Dashboard is in a remote-synced collection - create or update entry
- in-remote-synced?
- (do
- (log/infof "Creating remote sync object entry for dashboard %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "Dashboard" (:id object) status user-id))
-
- ;; Dashboard was tracked but moved out of remote-synced collection - mark as removed
- (and existing-entry (not in-remote-synced?))
- (do
- (log/infof "Dashboard %s moved out of remote-synced collection, marking as removed" (:id object))
- (create-or-update-remote-sync-object-entry! "Dashboard" (:id object) "removed" user-id)))))
-
-;; Document events
-(derive ::document-change-event :metabase/event)
-(derive :event/document-create ::document-change-event)
-(derive :event/document-update ::document-change-event)
-(derive :event/document-delete ::document-change-event)
-
-(methodical/defmethod events/publish-event! ::document-change-event
- [topic event]
- (let [{:keys [object user-id]} event
- in-remote-synced? (model-in-remote-synced-collection? object)
- existing-entry (t2/select-one :model/RemoteSyncObject :model_type "Document" :model_id (:id object))
- status (if (:archived object)
- "delete"
- (case topic
- :event/document-create "create"
- :event/document-update "update"
- :event/document-delete "delete"))]
- (cond
- ;; Document is in a remote-synced collection - create or update entry
- in-remote-synced?
- (do
- (log/infof "Creating remote sync object entry for document %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "Document" (:id object) status user-id))
-
- ;; Document was tracked but moved out of remote-synced collection - mark as removed
- (and existing-entry (not in-remote-synced?))
- (do
- (log/infof "Document %s moved out of remote-synced collection, marking as removed" (:id object))
- (create-or-update-remote-sync-object-entry! "Document" (:id object) "removed" user-id)))))
-
-;; Native query snippet events
-(derive ::snippet-change-event :metabase/event)
-(derive :event/snippet-create ::snippet-change-event)
-(derive :event/snippet-update ::snippet-change-event)
-(derive :event/snippet-delete ::snippet-change-event)
-
-(methodical/defmethod events/publish-event! ::snippet-change-event
- [topic event]
- (let [{:keys [object user-id]} event
- in-remote-synced? (model-in-remote-synced-collection? object)
- existing-entry (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id object))
- status (if (:archived object)
- "delete"
- (case topic
- :event/snippet-create "create"
- :event/snippet-update "update"
- :event/snippet-delete "delete"))]
- (cond
- ;; Snippet is in a remote-synced collection - create or update entry
- in-remote-synced?
- (do
- (log/infof "Creating remote sync object entry for snippet %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "NativeQuerySnippet" (:id object) status user-id))
-
- ;; Snippet was tracked but moved out of remote-synced collection - mark as removed
- (and existing-entry (not in-remote-synced?))
- (do
- (log/infof "Snippet %s moved out of remote-synced collection, marking as removed" (:id object))
- (create-or-update-remote-sync-object-entry! "NativeQuerySnippet" (:id object) "removed" user-id)))))
+(defmacro ^:private defmodel-change-handler
+ "Defines event derivations and handler for a standard model change event.
+
+ Usage:
+ (defmodel-change-handler card
+ {:model-type \"Card\"
+ :event-prefix :event/card
+ :log-name \"card\"
+ :hydrate-details (fn [id] (t2/select-one [:model/Card :name :collection_id :display] :id id))})
+
+ Configuration options:
+ - :model-type - String for RemoteSyncObject model_type (e.g. \"Card\")
+ - :event-prefix - Keyword prefix for events (e.g. :event/card)
+ - :log-name - String for log messages (e.g. \"card\")
+ - :hydrate-details - Function (fn [model-id]) that returns map with :name, :collection_id,
+ and optionally :display, :table_id, :table_name
+ - :archived-key - Key to check for archived status (default :archived)
+ - :in-sync-pred - Predicate fn (default model-in-remote-synced-collection?)"
+ [event-group {:keys [model-type event-prefix log-name hydrate-details archived-key in-sync-pred]
+ :or {archived-key :archived
+ in-sync-pred `model-in-remote-synced-collection?}}]
+ (let [parent-kw (keyword (str *ns*) (str (name event-group) "-change-event"))
+ create-kw (keyword (namespace event-prefix) (str (name event-prefix) "-create"))
+ update-kw (keyword (namespace event-prefix) (str (name event-prefix) "-update"))
+ delete-kw (keyword (namespace event-prefix) (str (name event-prefix) "-delete"))]
+ `(do
+ (derive ~parent-kw :metabase/event)
+ (derive ~create-kw ~parent-kw)
+ (derive ~update-kw ~parent-kw)
+ (derive ~delete-kw ~parent-kw)
+
+ (methodical/defmethod events/publish-event! ~parent-kw
+ [topic# event#]
+ (let [{:keys [~'object]} event#
+ in-remote-synced?# (~in-sync-pred ~'object)
+ existing-entry# (t2/select-one :model/RemoteSyncObject
+ :model_type ~model-type
+ :model_id (:id ~'object))
+ status# (if (get ~'object ~archived-key)
+ "delete"
+ (case topic#
+ ~create-kw "create"
+ ~update-kw "update"
+ ~delete-kw "delete"))]
+ (cond
+ in-remote-synced?#
+ (do
+ (log/infof "Creating remote sync object entry for %s %s (status: %s)"
+ ~log-name (:id ~'object) status#)
+ (create-or-update-remote-sync-object-entry!
+ ~model-type (:id ~'object) status# ~hydrate-details))
+
+ (and existing-entry# (not in-remote-synced?#))
+ (do
+ (log/infof "%s %s moved out of remote-synced collection, marking as removed"
+ ~log-name (:id ~'object))
+ (create-or-update-remote-sync-object-entry!
+ ~model-type (:id ~'object) "removed" ~hydrate-details))))))))
+
+;; Standard model change handlers
+
+(defmodel-change-handler :card
+ {:model-type "Card"
+ :event-prefix :event/card
+ :log-name "card"
+ :hydrate-details (fn [id] (t2/select-one [:model/Card :name :collection_id :display] :id id))})
+
+(defmodel-change-handler :dashboard
+ {:model-type "Dashboard"
+ :event-prefix :event/dashboard
+ :log-name "dashboard"
+ :hydrate-details (fn [id] (t2/select-one [:model/Dashboard :name :collection_id] :id id))})
+
+(defmodel-change-handler :document
+ {:model-type "Document"
+ :event-prefix :event/document
+ :log-name "document"
+ :hydrate-details (fn [id] (t2/select-one [:model/Document :name :collection_id] :id id))})
+
+(defmodel-change-handler :snippet
+ {:model-type "NativeQuerySnippet"
+ :event-prefix :event/snippet
+ :log-name "snippet"
+ :hydrate-details (fn [id] (t2/select-one [:model/NativeQuerySnippet :name :id] :id id))})
;; Collection create/update events - derive from common parent for shared handling
(derive ::collection-change-event :metabase/event)
(derive :event/collection-create ::collection-change-event)
(derive :event/collection-update ::collection-change-event)
+(defn- hydrate-collection-details
+ "Hydrates details for a Collection."
+ [id]
+ (t2/select-one [:model/Collection :name [:id :collection_id]] :id id))
+
+(defn- hydrate-table-details
+ "Hydrates details for a Table. For tables, table_id and table_name refer to themselves."
+ [id]
+ (when-let [table (t2/select-one [:model/Table :name :collection_id] :id id)]
+ (assoc table :table_id id :table_name (:name table))))
+
+(defn- track-published-tables-in-collection!
+ "When a collection becomes remote-synced, find all published tables in it
+ and create 'create' entries for them in RemoteSyncObject (if not already tracked)."
+ [collection-id]
+ (let [published-tables (t2/select :model/Table
+ :collection_id collection-id
+ :is_published true)]
+ (doseq [table published-tables]
+ (let [existing (t2/select-one :model/RemoteSyncObject
+ :model_type "Table"
+ :model_id (:id table))]
+ ;; Only create entry if not already tracked or was marked as removed/synced
+ (when (or (nil? existing)
+ (contains? #{"removed" "synced"} (:status existing)))
+ (log/infof "Creating remote sync object entry for published table %s in newly remote-synced collection %s"
+ (:id table) collection-id)
+ (create-or-update-remote-sync-object-entry! "Table" (:id table) "create" hydrate-table-details))))))
+
(methodical/defmethod events/publish-event! ::collection-change-event
[topic event]
- (let [{:keys [object user-id]} event
+ (let [{:keys [object]} event
is-remote-synced? (collections/remote-synced-collection? object)
existing-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id object))
+ was-remote-synced? (and existing-entry
+ (not (contains? #{"removed"} (:status existing-entry))))
status (if (:archived object)
"delete"
(case topic
@@ -233,39 +217,82 @@
is-remote-synced?
(do
(log/infof "Creating remote sync object entry for collection %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "Collection" (:id object) status user-id))
+ (create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details)
+ ;; If collection just became remote-synced, track all published tables in it
+ (when (not was-remote-synced?)
+ (track-published-tables-in-collection! (:id object))))
;; Collection was remote-synced but type changed - mark as removed
(and existing-entry (not is-remote-synced?))
(do
(log/infof "Collection %s type changed from remote-synced, marking as removed" (:id object))
- (create-or-update-remote-sync-object-entry! "Collection" (:id object) "removed" user-id)))))
+ (create-or-update-remote-sync-object-entry! "Collection" (:id object) "removed" hydrate-collection-details)))))
-;; Timeline events
-(derive ::timeline-change-event :metabase/event)
-(derive :event/timeline-create ::timeline-change-event)
-(derive :event/timeline-update ::timeline-change-event)
-(derive :event/timeline-delete ::timeline-change-event)
+(defmodel-change-handler :timeline
+ {:model-type "Timeline"
+ :event-prefix :event/timeline
+ :log-name "timeline"
+ :hydrate-details (fn [id] (t2/select-one [:model/Timeline :name :collection_id] :id id))})
-(methodical/defmethod events/publish-event! ::timeline-change-event
- [topic event]
- (let [{:keys [object user-id]} event
- in-remote-synced? (model-in-remote-synced-collection? object)
- existing-entry (t2/select-one :model/RemoteSyncObject :model_type "Timeline" :model_id (:id object))
- status (if (:archived object)
- "delete"
- (case topic
- :event/timeline-create "create"
- :event/timeline-update "update"
- :event/timeline-delete "delete"))]
- (cond
- in-remote-synced?
- (do
- (log/infof "Creating remote sync object entry for timeline %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "Timeline" (:id object) status user-id))
+;; Table events - only track published tables in remote-synced collections
- ;; Timeline was tracked but moved out of remote-synced collection - mark as removed
- (and existing-entry (not in-remote-synced?))
- (do
- (log/infof "Timeline %s moved out of remote-synced collection, marking as removed" (:id object))
- (create-or-update-remote-sync-object-entry! "Timeline" (:id object) "removed" user-id)))))
+(defn- published-table-in-remote-synced-collection?
+ "Check if a table is published AND in a remote-synced collection.
+ Tables are only considered part of remote sync when they are both published
+ and their collection_id points to a remote-synced collection."
+ [{:keys [is_published collection_id]}]
+ (boolean
+ (and is_published
+ (collections/remote-synced-collection? collection_id))))
+
+(defn- model-in-published-table-in-remote-synced-collection?
+ "Check if a model (field, segment) belongs to a published table in a remote-synced collection.
+ The model must have a :table_id that points to a table with :is_published true
+ and :collection_id in a remote-synced collection."
+ [{:keys [table_id]}]
+ (boolean
+ (when table_id
+ (when-let [table (t2/select-one :model/Table :id table_id)]
+ (published-table-in-remote-synced-collection? table)))))
+
+(defmodel-change-handler :table
+ {:model-type "Table"
+ :event-prefix :event/table
+ :log-name "table"
+ :archived-key :archived_at
+ :in-sync-pred published-table-in-remote-synced-collection?
+ :hydrate-details hydrate-table-details})
+
+;; Segment events - track segments in published tables in remote-synced collections
+
+(defn- hydrate-segment-details
+ "Hydrates details for a Segment, including parent table info."
+ [id]
+ (first (t2/query {:select [:s.name :s.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
+ :from [[:segment :s]]
+ :join [[:metabase_table :t] [:= :s.table_id :t.id]]
+ :where [:= :s.id id]})))
+
+(defmodel-change-handler :segment
+ {:model-type "Segment"
+ :event-prefix :event/segment
+ :log-name "segment"
+ :in-sync-pred model-in-published-table-in-remote-synced-collection?
+ :hydrate-details hydrate-segment-details})
+
+;; Field events - track fields in published tables in remote-synced collections
+
+(defn- hydrate-field-details
+ "Hydrates details for a Field, including parent table info."
+ [id]
+ (first (t2/query {:select [:f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
+ :from [[:metabase_field :f]]
+ :join [[:metabase_table :t] [:= :f.table_id :t.id]]
+ :where [:= :f.id id]})))
+
+(defmodel-change-handler :field
+ {:model-type "Field"
+ :event-prefix :event/field
+ :log-name "field"
+ :in-sync-pred model-in-published-table-in-remote-synced-collection?
+ :hydrate-details hydrate-field-details})
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 5cc45153394..bab21e401c4 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -34,30 +34,99 @@
"Card" [:id :name :collection_id :display]
"NativeQuerySnippet" [:id :name]
"Collection" [:id :name [:id :collection_id]]
+ "Segment" [:id :name :table_id]
[:id :name :collection_id]))
(defn- sync-objects!
"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and
inserts new ones for each imported entity, marking them as 'synced' with the given timestamp.
- Takes a timestamp instant and a map of imported entities grouped by model name, where each model maps to a set of
- entity IDs that were imported."
- [timestamp imported-entities-by-model]
+ Takes a timestamp instant, a map of imported entities grouped by model name (for standard models with entity_id),
+ and separate path collections for Table and Field models which use name-based identity."
+ [timestamp imported-entities-by-model table-paths field-paths]
(t2/delete! :model/RemoteSyncObject)
- (let [inserts (->> imported-entities-by-model
- (mapcat (fn [[model entity-ids]]
- (let [fields (model-fields-for-sync model)
- model-kw (keyword "model" model)]
- (t2/select (into [model-kw] fields) :entity_id [:in entity-ids]))))
- (map (fn [{:keys [id name collection_id display] :as model}]
- {:model_type (clojure.core/name (t2/model model))
- :model_id id
- :model_name name
- :model_collection_id collection_id
- :model_display (some-> display clojure.core/name)
- :status "synced"
- :status_changed_at timestamp})))]
- (t2/insert! :model/RemoteSyncObject inserts)))
+ (let [;; Standard models use entity_id UUIDs (excluding Segment which needs special handling)
+ standard-inserts (->> (dissoc imported-entities-by-model "Segment")
+ (mapcat (fn [[model entity-ids]]
+ (when (seq entity-ids)
+ (let [fields (model-fields-for-sync model)
+ model-kw (keyword "model" model)]
+ (t2/select (into [model-kw] fields) :entity_id [:in entity-ids])))))
+ (map (fn [{:keys [id name collection_id display] :as model}]
+ {:model_type (clojure.core/name (t2/model model))
+ :model_id id
+ :model_name name
+ :model_collection_id collection_id
+ :model_display (some-> display clojure.core/name)
+ :model_table_id nil
+ :model_table_name nil
+ :status "synced"
+ :status_changed_at timestamp})))
+ ;; Segments: need to join to get table info
+ segment-entity-ids (get imported-entities-by-model "Segment")
+ segment-inserts (when (seq segment-entity-ids)
+ (->> (t2/query {:select [:s.id :s.name :s.table_id [:t.name :table_name] [:t.collection_id :collection_id]]
+ :from [[:segment :s]]
+ :join [[:metabase_table :t] [:= :s.table_id :t.id]]
+ :where [:in :s.entity_id segment-entity-ids]})
+ (map (fn [{:keys [id name table_id table_name collection_id]}]
+ {:model_type "Segment"
+ :model_id id
+ :model_name name
+ :model_collection_id collection_id
+ :model_display nil
+ :model_table_id table_id
+ :model_table_name table_name
+ :status "synced"
+ :status_changed_at timestamp}))))
+ ;; Tables: batch lookup using join - include name and collection_id
+ ;; For tables, table_id = self id, table_name = self name
+ table-inserts (when (seq table-paths)
+ (->> (t2/query {:select [:t.id :t.name :t.collection_id]
+ :from [[:metabase_table :t]]
+ :join [[:metabase_database :db] [:= :db.id :t.db_id]]
+ :where (into [:or]
+ (for [{:keys [db_name schema table_name]} table-paths]
+ [:and
+ [:= :db.name db_name]
+ (if schema [:= :t.schema schema] [:is :t.schema nil])
+ [:= :t.name table_name]]))})
+ (map (fn [{:keys [id name collection_id]}]
+ {:model_type "Table"
+ :model_id id
+ :model_name name
+ :model_collection_id collection_id
+ :model_display nil
+ :model_table_id id
+ :model_table_name name
+ :status "synced"
+ :status_changed_at timestamp}))))
+ ;; Fields: batch lookup using join through table - include name and collection_id from table
+ field-inserts (when (seq field-paths)
+ (->> (t2/query {:select [:f.id :f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
+ :from [[:metabase_field :f]]
+ :join [[:metabase_table :t] [:= :t.id :f.table_id]
+ [:metabase_database :db] [:= :db.id :t.db_id]]
+ :where (into [:or]
+ (for [{:keys [db_name schema table_name field_name]} field-paths]
+ [:and
+ [:= :db.name db_name]
+ (if schema [:= :t.schema schema] [:is :t.schema nil])
+ [:= :t.name table_name]
+ [:= :f.name field_name]]))})
+ (map (fn [{:keys [id name table_id table_name collection_id]}]
+ {:model_type "Field"
+ :model_id id
+ :model_name name
+ :model_collection_id collection_id
+ :model_display nil
+ :model_table_id table_id
+ :model_table_name table_name
+ :status "synced"
+ :status_changed_at timestamp}))))
+ all-inserts (concat standard-inserts segment-inserts table-inserts field-inserts)]
+ (when (seq all-inserts)
+ (t2/insert! :model/RemoteSyncObject all-inserts))))
(defn- remove-unsynced!
"Deletes any remote sync content that was NOT part of the import.
@@ -156,20 +225,42 @@
:version (source.p/version snapshot)
:message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}
(log/infof (:message <>)))
- (let [ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters [#"collections/.*"]})
+ (let [ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters [#"collections/.*" #"databases/.*"]})
(source.ingestable/wrap-progress-ingestable task-id 0.7))
load-result (serdes/with-cache
(serialization/load-metabase! ingestable-snapshot))
- imported-entities-by-model (->> (:seen load-result)
- (map last) ; Get the last element of each path (the entity itself)
+ seen-paths (:seen load-result)
+ data-model-models #{"Table" "Field"}
+ ;; For standard models (Collection, Card, Segment, etc.), extract entity_id as before
+ imported-entities-by-model (->> seen-paths
+ (remove #(data-model-models (:model (last %))))
+ (map last)
(group-by :model)
(map (fn [[model entities]]
[model (set (map :id entities))]))
- (into {}))]
+ (into {}))
+ ;; For Tables, capture full path for unique identification
+ ;; Path format: [{:model "Database" :id db-name} {:model "Schema" :id schema}? {:model "Table" :id table-name}]
+ table-paths (->> seen-paths
+ (filter #(= "Table" (:model (last %))))
+ (map (fn [path]
+ {:db_name (-> path first :id)
+ :schema (when (= 3 (count path)) (-> path second :id))
+ :table_name (-> path last :id)})))
+ ;; For Fields, path format includes Table
+ ;; [{:model "Database"} {:model "Schema"}? {:model "Table"} {:model "Field" :id field-name}]
+ field-paths (->> seen-paths
+ (filter #(= "Field" (:model (last %))))
+ (map (fn [path]
+ (let [table-idx (dec (count path))]
+ {:db_name (-> path first :id)
+ :schema (when (> (count path) 3) (-> path second :id))
+ :table_name (-> path (nth (dec table-idx)) :id)
+ :field_name (-> path last :id)}))))]
(remote-sync.task/update-progress! task-id 0.8)
(t2/with-transaction [_conn]
(remove-unsynced! (all-top-level-remote-synced-collections) imported-entities-by-model)
- (sync-objects! sync-timestamp imported-entities-by-model)
+ (sync-objects! sync-timestamp imported-entities-by-model table-paths field-paths)
(when (and (nil? (collection/remote-synced-collection)) (= :read-write (settings/remote-sync-type)))
(collection/create-remote-synced-collection!)))
(remote-sync.task/update-progress! task-id 0.95)
@@ -224,8 +315,50 @@
[:= :rso.model_id :c.id]]]
:where [:= :location "/"]})
(map #(str "collections/" (:entity_id %))))
- written-version (source/store! models top-level-removed-prefixes snapshot task-id message)]
+ ;; Add removed table paths - join to get db_name + schema in one query
+ removed-table-paths (->> (t2/query {:select [[:t.name :table_name] :t.schema [:db.name :db_name]]
+ :from [[:metabase_table :t]]
+ :join [[:remote_sync_object :rso]
+ [:and [:= :rso.model_type [:inline "Table"]]
+ [:= :rso.status [:inline "removed"]]
+ [:= :rso.model_id :t.id]]
+ [:metabase_database :db]
+ [:= :db.id :t.db_id]]})
+ (map (fn [{:keys [table_name schema db_name]}]
+ (str/join "/" (cond-> ["databases" db_name]
+ schema (conj "schemas" schema)
+ true (conj "tables" table_name))))))
+ ;; Add removed/deleted segment paths - join to get all path components in one query
+ ;; Segments use entity_id for path construction
+ ;; Include both "removed" (moved out of remote-synced collection) and "delete" (archived)
+ removed-segment-paths (->> (t2/query {:select [:s.entity_id
+ [:t.name :table_name]
+ :t.schema
+ [:db.name :db_name]]
+ :from [[:segment :s]]
+ :join [[:remote_sync_object :rso]
+ [:and [:= :rso.model_type [:inline "Segment"]]
+ [:in :rso.status [[:inline "removed"] [:inline "delete"]]]
+ [:= :rso.model_id :s.id]]
+ [:metabase_table :t] [:= :t.id :s.table_id]
+ [:metabase_database :db] [:= :db.id :t.db_id]]})
+ (map (fn [{:keys [entity_id table_name schema db_name]}]
+ (str/join "/" (cond-> ["databases" db_name]
+ schema (conj "schemas" schema)
+ true (conj "tables" table_name "segments" entity_id))))))
+ all-delete-prefixes (concat top-level-removed-prefixes
+ removed-table-paths
+ removed-segment-paths)
+ written-version (source/store! models all-delete-prefixes snapshot task-id message)]
(remote-sync.task/set-version! task-id written-version))
+ ;; Delete removed Table/Field/Segment entries (they've been deleted from remote)
+ (t2/delete! :model/RemoteSyncObject
+ :model_type [:in ["Table" "Field" "Segment"]]
+ :status "removed")
+ ;; Also delete archived Segment entries (status "delete")
+ (t2/delete! :model/RemoteSyncObject
+ :model_type "Segment"
+ :status "delete")
(t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})))
{:status :success
:version (source.p/version snapshot)}
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj
index ea1e0a0262b..930515adea9 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj
@@ -1,6 +1,12 @@
(ns metabase-enterprise.remote-sync.models.remote-sync-object
+ "Model and queries for tracking remote-synced objects and their dirty state.
+
+ The RemoteSyncObject table stores denormalized information about objects that are
+ tracked for remote synchronization. This includes the object's name, collection_id,
+ and for Field/Segment/Table models, parent table information."
(:require
[clojure.set :as set]
+ [metabase.util :as u]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -10,26 +16,29 @@
(derive :model/RemoteSyncObject :metabase/model)
+;;; ------------------------------------------------- Public API -------------------------------------------------------
+
(defn dirty-global?
"Checks if any collection has changes since the last sync.
-
Returns true if any remote-synced object has a status other than 'synced', false otherwise."
[]
(t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))
(defn dirty-for-global
"Gets all models in any collection that are dirty with their sync status.
-
- Returns a sequence of model maps that have changed since the last remote sync, including details about their
- current state and sync status."
+ Returns a sequence of model maps that have changed since the last remote sync,
+ including details about their current state and sync status."
[]
(->> (t2/select :model/RemoteSyncObject :status [:not= "synced"])
(map #(-> %
- (dissoc :id)
+ (dissoc :id :status_changed_at)
(set/rename-keys {:model_id :id
:model_name :name
:model_type :model
:model_collection_id :collection_id
:model_display :display
- :status :sync_status})))
+ :model_table_id :table_id
+ :model_table_name :table_name
+ :status :sync_status})
+ (update :model u/lower-case-en)))
(into [])))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
index 1865607d2a3..fb163aaee1c 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
@@ -41,7 +41,9 @@
[:authority_level {:optional true} [:maybe :string]]
[:display {:optional true} [:maybe :string]]
[:query_type {:optional true} [:maybe :string]]
- [:description {:optional true} [:maybe :string]]])
+ [:description {:optional true} [:maybe :string]]
+ [:table_id {:optional true} [:maybe pos-int?]]
+ [:table_name {:optional true} [:maybe :string]]])
;;; ------------------------------------------- API Response Schemas -------------------------------------------
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index f00f4371704..dfa13110a32 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -178,10 +178,11 @@
coll-set (get by-model "Collection")
;; When targets are specified, also include Tables found via descendants
;; (published tables in target collections). These are extracted by ID, not all.
- targeted-tables (when (seq targets) (get by-model "Table"))
+ targeted-data-model (when (seq targets)
+ (select-keys by-model serdes.models/data-model-in-collection))
by-model (cond-> (select-keys by-model models)
;; Add Tables back if they were found in descendants
- (seq targeted-tables) (assoc "Table" targeted-tables)
+ (seq targeted-data-model) (merge targeted-data-model)
;; Remove analytics cards from extraction - they have stable entity_ids across instances
;; so cards that reference them can still be exported and imported correctly
(and analytics-card-ids (contains? by-model "Card"))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index d008da6282d..2f370655c2a 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -11,6 +11,13 @@
"Table"
"Channel"])
+(def data-model-in-collection
+ "Data model types that can be found in collections (via published tables).
+ These are extracted by ID when discovered via descendants, even if no-data-model is set."
+ ["Table"
+ "Field"
+ "Segment"])
+
(def content
"Content model types"
["Action"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
index b2406b5b0f4..a33e4b93838 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
@@ -343,7 +343,7 @@
:model/NativeQuerySnippet snippet {:name "Test Snippet"
:content "SELECT 1"
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"NativeQuerySnippet" #{(:entity_id snippet)}})
+ (#'impl/sync-objects! (t/instant) {"NativeQuerySnippet" #{(:entity_id snippet)}} nil nil)
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))]
(is (= "synced" (:status initial-entry)))
@@ -514,8 +514,9 @@
(testing "create-remote-sync-object-entry! creates new entry when none exists"
(mt/with-current-user (mt/user->id :rasta)
(t2/delete! :model/RemoteSyncObject)
- (let [existing-card-id (t2/select-one-fn :id [:model/Card :id])]
- (#'lib.events/create-or-update-remote-sync-object-entry! "Card" existing-card-id "create" 456)
+ (let [existing-card-id (t2/select-one-fn :id [:model/Card :id])
+ hydrate-fn (fn [id] (t2/select-one [:model/Card :name :collection_id :display] :id id))]
+ (#'lib.events/create-or-update-remote-sync-object-entry! "Card" existing-card-id "create" hydrate-fn)
(let [entries (t2/select :model/RemoteSyncObject)]
(is (= 1 (count entries)))
(is (=? {:model_type "Card"
@@ -528,14 +529,15 @@
(mt/with-current-user (mt/user->id :rasta)
(t2/delete! :model/RemoteSyncObject)
(let [clock-t1 (t/mock-clock (t/instant "2024-01-01T10:00:00Z") (t/zone-id "UTC"))
- existing-dashboard-id (t2/select-one-fn :id [:model/Dashboard :id])]
+ existing-dashboard-id (t2/select-one-fn :id [:model/Dashboard :id])
+ hydrate-fn (fn [id] (t2/select-one [:model/Dashboard :name :collection_id] :id id))]
(t/with-clock clock-t1
- (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "update"))
+ (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "update" hydrate-fn))
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id existing-dashboard-id)
initial-time (:status_changed_at initial-entry)
clock-t2 (t/mock-clock (t/instant "2024-01-01T11:00:00Z") (t/zone-id "UTC"))]
(t/with-clock clock-t2
- (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "synced"))
+ (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "synced" hydrate-fn))
(let [entries (t2/select :model/RemoteSyncObject :model_type "Dashboard" :model_id existing-dashboard-id)]
(is (= 1 (count entries)))
(let [update-entry (first entries)]
@@ -548,13 +550,14 @@
(mt/with-current-user (mt/user->id :rasta)
(t2/delete! :model/RemoteSyncObject)
(let [clock-t1 (t/mock-clock (t/instant "2024-01-01T10:00:00Z") (t/zone-id "UTC"))
- existing-dashboard-id (t2/select-one-fn :id [:model/Dashboard :id])]
+ existing-dashboard-id (t2/select-one-fn :id [:model/Dashboard :id])
+ hydrate-fn (fn [id] (t2/select-one [:model/Dashboard :name :collection_id] :id id))]
(t/with-clock clock-t1
- (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "create"))
+ (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "create" hydrate-fn))
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id existing-dashboard-id)
clock-t2 (t/mock-clock (t/instant "2024-01-01T11:00:00Z") (t/zone-id "UTC"))]
(t/with-clock clock-t2
- (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "synced"))
+ (#'lib.events/create-or-update-remote-sync-object-entry! "Dashboard" existing-dashboard-id "synced" hydrate-fn))
(let [entries (t2/select :model/RemoteSyncObject :model_type "Dashboard" :model_id existing-dashboard-id)]
(is (= 1 (count entries)))
(let [update-entry (first entries)]
@@ -562,12 +565,13 @@
(is (= "create" (:status update-entry)))
(is (= (:status_changed_at initial-entry) (:status_changed_at update-entry))))))))))
-(deftest create-remote-sync-object-entry-uses-current-user-test
- (testing "create-remote-sync-object-entry! uses current user when user-id not specified"
+(deftest create-remote-sync-object-entry-with-hydrate-fn-test
+ (testing "create-remote-sync-object-entry! creates entry with hydrated details from hydrate-fn"
(mt/with-current-user (mt/user->id :rasta)
(t2/delete! :model/RemoteSyncObject)
- (let [existing-collection-id (t2/select-one-fn :id [:model/Collection :id])]
- (#'lib.events/create-or-update-remote-sync-object-entry! "Collection" existing-collection-id "create")
+ (let [existing-collection-id (t2/select-one-fn :id [:model/Collection :id])
+ hydrate-fn (fn [id] (t2/select-one [:model/Collection :name [:id :collection_id]] :id id))]
+ (#'lib.events/create-or-update-remote-sync-object-entry! "Collection" existing-collection-id "create" hydrate-fn)
(let [entries (t2/select :model/RemoteSyncObject)]
(is (= 1 (count entries)))
(is (=? {:model_type "Collection"
@@ -582,7 +586,7 @@
:model/Card card {:name "Test Card"
:dataset_query (mt/mbql-query venues)
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Card" #{(:entity_id card)}})
+ (#'impl/sync-objects! (t/instant) {"Card" #{(:entity_id card)}} nil nil)
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))]
(is (= "synced" (:status initial-entry)))
@@ -618,7 +622,7 @@
:model/Collection normal-collection {:name "Normal"}
:model/Dashboard dashboard {:name "Test Dashboard"
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Dashboard" #{(:entity_id dashboard)}})
+ (#'impl/sync-objects! (t/instant) {"Dashboard" #{(:entity_id dashboard)}} nil nil)
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id (:id dashboard))]
(is (= "synced" (:status initial-entry)))
@@ -652,7 +656,7 @@
(mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
:model/Collection normal-collection {:name "Normal"}
:model/Document document {:collection_id (u/the-id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Document" #{(:entity_id document)}})
+ (#'impl/sync-objects! (t/instant) {"Document" #{(:entity_id document)}} nil nil)
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Document" :model_id (:id document))]
(is (= "synced" (:status initial-entry)))
@@ -683,7 +687,7 @@
(deftest existing-collection-type-changed-from-remote-synced-test
(testing "existing collection type changed from remote-synced is marked as removed"
(mt/with-temp [:model/Collection collection {:is_remote_synced true :name "Remote-Sync"}]
- (#'impl/sync-objects! (t/instant) {"Collection" #{(:entity_id collection)}})
+ (#'impl/sync-objects! (t/instant) {"Collection" #{(:entity_id collection)}} nil nil)
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id collection))]
(is (= "synced" (:status initial-entry)))
(events/publish-event! :event/collection-update
@@ -832,7 +836,7 @@
:model/Collection normal-collection {:name "Normal Collection"}
:model/Timeline timeline {:name "Test Timeline"
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Timeline" #{(:entity_id timeline)}})
+ (#'impl/sync-objects! (t/instant) {"Timeline" #{(:entity_id timeline)}} nil nil)
(is (= 1 (count (t2/select :model/RemoteSyncObject))))
(events/publish-event! :event/timeline-update
{:object (assoc timeline :collection_id (:id normal-collection))
@@ -886,3 +890,436 @@
(let [hard-delete-entry (t2/select-one :model/RemoteSyncObject :model_type "Document" :model_id (:id doc))]
(is (= "delete" (:status hard-delete-entry)))
(is (not (nil? hard-delete-entry)))))))
+
+;;; Table Event Tests
+
+(deftest published-table-in-remote-synced-collection-helper-test
+ (testing "published-table-in-remote-synced-collection? returns true for published tables in remote-synced collections"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}]
+ (is (true? (#'lib.events/published-table-in-remote-synced-collection?
+ {:is_published true :collection_id (:id remote-sync-collection)})))))
+ (testing "published-table-in-remote-synced-collection? returns false when not published"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}]
+ (is (false? (#'lib.events/published-table-in-remote-synced-collection?
+ {:is_published false :collection_id (:id remote-sync-collection)})))))
+ (testing "published-table-in-remote-synced-collection? returns false when collection is not remote-synced"
+ (mt/with-temp [:model/Collection normal-collection {:name "Normal"}]
+ (is (false? (#'lib.events/published-table-in-remote-synced-collection?
+ {:is_published true :collection_id (:id normal-collection)})))))
+ (testing "published-table-in-remote-synced-collection? returns false when collection_id is nil"
+ (is (false? (#'lib.events/published-table-in-remote-synced-collection?
+ {:is_published true :collection_id nil})))))
+
+(deftest table-update-event-creates-entry-for-published-table-test
+ (testing "table-update event creates remote sync object entry for published table in remote-synced collection"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/table-update
+ {:object table :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 1 (count entries)))
+ (is (=? {:model_type "Table"
+ :model_id (:id table)
+ :status "update"}
+ (first entries)))))))
+
+(deftest table-update-event-no-entry-for-unpublished-table-test
+ (testing "table-update event doesn't create entry for unpublished table"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published false
+ :collection_id (:id remote-sync-collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/table-update
+ {:object table :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 0 (count entries)))))))
+
+(deftest table-update-event-no-entry-for-normal-collection-test
+ (testing "table-update event doesn't create entry for table in non-remote-synced collection"
+ (mt/with-temp [:model/Collection normal-collection {:name "Normal"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id normal-collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/table-update
+ {:object table :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 0 (count entries)))))))
+
+(deftest table-update-archived-sets-delete-test
+ (testing "table-update event with archived_at sets delete status"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/table-update
+ {:object (assoc table :archived_at (t/offset-date-time))
+ :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 1 (count entries)))
+ (is (=? {:model_type "Table"
+ :model_id (:id table)
+ :status "delete"}
+ (first entries)))))))
+
+(deftest table-unpublished-marks-as-removed-test
+ (testing "table that becomes unpublished is marked as removed"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ ;; First, create an entry with status "synced" directly
+ (t2/insert! :model/RemoteSyncObject {:model_type "Table"
+ :model_id (:id table)
+ :model_name "Test Table"
+ :model_table_id (:id table)
+ :model_table_name "Test Table"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)})
+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (is (= "synced" (:status initial-entry)))
+ ;; Now unpublish the table
+ (events/publish-event! :event/table-update
+ {:object (assoc table :is_published false)
+ :user-id (mt/user->id :rasta)})
+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (is (= "removed" (:status update-entry))))))))
+
+(deftest table-moved-to-normal-collection-marks-as-removed-test
+ (testing "table moved to non-remote-synced collection is marked as removed"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Collection normal-collection {:name "Normal"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ ;; First, create an entry with status "synced" directly
+ (t2/insert! :model/RemoteSyncObject {:model_type "Table"
+ :model_id (:id table)
+ :model_name "Test Table"
+ :model_table_id (:id table)
+ :model_table_name "Test Table"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)})
+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (is (= "synced" (:status initial-entry)))
+ ;; Now move to normal collection
+ (events/publish-event! :event/table-update
+ {:object (assoc table :collection_id (:id normal-collection))
+ :user-id (mt/user->id :rasta)})
+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id (:id table))]
+ (is (= "removed" (:status update-entry))))))))
+
+(deftest table-event-derivation-test
+ (testing "table events properly derive from :metabase/event"
+ (is (isa? ::lib.events/table-change-event :metabase/event))
+ (is (isa? :event/table-create ::lib.events/table-change-event))
+ (is (isa? :event/table-update ::lib.events/table-change-event))
+ (is (isa? :event/table-delete ::lib.events/table-change-event))))
+
+(deftest collection-becomes-remote-synced-tracks-existing-tables-test
+ (testing "when collection becomes remote-synced, existing published tables are tracked with 'create' status"
+ (mt/with-temp [:model/Collection collection {:name "Soon Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ ;; Collection is not yet remote-synced, fire update event
+ (events/publish-event! :event/collection-update
+ {:object collection :user-id (mt/user->id :rasta)})
+ (is (= 0 (count (t2/select :model/RemoteSyncObject))))
+ ;; Now make collection remote-synced
+ (events/publish-event! :event/collection-update
+ {:object (assoc collection :is_remote_synced true)
+ :user-id (mt/user->id :rasta)})
+ (let [table-entries (t2/select :model/RemoteSyncObject :model_type "Table")
+ collection-entries (t2/select :model/RemoteSyncObject :model_type "Collection")]
+ ;; Should have both collection and table entries
+ (is (= 1 (count collection-entries)))
+ (is (= 1 (count table-entries)))
+ (is (=? {:model_type "Table"
+ :model_id (:id table)
+ :status "create"}
+ (first table-entries))))))
+ (testing "unpublished tables are not tracked when collection becomes remote-synced"
+ (mt/with-temp [:model/Collection collection {:name "Soon Remote-Sync"}
+ :model/Table _ {:name "Test Table"
+ :is_published false
+ :collection_id (:id collection)}]
+ (t2/delete! :model/RemoteSyncObject)
+ ;; Make collection remote-synced
+ (events/publish-event! :event/collection-update
+ {:object (assoc collection :is_remote_synced true)
+ :user-id (mt/user->id :rasta)})
+ (let [table-entries (t2/select :model/RemoteSyncObject :model_type "Table")]
+ (is (= 0 (count table-entries)))))))
+
+;;; Segment Event Tests
+
+(deftest model-in-published-table-in-remote-synced-collection-helper-test
+ (testing "model-in-published-table-in-remote-synced-collection? returns true for models in published tables in remote-synced collections"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}]
+ (is (true? (#'lib.events/model-in-published-table-in-remote-synced-collection?
+ {:table_id (:id table)})))))
+ (testing "model-in-published-table-in-remote-synced-collection? returns false when table is not published"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published false
+ :collection_id (:id remote-sync-collection)}]
+ (is (false? (#'lib.events/model-in-published-table-in-remote-synced-collection?
+ {:table_id (:id table)})))))
+ (testing "model-in-published-table-in-remote-synced-collection? returns false when collection is not remote-synced"
+ (mt/with-temp [:model/Collection normal-collection {:name "Normal"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id normal-collection)}]
+ (is (false? (#'lib.events/model-in-published-table-in-remote-synced-collection?
+ {:table_id (:id table)})))))
+ (testing "model-in-published-table-in-remote-synced-collection? returns false when table_id is nil"
+ (is (false? (#'lib.events/model-in-published-table-in-remote-synced-collection?
+ {:table_id nil})))))
+
+(deftest segment-create-event-creates-entry-test
+ (testing "segment-create event creates remote sync object entry for segment in published table in remote-synced collection"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :definition {:source-table (:id table)
+ :filter [:> [:field 1 nil] 0]}}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/segment-create
+ {:object segment :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 1 (count entries)))
+ (is (=? {:model_type "Segment"
+ :model_id (:id segment)
+ :status "create"}
+ (first entries)))))))
+
+(deftest segment-update-event-creates-entry-test
+ (testing "segment-update event creates remote sync object entry with update status"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :definition {:source-table (:id table)
+ :filter [:> [:field 1 nil] 0]}}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/segment-update
+ {:object segment :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 1 (count entries)))
+ (is (=? {:model_type "Segment"
+ :model_id (:id segment)
+ :status "update"}
+ (first entries)))))))
+
+(deftest segment-update-archived-sets-delete-test
+ (testing "segment-update event with archived=true sets delete status"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :definition {:source-table (:id table)
+ :filter [:> [:field 1 nil] 0]}}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/segment-update
+ {:object (assoc segment :archived true)
+ :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 1 (count entries)))
+ (is (=? {:model_type "Segment"
+ :model_id (:id segment)
+ :status "delete"}
+ (first entries)))))))
+
+(deftest segment-delete-event-creates-entry-test
+ (testing "segment-delete event creates remote sync object entry with delete status"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :definition {:source-table (:id table)
+ :filter [:> [:field 1 nil] 0]}}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/segment-delete
+ {:object segment :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 1 (count entries)))
+ (is (=? {:model_type "Segment"
+ :model_id (:id segment)
+ :status "delete"}
+ (first entries)))))))
+
+(deftest segment-event-in-normal-collection-no-entry-test
+ (testing "segment events in non-remote-synced collections don't create entries"
+ (mt/with-temp [:model/Collection normal-collection {:name "Normal"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id normal-collection)}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :definition {:source-table (:id table)
+ :filter [:> [:field 1 nil] 0]}}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/segment-create
+ {:object segment :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 0 (count entries)))))))
+
+(deftest segment-event-unpublished-table-no-entry-test
+ (testing "segment events for unpublished tables don't create entries"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published false
+ :collection_id (:id remote-sync-collection)}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :definition {:source-table (:id table)
+ :filter [:> [:field 1 nil] 0]}}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/segment-create
+ {:object segment :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 0 (count entries)))))))
+
+(deftest existing-segment-table-unpublished-marks-as-removed-test
+ (testing "existing segment is marked as removed when its table is unpublished"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :definition {:source-table (:id table)
+ :filter [:> [:field 1 nil] 0]}}]
+ (t2/delete! :model/RemoteSyncObject)
+ ;; First create a synced entry
+ (t2/insert! :model/RemoteSyncObject {:model_type "Segment"
+ :model_id (:id segment)
+ :model_name "Test Segment"
+ :model_table_id (:id table)
+ :model_table_name "Test Table"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)})
+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Segment" :model_id (:id segment))]
+ (is (= "synced" (:status initial-entry)))
+ ;; Now "unpublish" the table by simulating an update where the segment's table is no longer in sync scope
+ (events/publish-event! :event/segment-update
+ {:object (assoc segment :table_id (:id table))
+ :user-id (mt/user->id :rasta)})
+ ;; Since table is still published, should be "update"
+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Segment" :model_id (:id segment))]
+ (is (= "update" (:status update-entry))))))))
+
+(deftest segment-event-derivation-test
+ (testing "segment events properly derive from :metabase/event"
+ (is (isa? ::lib.events/segment-change-event :metabase/event))
+ (is (isa? :event/segment-create ::lib.events/segment-change-event))
+ (is (isa? :event/segment-update ::lib.events/segment-change-event))
+ (is (isa? :event/segment-delete ::lib.events/segment-change-event))))
+
+;;; Field Event Tests
+
+(deftest field-update-event-creates-entry-test
+ (testing "field-update event creates remote sync object entry for field in published table in remote-synced collection"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}
+ :model/Field field {:name "test_field"
+ :table_id (:id table)
+ :base_type :type/Text}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/field-update
+ {:object field :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 1 (count entries)))
+ (is (=? {:model_type "Field"
+ :model_id (:id field)
+ :status "update"}
+ (first entries)))))))
+
+(deftest field-update-event-no-entry-for-unpublished-table-test
+ (testing "field-update event doesn't create entry for unpublished table"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Table table {:name "Test Table"
+ :is_published false
+ :collection_id (:id remote-sync-collection)}
+ :model/Field field {:name "test_field"
+ :table_id (:id table)
+ :base_type :type/Text}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/field-update
+ {:object field :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 0 (count entries)))))))
+
+(deftest field-update-event-no-entry-for-normal-collection-test
+ (testing "field-update event doesn't create entry for table in non-remote-synced collection"
+ (mt/with-temp [:model/Collection normal-collection {:name "Normal"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id normal-collection)}
+ :model/Field field {:name "test_field"
+ :table_id (:id table)
+ :base_type :type/Text}]
+ (t2/delete! :model/RemoteSyncObject)
+ (events/publish-event! :event/field-update
+ {:object field :user-id (mt/user->id :rasta)})
+ (let [entries (t2/select :model/RemoteSyncObject)]
+ (is (= 0 (count entries)))))))
+
+(deftest existing-field-table-unpublished-marks-as-removed-test
+ (testing "existing field is marked as removed when its table is no longer in sync scope"
+ (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
+ :model/Collection normal-collection {:name "Normal"}
+ :model/Table table {:name "Test Table"
+ :is_published true
+ :collection_id (:id remote-sync-collection)}
+ :model/Field field {:name "test_field"
+ :table_id (:id table)
+ :base_type :type/Text}]
+ (t2/delete! :model/RemoteSyncObject)
+ ;; First create a synced entry
+ (t2/insert! :model/RemoteSyncObject {:model_type "Field"
+ :model_id (:id field)
+ :model_name "test_field"
+ :model_table_id (:id table)
+ :model_table_name "Test Table"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)})
+ (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Field" :model_id (:id field))]
+ (is (= "synced" (:status initial-entry)))
+ ;; Move table to normal collection
+ (t2/update! :model/Table (:id table) {:collection_id (:id normal-collection)})
+ ;; Now trigger field update event - field's table is no longer in sync scope
+ (events/publish-event! :event/field-update
+ {:object field :user-id (mt/user->id :rasta)})
+ (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "Field" :model_id (:id field))]
+ (is (= "removed" (:status update-entry))))))))
+
+(deftest field-event-derivation-test
+ (testing "field events properly derive from :metabase/event"
+ (is (isa? ::lib.events/field-change-event :metabase/event))
+ (is (isa? :event/field-create ::lib.events/field-change-event))
+ (is (isa? :event/field-update ::lib.events/field-change-event))
+ (is (isa? :event/field-delete ::lib.events/field-change-event))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index ab39064de49..9ad21d8cf85 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -274,32 +274,25 @@
(deftest export!-calls-update-progress-with-expected-values-test
(testing "export! calls update-progress! with expected progress values"
- ;; Clear any existing remote-synced collections to ensure consistent progress call count
- (let [existing-synced-ids (t2/select-pks-set :model/Collection :is_remote_synced true)]
- (try
- (when (seq existing-synced-ids)
- (t2/update! :model/Collection :id [:in existing-synced-ids] {:is_remote_synced false}))
- (mt/dataset test-data
- (mt/with-temporary-setting-values [remote-sync-type :read-write]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
- (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}
- :model/Collection _ {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-2xxxx" :location "/"}
- :model/Card _ {:collection_id coll-id}]
- (let [mock-source (test-helpers/create-mock-source)
- progress-calls (atom [])]
- (with-redefs [remote-sync.task/update-progress!
- (fn [task-id progress]
- (swap! progress-calls conj {:task-id task-id :progress progress}))]
- (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
- (is (= :success (:status result)))
- ;; Verify progress was called with expected values
- (is (= 4 (count @progress-calls)))
- (is (= task-id (:task-id (first @progress-calls))))
- ;; Check progress value is expected
- (is (= 0.3 (:progress (first @progress-calls)))))))))))
- (finally
- (when (seq existing-synced-ids)
- (t2/update! :model/Collection :id [:in existing-synced-ids] {:is_remote_synced true})))))))
+ (mt/dataset test-data
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}
+ :model/Collection _ {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-2xxxx" :location "/"}
+ :model/Card _ {:collection_id coll-id}]
+ (let [mock-source (test-helpers/create-mock-source)
+ progress-calls (atom [])]
+ (with-redefs [remote-sync.task/update-progress!
+ (fn [task-id progress]
+ (swap! progress-calls conj {:task-id task-id :progress progress}))]
+ (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (is (= :success (:status result)))
+ ;; Verify progress was called with expected values
+ ;; 1 initial call at 0.3, then N calls during store! for each entity
+ (is (pos? (count @progress-calls)))
+ (is (= task-id (:task-id (first @progress-calls))))
+ ;; Check first progress value is expected (before store!)
+ (is (= 0.3 (:progress (first @progress-calls)))))))))))))
(deftest import!-resets-remote-sync-object-table-test
(testing "import! deletes and recreates RemoteSyncObject table with synced status"
@@ -549,7 +542,8 @@
(mt/with-model-cleanup [:model/Collection]
(let [clear-called? (atom false)]
(mt/with-temp [:model/Collection _ {:name "Remote Collection" :is_remote_synced true :location "/"}]
- (mt/with-temporary-setting-values [remote-sync-enabled false]
+ (mt/with-temporary-setting-values [remote-sync-url nil
+ remote-sync-enabled false]
(with-redefs [collection/clear-remote-synced-collection! (fn [] (reset! clear-called? true))]
(let [result (impl/finish-remote-config!)]
(is (nil? result)
@@ -599,3 +593,326 @@
(is (some? child-collection) "Child collection should be imported")
(is (true? (:is_remote_synced child-collection)) "Child is_remote_synced should be true")
(is (nil? (:type child-collection)) "Child type should be nil"))))))))
+
+;; Table/Field/Segment tracking tests
+
+(deftest export!-deletes-files-for-removed-tables-test
+ (testing "export! deletes files from git source for tables with 'removed' status"
+ (mt/with-model-cleanup [:model/RemoteSyncObject]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id}
+ :model/Collection {coll-id :id}
+ {:name "Active Collection"
+ :is_remote_synced true
+ :entity_id "active-coll-xxxxxxxxx"
+ :location "/"}]
+ ;; Mark the table as 'removed' in RemoteSyncObject
+ (t2/insert! :model/RemoteSyncObject
+ [{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
+ {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
+ (let [initial-files {"main" {"databases/test-db/tables/test-table/test-table.yaml"
+ (test-helpers/generate-table-yaml "test-table" "test-db")}}
+ mock-source (test-helpers/create-mock-source :initial-files initial-files)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (is (= :success (:status result)))
+ (let [files-after-export (get @(:files-atom mock-source) "main")]
+ (is (not (some #(str/includes? % "test-table") (keys files-after-export)))
+ "Removed table files should be deleted after export")))))))))
+
+(deftest export!-deletes-files-for-removed-segments-test
+ (testing "export! deletes files from git source for segments with 'removed' status"
+ (mt/with-model-cleanup [:model/RemoteSyncObject :model/Segment]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id}
+ :model/Collection {coll-id :id}
+ {:name "Active Collection"
+ :is_remote_synced true
+ :entity_id "active-coll-xxxxxxxxx"
+ :location "/"}
+ :model/Segment {segment-id :id}
+ {:name "Test Segment"
+ :table_id table-id
+ :definition {:source-table table-id
+ :filter [:> [:field 1 nil] 0]}
+ :entity_id "test-segment-xxxxxxxx"}]
+ ;; Mark the segment as 'removed' in RemoteSyncObject
+ (t2/insert! :model/RemoteSyncObject
+ [{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
+ {:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
+ (let [initial-files {"main" {"databases/test-db/tables/test-table/segments/test-segment-xxxxxxxx_Test Segment.yaml"
+ (test-helpers/generate-segment-yaml "Test Segment" "test-table" "test-db")}}
+ mock-source (test-helpers/create-mock-source :initial-files initial-files)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (is (= :success (:status result)))
+ (let [files-after-export (get @(:files-atom mock-source) "main")]
+ (is (not (some #(str/includes? % "Test Segment") (keys files-after-export)))
+ "Removed segment files should be deleted after export")))))))))
+
+(deftest export!-cleans-up-removed-table-entries-test
+ (testing "export! deletes RemoteSyncObject entries for 'removed' Tables/Fields/Segments after successful export"
+ (mt/with-model-cleanup [:model/RemoteSyncObject :model/Segment]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id}
+ :model/Field {field-id :id} {:name "test-field" :table_id table-id :base_type :type/Text}
+ :model/Collection {coll-id :id}
+ {:name "Active Collection"
+ :is_remote_synced true
+ :entity_id "active-coll-xxxxxxxxx"
+ :location "/"}
+ :model/Segment {segment-id :id}
+ {:name "Test Segment"
+ :table_id table-id
+ :definition {:source-table table-id
+ :filter [:> [:field 1 nil] 0]}
+ :entity_id "test-segment-xxxxxxxx"}]
+ ;; Create RemoteSyncObject entries with 'removed' status
+ (t2/insert! :model/RemoteSyncObject
+ [{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
+ {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
+ {:model_type "Field" :model_id field-id :model_name "test-field" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
+ {:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
+ ;; Verify entries exist before export
+ (is (= 4 (t2/count :model/RemoteSyncObject)))
+ (let [mock-source (test-helpers/create-mock-source)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (is (= :success (:status result)))
+ ;; After export, removed Table/Field/Segment entries should be deleted
+ (is (nil? (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id table-id))
+ "Table entry should be deleted after export")
+ (is (nil? (t2/select-one :model/RemoteSyncObject :model_type "Field" :model_id field-id))
+ "Field entry should be deleted after export")
+ (is (nil? (t2/select-one :model/RemoteSyncObject :model_type "Segment" :model_id segment-id))
+ "Segment entry should be deleted after export")
+ ;; Collection should remain and be marked as synced
+ (let [coll-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)]
+ (is (some? coll-entry) "Collection entry should remain")
+ (is (= "synced" (:status coll-entry)) "Collection should be marked as synced")))))))))
+
+(deftest export!-generates-correct-delete-paths-for-tables-with-schema-test
+ (testing "export! generates correct delete paths for tables with schema"
+ (mt/with-model-cleanup [:model/RemoteSyncObject]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id :schema "PUBLIC"}
+ :model/Collection {coll-id :id}
+ {:name "Active Collection"
+ :is_remote_synced true
+ :entity_id "active-coll-xxxxxxxxx"
+ :location "/"}]
+ ;; Mark the table as 'removed' in RemoteSyncObject
+ (t2/insert! :model/RemoteSyncObject
+ [{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
+ {:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
+ (let [;; Path should include schema: databases/{db}/schemas/{schema}/tables/{table}
+ initial-files {"main" {"databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"
+ (test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}
+ mock-source (test-helpers/create-mock-source :initial-files initial-files)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (is (= :success (:status result)))
+ (let [files-after-export (get @(:files-atom mock-source) "main")]
+ (is (not (some #(str/includes? % "test-table") (keys files-after-export)))
+ "Removed table files should be deleted after export (including schema path)")))))))))
+
+(deftest export!-excludes-archived-segments-test
+ (testing "export! excludes archived segments from export (via skip-archived flag)"
+ (mt/with-model-cleanup [:model/RemoteSyncObject :model/Segment]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Collection {coll-id :id}
+ {:name "Test Collection"
+ :is_remote_synced true
+ :entity_id "test-collection-1xxxx"
+ :location "/"}
+ ;; Table must have collection_id and is_published to be included as Collection descendant
+ :model/Table {table-id :id} {:name "test-table"
+ :db_id db-id
+ :collection_id coll-id
+ :is_published true}
+ :model/Segment {_active-seg-id :id}
+ {:name "Active Segment"
+ :table_id table-id
+ :definition {:source-table table-id
+ :filter [:> [:field 1 nil] 0]}
+ :entity_id "active-segment-xxxxxx"
+ :archived false}
+ :model/Segment {_archived-seg-id :id}
+ {:name "Archived Segment"
+ :table_id table-id
+ :definition {:source-table table-id
+ :filter [:> [:field 2 nil] 0]}
+ :entity_id "archived-segment-xxxx"
+ :archived true}]
+ ;; Track the collection so export runs
+ (t2/insert! :model/RemoteSyncObject
+ [{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])
+ (let [mock-source (test-helpers/create-mock-source)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (is (= :success (:status result)))
+ (let [files-after-export (get @(:files-atom mock-source) "main")
+ file-keys (keys files-after-export)]
+ ;; File paths use entity_id, e.g. "active-segment-xxxxxx_active_segment.yaml"
+ (is (some #(str/includes? % "active-segment-xxxxxx") file-keys)
+ "Active segment should be exported")
+ (is (not (some #(str/includes? % "archived-segment-xxxx") file-keys))
+ "Archived segment should NOT be exported")))))))))
+
+(deftest export!-deletes-files-for-archived-segments-test
+ (testing "export! deletes files from git source for segments with 'delete' status (archived segments)"
+ (mt/with-model-cleanup [:model/RemoteSyncObject :model/Segment]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id}
+ :model/Collection {coll-id :id}
+ {:name "Active Collection"
+ :is_remote_synced true
+ :entity_id "active-coll-xxxxxxxxx"
+ :location "/"}
+ :model/Segment {segment-id :id}
+ {:name "Archived Segment"
+ :table_id table-id
+ :definition {:source-table table-id
+ :filter [:> [:field 1 nil] 0]}
+ :entity_id "archived-seg-xxxxxxxx"
+ :archived true}]
+ ;; Mark the segment as 'delete' in RemoteSyncObject (simulating archival event)
+ (t2/insert! :model/RemoteSyncObject
+ [{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
+ {:model_type "Segment" :model_id segment-id :model_name "Archived Segment" :model_table_id table-id :model_table_name "test-table" :status "delete" :status_changed_at (t/offset-date-time)}])
+ (let [initial-files {"main" {"databases/test-db/tables/test-table/segments/archived-seg-xxxxxxxx_Archived Segment.yaml"
+ (test-helpers/generate-segment-yaml "Archived Segment" "test-table" "test-db")}}
+ mock-source (test-helpers/create-mock-source :initial-files initial-files)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (is (= :success (:status result)))
+ (let [files-after-export (get @(:files-atom mock-source) "main")]
+ (is (not (some #(str/includes? % "archived-seg-xxxxxxxx") (keys files-after-export)))
+ "Archived segment files should be deleted after export"))
+ (testing "RemoteSyncObject entry is cleaned up after export"
+ (is (nil? (t2/select-one :model/RemoteSyncObject :model_type "Segment" :model_id segment-id))
+ "RemoteSyncObject entry for archived segment should be deleted")))))))))
+
+;; Import Table/Field/Segment tracking tests
+
+(deftest import!-tracks-tables-in-remote-sync-object-test
+ (testing "import! creates RemoteSyncObject entries for imported Tables"
+ (mt/with-model-cleanup [:model/RemoteSyncObject]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id}
+ :model/Collection {coll-id :id}
+ {:name "Test Collection"
+ :is_remote_synced true
+ :entity_id "test-collection-1xxxx"
+ :location "/"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
+ "databases/test-db/tables/test-table/test-table.yaml"
+ (test-helpers/generate-table-yaml "test-table" "test-db")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ ;; Verify RemoteSyncObject entries exist
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id coll-id
+ :status "synced")
+ "Collection should be tracked in RemoteSyncObject")
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Table"
+ :model_id table-id
+ :status "synced")
+ "Table should be tracked in RemoteSyncObject")))))))
+
+(deftest import!-tracks-fields-in-remote-sync-object-test
+ (testing "import! creates RemoteSyncObject entries for imported Fields"
+ (mt/with-model-cleanup [:model/RemoteSyncObject]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id}
+ :model/Field {field-id :id} {:name "test-field" :table_id table-id :base_type :type/Text}
+ :model/Collection _ {:name "Test Collection"
+ :is_remote_synced true
+ :entity_id "test-collection-1xxxx"
+ :location "/"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
+ "databases/test-db/tables/test-table/fields/test-field.yaml"
+ (test-helpers/generate-field-yaml "test-field" "test-table" "test-db")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ ;; Verify RemoteSyncObject entry exists for the field
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Field"
+ :model_id field-id
+ :status "synced")
+ "Field should be tracked in RemoteSyncObject")))))))
+
+(deftest import!-tracks-segments-in-remote-sync-object-test
+ (testing "import! creates RemoteSyncObject entries for imported Segments"
+ (mt/with-model-cleanup [:model/RemoteSyncObject :model/Segment]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id}
+ :model/Field {field-id :id} {:name "test-field" :table_id table-id :base_type :type/Integer}
+ :model/Collection _ {:name "Test Collection"
+ :is_remote_synced true
+ :entity_id "test-collection-1xxxx"
+ :location "/"}
+ :model/Segment {segment-id :id}
+ {:name "Test Segment"
+ :table_id table-id
+ :definition {:source-table table-id
+ :filter [:> [:field field-id nil] 0]}
+ :entity_id "TNdMrOCMHrQc_UtvCbTC5"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
+ "databases/test-db/tables/test-table/test-table.yaml"
+ (test-helpers/generate-table-yaml "test-table" "test-db")
+ "databases/test-db/tables/test-table/fields/test-field.yaml"
+ (test-helpers/generate-field-yaml "test-field" "test-table" "test-db" :base-type "type/Integer" :database-type "INTEGER")
+ "databases/test-db/tables/test-table/segments/TNdMrOCMHrQc_UtvCbTC5_test_segment.yaml"
+ (test-helpers/generate-segment-yaml "Test Segment" "test-table" "test-db"
+ :entity-id "TNdMrOCMHrQc_UtvCbTC5"
+ :filter-field-name "test-field")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ ;; Verify RemoteSyncObject entry exists for the segment
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Segment"
+ :model_id segment-id
+ :status "synced")
+ "Segment should be tracked in RemoteSyncObject")))))))
+
+(deftest import!-tracks-tables-with-schema-in-remote-sync-object-test
+ (testing "import! creates RemoteSyncObject entries for imported Tables with schema"
+ (mt/with-model-cleanup [:model/RemoteSyncObject]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:name "test-table" :db_id db-id :schema "PUBLIC"}
+ :model/Collection _ {:name "Test Collection"
+ :is_remote_synced true
+ :entity_id "test-collection-1xxxx"
+ :location "/"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
+ "databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"
+ (test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ ;; Verify RemoteSyncObject entry exists for the table
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Table"
+ :model_id table-id
+ :status "synced")
+ "Table with schema should be tracked in RemoteSyncObject")))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.clj
index a8d1bf7a0bf..71ceb441e18 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.clj
@@ -347,3 +347,191 @@
:status_changed_at (java.time.OffsetDateTime/now)}]
(is (true? (rs-object/dirty-global?)))
(is (= 1 (count (rs-object/dirty-for-global))))))))
+
+;;; ------------------------------------------------------------------------------------------------
+;;; Tests for Table, Field, and Segment in dirty-state queries
+;;; ------------------------------------------------------------------------------------------------
+
+(deftest dirty-for-global-includes-tables-test
+ (testing "dirty-for-global includes table items with table_id and table_name"
+ (mt/with-temp
+ [:model/Collection collection {:name "Test Collection"
+ :location "/"}
+ :model/Table table {:name "Test Table"
+ :collection_id (:id collection)
+ :is_published true}
+ :model/RemoteSyncObject _ {:model_type "Table"
+ :model_id (:id table)
+ :model_name "Test Table"
+ :model_collection_id (:id collection)
+ :model_table_id (:id table)
+ :model_table_name "Test Table"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}]
+ (let [dirty-items (rs-object/dirty-for-global)]
+ (is (= 1 (count dirty-items)))
+ (let [item (first dirty-items)]
+ (is (= (:id table) (:id item)))
+ (is (= "Test Table" (:name item)))
+ (is (= "table" (:model item)))
+ (is (= "pending" (:sync_status item)))
+ ;; For tables, table_id and table_name refer to themselves
+ (is (= (:id table) (:table_id item)))
+ (is (= "Test Table" (:table_name item))))))))
+
+(deftest dirty-for-global-includes-fields-test
+ (testing "dirty-for-global includes field items with table_id and table_name from parent table"
+ (mt/with-temp
+ [:model/Collection collection {:name "Test Collection"
+ :location "/"}
+ :model/Table table {:name "Test Table"
+ :collection_id (:id collection)
+ :is_published true}
+ :model/Field field {:name "test_field"
+ :table_id (:id table)
+ :base_type :type/Text
+ :database_type "TEXT"}
+ :model/RemoteSyncObject _ {:model_type "Field"
+ :model_id (:id field)
+ :model_name "test_field"
+ :model_collection_id (:id collection)
+ :model_table_id (:id table)
+ :model_table_name "Test Table"
+ :status "update"
+ :status_changed_at (java.time.OffsetDateTime/now)}]
+ (let [dirty-items (rs-object/dirty-for-global)]
+ (is (= 1 (count dirty-items)))
+ (let [item (first dirty-items)]
+ (is (= (:id field) (:id item)))
+ (is (= "test_field" (:name item)))
+ (is (= "field" (:model item)))
+ (is (= "update" (:sync_status item)))
+ ;; collection_id should come from the parent table
+ (is (= (:id collection) (:collection_id item)))
+ ;; table_id and table_name should be from the parent table
+ (is (= (:id table) (:table_id item)))
+ (is (= "Test Table" (:table_name item))))))))
+
+(deftest dirty-for-global-includes-segments-test
+ (testing "dirty-for-global includes segment items with table_id and table_name from parent table"
+ (mt/with-temp
+ [:model/Collection collection {:name "Test Collection"
+ :location "/"}
+ :model/Table table {:name "Test Table"
+ :collection_id (:id collection)
+ :is_published true}
+ :model/Field field {:name "test_field"
+ :table_id (:id table)
+ :base_type :type/Text
+ :database_type "TEXT"}
+ :model/Segment segment {:name "Test Segment"
+ :table_id (:id table)
+ :creator_id (mt/user->id :rasta)
+ :definition {:source-table (:id table)
+ :filter [:= [:field (:id field) nil] "test"]}}
+ :model/RemoteSyncObject _ {:model_type "Segment"
+ :model_id (:id segment)
+ :model_name "Test Segment"
+ :model_collection_id (:id collection)
+ :model_table_id (:id table)
+ :model_table_name "Test Table"
+ :status "create"
+ :status_changed_at (java.time.OffsetDateTime/now)}]
+ (let [dirty-items (rs-object/dirty-for-global)]
+ (is (= 1 (count dirty-items)))
+ (let [item (first dirty-items)]
+ (is (= (:id segment) (:id item)))
+ (is (= "Test Segment" (:name item)))
+ (is (= "segment" (:model item)))
+ (is (= "create" (:sync_status item)))
+ ;; collection_id should come from the parent table
+ (is (= (:id collection) (:collection_id item)))
+ ;; table_id and table_name should be from the parent table
+ (is (= (:id table) (:table_id item)))
+ (is (= "Test Table" (:table_name item))))))))
+
+(deftest dirty-for-global-all-model-types-including-new-test
+ (testing "dirty-for-global handles all supported model types including Table, Field, and Segment"
+ (mt/with-temp
+ [:model/Collection collection {:name "Test Collection"
+ :location "/"}
+ :model/Collection snip-collection {:name "Snippet Collection"
+ :namespace "snippets"
+ :location "/"}
+ :model/Card card {:name "Card Item"
+ :collection_id (:id collection)
+ :creator_id (mt/user->id :rasta)
+ :display "table"
+ :dataset_query (mt/native-query {:query "SELECT 1"})}
+ :model/Dashboard dashboard {:name "Dashboard Item"
+ :collection_id (:id collection)
+ :creator_id (mt/user->id :rasta)}
+ :model/Document document {:name "Document Item"
+ :collection_id (:id collection)
+ :creator_id (mt/user->id :rasta)}
+ :model/NativeQuerySnippet snippet {:name "Snippet Item"
+ :collection_id (:id snip-collection)
+ :creator_id (mt/user->id :rasta)
+ :content "SELECT * FROM table"}
+ :model/Table table {:name "Table Item"
+ :collection_id (:id collection)
+ :is_published true}
+ :model/Field field {:name "field_item"
+ :table_id (:id table)
+ :base_type :type/Text
+ :database_type "TEXT"}
+ :model/Segment segment {:name "Segment Item"
+ :table_id (:id table)
+ :creator_id (mt/user->id :rasta)
+ :definition {:source-table (:id table)
+ :filter [:= [:field (:id field) nil] "test"]}}
+ :model/RemoteSyncObject _ {:model_type "Collection"
+ :model_id (:id collection)
+ :model_name "Test Collection"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}
+ :model/RemoteSyncObject _ {:model_type "Card"
+ :model_id (:id card)
+ :model_name "Card Item"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}
+ :model/RemoteSyncObject _ {:model_type "Dashboard"
+ :model_id (:id dashboard)
+ :model_name "Dashboard Item"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}
+ :model/RemoteSyncObject _ {:model_type "Document"
+ :model_id (:id document)
+ :model_name "Document Item"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}
+ :model/RemoteSyncObject _ {:model_type "NativeQuerySnippet"
+ :model_id (:id snippet)
+ :model_name "Snippet Item"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}
+ :model/RemoteSyncObject _ {:model_type "Table"
+ :model_id (:id table)
+ :model_name "Table Item"
+ :model_table_id (:id table)
+ :model_table_name "Table Item"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}
+ :model/RemoteSyncObject _ {:model_type "Field"
+ :model_id (:id field)
+ :model_name "field_item"
+ :model_table_id (:id table)
+ :model_table_name "Table Item"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}
+ :model/RemoteSyncObject _ {:model_type "Segment"
+ :model_id (:id segment)
+ :model_name "Segment Item"
+ :model_table_id (:id table)
+ :model_table_name "Table Item"
+ :status "pending"
+ :status_changed_at (java.time.OffsetDateTime/now)}]
+ (let [dirty-items (rs-object/dirty-for-global)
+ models (set (map :model dirty-items))]
+ (is (= 8 (count dirty-items)))
+ (is (= #{"collection" "card" "dashboard" "document" "nativequerysnippet" "table" "field" "segment"} models))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj
index 78a58f54f71..be724afee22 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj
@@ -365,3 +365,193 @@
(testing "Regular collection remains editable by superuser"
(is (true? (mi/can-write? regular-coll))
"Regular collection should be writable")))))))))
+
+;;; ------------------------------------------------ Table Remote Sync Tests ------------------------------------------------
+
+(deftest table-remote-synced-permissions-read-only-test
+ (testing "can_write should be false for published tables in remote-synced collections when remote-sync-type is read-only"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-only]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Published tables in remote-synced collections have can_write=false"
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published true
+ :collection_id library-coll-id})
+ (try
+ (is (false? (mi/can-write? (t2/select-one :model/Table :id (mt/id :venues))))
+ "Published table in remote-synced collection should not be writable when remote-sync-type is read-only")
+ (finally
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published false
+ :collection_id nil})))))))))
+
+(deftest table-remote-synced-permissions-read-write-test
+ (testing "can_write should be true for published tables in remote-synced collections when remote-sync-type is read-write"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-write]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Published tables in remote-synced collections have can_write=true"
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published true
+ :collection_id library-coll-id})
+ (try
+ (is (true? (mi/can-write? (t2/select-one :model/Table :id (mt/id :venues))))
+ "Published table in remote-synced collection should be writable when remote-sync-type is read-write")
+ (finally
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published false
+ :collection_id nil})))))))))
+
+(deftest table-unpublished-in-remote-synced-collection-test
+ (testing "can_write should be true for unpublished tables in remote-synced collections regardless of remote-sync-type"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-only]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Unpublished tables in remote-synced collections have can_write=true"
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published false
+ :collection_id library-coll-id})
+ (try
+ (is (true? (mi/can-write? (t2/select-one :model/Table :id (mt/id :venues))))
+ "Unpublished table in remote-synced collection should be writable even when remote-sync-type is read-only")
+ (finally
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published false
+ :collection_id nil})))))))))
+
+(deftest table-regular-collection-unaffected-test
+ (testing "can_write for tables in regular collections should be unaffected by remote-sync-type setting"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Collection {regular-coll-id :id} {:name "Regular Collection"
+ :type nil}]
+ (doseq [remote-sync-setting [:read-only :read-write]]
+ (testing (str "When remote-sync-type is " remote-sync-setting)
+ (mt/with-temporary-setting-values [settings/remote-sync-type remote-sync-setting]
+ (testing "Published tables in regular collections have can_write=true"
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published true
+ :collection_id regular-coll-id})
+ (try
+ (is (true? (mi/can-write? (t2/select-one :model/Table :id (mt/id :venues))))
+ "Published table in regular collection should be writable regardless of remote-sync-type")
+ (finally
+ (t2/update! :model/Table (mt/id :venues)
+ {:is_published false
+ :collection_id nil})))))))))))
+
+;;; ------------------------------------------------ Field Remote Sync Tests ------------------------------------------------
+
+(deftest field-remote-synced-permissions-read-only-test
+ (testing "can_write should be false for fields of published tables in remote-synced collections when remote-sync-type is read-only"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-only]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Fields on published tables in remote-synced collections have can_write=false"
+ (let [table-id (mt/id :venues)
+ field-id (mt/id :venues :name)]
+ (t2/update! :model/Table table-id
+ {:is_published true
+ :collection_id library-coll-id})
+ (try
+ (is (false? (mi/can-write? (t2/select-one :model/Field :id field-id)))
+ "Field on published table in remote-synced collection should not be writable when remote-sync-type is read-only")
+ (finally
+ (t2/update! :model/Table table-id
+ {:is_published false
+ :collection_id nil}))))))))))
+
+(deftest field-remote-synced-permissions-read-write-test
+ (testing "can_write should be true for fields of published tables in remote-synced collections when remote-sync-type is read-write"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-write]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Fields on published tables in remote-synced collections have can_write=true"
+ (let [table-id (mt/id :venues)
+ field-id (mt/id :venues :name)]
+ (t2/update! :model/Table table-id
+ {:is_published true
+ :collection_id library-coll-id})
+ (try
+ (is (true? (mi/can-write? (t2/select-one :model/Field :id field-id)))
+ "Field on published table in remote-synced collection should be writable when remote-sync-type is read-write")
+ (finally
+ (t2/update! :model/Table table-id
+ {:is_published false
+ :collection_id nil}))))))))))
+
+;;; ------------------------------------------------ Segment Remote Sync Tests ------------------------------------------------
+
+(deftest segment-remote-synced-permissions-read-only-test
+ (testing "can_write should be false for segments of published tables in remote-synced collections when remote-sync-type is read-only"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-only]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Segments on published tables in remote-synced collections have can_write=false"
+ (let [table-id (mt/id :venues)]
+ (t2/update! :model/Table table-id
+ {:is_published true
+ :collection_id library-coll-id})
+ (try
+ ;; Use empty definition {} which is valid for segments
+ (mt/with-temp [:model/Segment {segment-id :id} {:name "Test Segment"
+ :table_id table-id
+ :definition {}}]
+ (is (false? (mi/can-write? (t2/select-one :model/Segment :id segment-id)))
+ "Segment on published table in remote-synced collection should not be writable when remote-sync-type is read-only"))
+ (finally
+ (t2/update! :model/Table table-id
+ {:is_published false
+ :collection_id nil}))))))))))
+
+(deftest segment-remote-synced-permissions-read-write-test
+ (testing "can_write should be true for segments of published tables in remote-synced collections when remote-sync-type is read-write"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-write]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Segments on published tables in remote-synced collections have can_write=true"
+ (let [table-id (mt/id :venues)]
+ (t2/update! :model/Table table-id
+ {:is_published true
+ :collection_id library-coll-id})
+ (try
+ ;; Use empty definition {} which is valid for segments
+ (mt/with-temp [:model/Segment {segment-id :id} {:name "Test Segment"
+ :table_id table-id
+ :definition {}}]
+ (is (true? (mi/can-write? (t2/select-one :model/Segment :id segment-id)))
+ "Segment on published table in remote-synced collection should be writable when remote-sync-type is read-write"))
+ (finally
+ (t2/update! :model/Table table-id
+ {:is_published false
+ :collection_id nil}))))))))))
+
+(deftest segment-creation-blocked-in-read-only-mode-test
+ (testing "can_create should be false for segments on published tables in remote-synced collections when remote-sync-type is read-only"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temporary-setting-values [settings/remote-sync-type :read-only]
+ (mt/with-temp [:model/Collection {library-coll-id :id} {:name "Library Collection"
+ :is_remote_synced true}]
+ (testing "Segment creation should be blocked on published tables in remote-synced collections"
+ (let [table-id (mt/id :venues)]
+ (t2/update! :model/Table table-id
+ {:is_published true
+ :collection_id library-coll-id})
+ (try
+ ;; Pass the table directly to can-create? so it uses the updated values
+ (let [table (t2/select-one :model/Table :id table-id)]
+ (is (false? (mi/can-create? :model/Segment {:table_id table-id
+ :table table
+ :name "New Segment"
+ :definition {}}))
+ "Segment creation should be blocked on published table in remote-synced collection when remote-sync-type is read-only"))
+ (finally
+ (t2/update! :model/Table table-id
+ {:is_published false
+ :collection_id nil}))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 6ce75131275..ba3e0f9e810 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -318,3 +318,196 @@ width: fixed
(def clean-remote-sync-state
"Fixture to make sure sync state is clean"
(t/compose-fixtures clean-object clean-task-table))
+
+(defn generate-table-yaml
+ "Generates YAML content for a table.
+
+ Args:
+ table-name: The name of the table (used as entity_id).
+ db-name: The name of the database containing this table.
+
+ Returns:
+ A string containing the YAML representation of the table."
+ [table-name db-name & {:keys [schema is-published description]
+ :or {is-published true
+ description nil}}]
+ (format "name: %s
+description: %s
+entity_type: entity/GenericTable
+active: true
+display_name: %s
+visibility_type: null
+schema: %s
+points_of_interest: null
+caveats: null
+show_in_getting_started: false
+field_order: database
+initial_sync_status: complete
+is_upload: false
+database_require_filter: false
+is_defective_duplicate: false
+is_writable: false
+data_authority: unconfigured
+data_source: null
+owner_email: null
+owner_user_id: null
+is_published: %s
+created_at: '2024-08-28T09:46:18.671622Z'
+archived_at: null
+deactivated_at: null
+data_layer: null
+db_id: %s
+collection_id: null
+serdes/meta:
+- id: %s
+ model: Database
+%s- id: %s
+ model: Table
+"
+ table-name
+ (or description "null")
+ (str/replace (u/upper-case-en table-name) #"_" " ")
+ (or schema "null")
+ is-published
+ db-name
+ db-name
+ (if schema (format "- id: %s\n model: Schema\n" schema) "")
+ table-name))
+
+(defn generate-field-yaml
+ "Generates YAML content for a field.
+
+ Args:
+ field-name: The name of the field (used as entity_id).
+ table-name: The name of the table containing this field.
+ db-name: The name of the database containing this field.
+
+ Returns:
+ A string containing the YAML representation of the field."
+ [field-name table-name db-name & {:keys [schema base-type description database-type]
+ :or {base-type "type/Text"
+ database-type "VARCHAR"
+ description nil}}]
+ (format "name: %s
+display_name: %s
+description: %s
+created_at: '2024-08-28T09:46:18.671622Z'
+active: true
+visibility_type: normal
+table_id:
+- %s
+- %s
+- %s
+database_type: %s
+base_type: %s
+effective_type: null
+semantic_type: null
+database_is_auto_increment: false
+database_required: false
+fk_target_field_id: null
+dimensions: []
+json_unfolding: false
+parent_id: null
+coercion_strategy: null
+preview_display: true
+position: 1
+custom_position: 0
+database_position: 0
+has_field_values: null
+settings: null
+caveats: null
+points_of_interest: null
+nfc_path: null
+serdes/meta:
+- id: %s
+ model: Database
+%s- id: %s
+ model: Table
+- id: %s
+ model: Field
+database_default: null
+database_indexed: null
+database_is_generated: null
+database_is_nullable: null
+database_is_pk: null
+database_partitioned: null
+"
+ field-name
+ (str/replace (u/upper-case-en field-name) #"_" " ")
+ (or description "null")
+ db-name
+ (or schema "null")
+ table-name
+ database-type
+ base-type
+ db-name
+ (if schema (format "- id: %s\n model: Schema\n" schema) "")
+ table-name
+ field-name))
+
+(defn generate-segment-yaml
+ "Generates YAML content for a segment.
+
+ Args:
+ segment-name: The name of the segment (used as entity_id).
+ table-name: The name of the table containing this segment.
+ db-name: The name of the database containing this segment.
+ filter-field-name: The name of the field to use in the filter clause.
+
+ Returns:
+ A string containing the YAML representation of the segment."
+ [segment-name table-name db-name & {:keys [schema description entity-id filter-field-name]
+ :or {description "Test segment"
+ filter-field-name "Some Field"}}]
+ (let [eid (or entity-id segment-name)]
+ (format "archived: false
+caveats: null
+created_at: '2024-08-28T09:46:18.671622Z'
+creator_id: [email protected]
+definition:
+ database: %s
+ query:
+ filter:
+ - <
+ - - field
+ - - %s
+ - %s
+ - %s
+ - %s
+ - null
+ - 18
+ source-table:
+ - %s
+ - %s
+ - %s
+ type: query
+description: %s
+entity_id: %s
+name: %s
+points_of_interest: null
+show_in_getting_started: false
+table_id:
+- %s
+- %s
+- %s
+serdes/meta:
+- id: %s
+ label: %s
+ model: Segment
+"
+ db-name
+ db-name
+ (or schema "null")
+ table-name
+ filter-field-name
+ db-name
+ (or schema "null")
+ table-name
+ description
+ eid
+ segment-name
+ db-name
+ (or schema "null")
+ table-name
+ eid
+ (str/replace (u/lower-case-en segment-name) #"\s+" "_"))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index e18c09b7d9b..0e4a110fcfe 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -2512,3 +2512,50 @@
(is (not (contains? table-names "Other Table"))))
(testing "database is not exported (no-data-model: true)"
(is (empty? (filter #(= "Database" (-> % :serdes/meta last :model)) extracted)))))))))
+
+(deftest collection-export-includes-fields-and-segments-test
+ (testing "Exporting a collection with published tables includes their Fields and Segments"
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
+ :model/Collection {coll-id :id} {:name "Export Collection"}
+ :model/Table {table-id :id} {:name "Published Table"
+ :db_id db-id
+ :is_published true
+ :collection_id coll-id}
+ :model/Field {field1-id :id} {:name "Field One"
+ :table_id table-id
+ :base_type :type/Integer}
+ :model/Field {field2-id :id} {:name "Field Two"
+ :table_id table-id
+ :base_type :type/Text}
+ :model/Segment {segment-id :id} {:name "Test Segment"
+ :table_id table-id
+ :definition {}}
+ ;; Create another table with fields that should NOT be exported
+ :model/Table {other-table-id :id} {:name "Other Table"
+ :db_id db-id
+ :is_published false}
+ :model/Field _ {:name "Other Field"
+ :table_id other-table-id
+ :base_type :type/Integer}]
+ (let [opts {:targets [["Collection" coll-id]]
+ :no-data-model true
+ :no-settings true}
+ extracted (into [] (extract/extract opts))
+ field-ids (->> extracted
+ (filter #(= "Field" (-> % :serdes/meta last :model)))
+ (map #(t2/select-one-pk :model/Field
+ :table_id (t2/select-one-pk :model/Table :name (-> % :serdes/meta butlast last :id))
+ :name (-> % :serdes/meta last :id)))
+ set)
+ segment-ids (->> extracted
+ (filter #(= "Segment" (-> % :serdes/meta last :model)))
+ (map #(:id (t2/select-one :model/Segment :entity_id (:entity_id %))))
+ set)]
+ (testing "fields from published table are exported"
+ (is (contains? field-ids field1-id))
+ (is (contains? field-ids field2-id)))
+ (testing "segments from published table are exported"
+ (is (contains? segment-ids segment-id)))
+ (testing "fields from non-published table are NOT exported"
+ (is (= 2 (count field-ids)))))))))
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.tsx
index 24635bc1b2d..7af4ffaa079 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.tsx
@@ -18,8 +18,10 @@ import { PublishTableModal } from "./PublishTableModal";
export const CreateMenu = ({
metricCollectionId,
+ canWriteToMetricCollection,
}: {
metricCollectionId?: CollectionId;
+ canWriteToMetricCollection?: boolean;
}) => {
const dispatch = useDispatch();
const [modal, setModal] = useState<"snippet-folder" | "publish-table">();
@@ -29,7 +31,8 @@ export const CreateMenu = ({
const hasNativeWrite = useSelector(canUserCreateNativeQueries);
const hasDataAccess = useSelector(canUserCreateQueries);
- const canCreateMetric = hasDataAccess && metricCollectionId;
+ const canCreateMetric =
+ hasDataAccess && metricCollectionId && canWriteToMetricCollection;
const menuItems = [
isAdmin && (
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.unit.spec.tsx
index 29e3eb7fa92..d3deaeb954a 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/CreateMenu.unit.spec.tsx
@@ -12,7 +12,15 @@ import { createMockState } from "metabase-types/store/mocks/state";
import { CreateMenu } from "./CreateMenu";
-const setup = ({ user }: { user?: Partial<User> } = {}) => {
+interface SetupOptions {
+ user?: Partial<User>;
+ canWriteToMetricCollection?: boolean;
+}
+
+const setup = ({
+ user,
+ canWriteToMetricCollection = true,
+}: SetupOptions = {}) => {
const state = createMockState({
settings: mockSettings({
"token-features": createMockTokenFeatures({
@@ -22,9 +30,15 @@ const setup = ({ user }: { user?: Partial<User> } = {}) => {
currentUser: createMockUser(user),
});
setupEnterprisePlugins();
- return renderWithProviders(<CreateMenu metricCollectionId={1} />, {
- storeInitialState: state,
- });
+ return renderWithProviders(
+ <CreateMenu
+ metricCollectionId={1}
+ canWriteToMetricCollection={canWriteToMetricCollection}
+ />,
+ {
+ storeInitialState: state,
+ },
+ );
};
describe("CreateMenu", () => {
@@ -45,6 +59,25 @@ describe("CreateMenu", () => {
).toEqual(["Metric"]);
});
+ it("does not render Metric option when canWriteToMetricCollection is false", async () => {
+ setup({
+ user: {
+ is_superuser: true,
+ permissions: {
+ can_create_queries: true,
+ can_create_native_queries: true,
+ },
+ },
+ canWriteToMetricCollection: false,
+ });
+
+ await userEvent.click(screen.getByRole("button", { name: /New/ }));
+
+ expect(
+ screen.getAllByRole("menuitem").map((item) => item.textContent),
+ ).toEqual(["Publish a table", "New snippet", "New snippet folder"]);
+ });
+
it("renders all options when user has full permissions", async () => {
setup({
user: {
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/LibrarySectionLayout.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/LibrarySectionLayout.tsx
index d5912c5e54c..cf61026aa59 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/LibrarySectionLayout.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/LibrarySectionLayout.tsx
@@ -6,6 +6,7 @@ import _ from "underscore";
import { useListCollectionsTreeQuery } from "metabase/api";
import { isLibraryCollection } from "metabase/collections/utils";
import DateTime from "metabase/common/components/DateTime";
+import { useSetting } from "metabase/common/hooks";
import { usePageTitle } from "metabase/hooks/use-page-title";
import { useDispatch } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
@@ -40,7 +41,7 @@ import {
useErrorHandling,
} from "./hooks";
import { type TreeItem, isCollection } from "./types";
-import { getWritableCollection } from "./utils";
+import { getAccessibleCollection } from "./utils";
export function LibrarySectionLayout() {
usePageTitle(t`Library`);
@@ -72,15 +73,36 @@ export function LibrarySectionLayout() {
"include-library": true,
});
- const libraryCollection = collections.find(isLibraryCollection);
+ const isInstanceRemoteSyncEnabled = Boolean(
+ useSetting("remote-sync-enabled"),
+ );
+
+ const libraryCollection = useMemo(
+ () => collections.find(isLibraryCollection),
+ [collections],
+ );
- const tableCollection =
- libraryCollection &&
- getWritableCollection(libraryCollection, "library-data");
+ const tableCollection = useMemo(
+ () =>
+ libraryCollection &&
+ getAccessibleCollection(
+ libraryCollection,
+ "library-data",
+ isInstanceRemoteSyncEnabled,
+ ),
+ [libraryCollection, isInstanceRemoteSyncEnabled],
+ );
- const metricCollection =
- libraryCollection &&
- getWritableCollection(libraryCollection, "library-metrics");
+ const metricCollection = useMemo(
+ () =>
+ libraryCollection &&
+ getAccessibleCollection(
+ libraryCollection,
+ "library-metrics",
+ isInstanceRemoteSyncEnabled,
+ ),
+ [libraryCollection, isInstanceRemoteSyncEnabled],
+ );
const handleItemSelect = useCallback(
(item: TreeItem) => {
@@ -119,8 +141,10 @@ export function LibrarySectionLayout() {
const isLoading = loadingTables || loadingMetrics || loadingSnippets;
useErrorHandling(tablesError || metricsError || snippetsError);
- const libraryHasContent = combinedTree.some(
- (node) => node.children && node.children.length > 0,
+ const libraryHasContent = useMemo(
+ () =>
+ combinedTree.some((node) => node.children && node.children.length > 0),
+ [combinedTree],
);
const libraryColumnDef = useMemo<TreeTableColumnDef<TreeItem>[]>(
@@ -238,7 +262,10 @@ export function LibrarySectionLayout() {
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
/>
- <CreateMenu metricCollectionId={metricCollection?.id} />
+ <CreateMenu
+ metricCollectionId={metricCollection?.id}
+ canWriteToMetricCollection={metricCollection?.can_write}
+ />
</Flex>
<Card withBorder p={0}>
{isLoading ? (
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/utils.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/utils.ts
index 0aeb9d48898..5c8ba4b0089 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/utils.ts
@@ -83,3 +83,25 @@ export function getWritableCollection(
);
return collection?.can_write ? collection : undefined;
}
+
+export function getAccessibleCollection(
+ rootCollection: Collection,
+ type: CollectionType,
+ isInstanceRemoteSyncEnabled: boolean,
+) {
+ const collection = rootCollection.children?.find(
+ (collection) => collection.type === type,
+ );
+ if (!collection) {
+ return undefined;
+ }
+
+ // If instance remote-sync is enabled AND this collection has remote-sync enabled,
+ // show it even if read-only. Otherwise, only show writable collections.
+ const isCollectionRemoteSynced =
+ isInstanceRemoteSyncEnabled && collection.is_remote_synced;
+ if (isCollectionRemoteSynced) {
+ return collection;
+ }
+ return collection.can_write ? collection : undefined;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/utils.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/utils.unit.spec.ts
new file mode 100644
index 00000000000..d334b52cd64
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/app/pages/LibrarySectionLayout/utils.unit.spec.ts
@@ -0,0 +1,203 @@
+import { createMockCollection } from "metabase-types/api/mocks";
+
+import { getAccessibleCollection, getWritableCollection } from "./utils";
+
+describe("getWritableCollection", () => {
+ it("returns collection when can_write is true", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-data",
+ can_write: true,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getWritableCollection(rootCollection, "library-data");
+
+ expect(result).toEqual(childCollection);
+ });
+
+ it("returns undefined when can_write is false", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-data",
+ can_write: false,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getWritableCollection(rootCollection, "library-data");
+
+ expect(result).toBeUndefined();
+ });
+
+ it("returns undefined when collection type does not exist", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-metrics",
+ can_write: true,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getWritableCollection(rootCollection, "library-data");
+
+ expect(result).toBeUndefined();
+ });
+});
+
+describe("getAccessibleCollection", () => {
+ describe("when instance remote-sync is disabled", () => {
+ const isInstanceRemoteSyncEnabled = false;
+
+ it("returns collection when can_write is true", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-data",
+ can_write: true,
+ is_remote_synced: false,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getAccessibleCollection(
+ rootCollection,
+ "library-data",
+ isInstanceRemoteSyncEnabled,
+ );
+
+ expect(result).toEqual(childCollection);
+ });
+
+ it("returns undefined when can_write is false even if collection is remote synced", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-data",
+ can_write: false,
+ is_remote_synced: true,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getAccessibleCollection(
+ rootCollection,
+ "library-data",
+ isInstanceRemoteSyncEnabled,
+ );
+
+ expect(result).toBeUndefined();
+ });
+ });
+
+ describe("when instance remote-sync is enabled", () => {
+ const isInstanceRemoteSyncEnabled = true;
+
+ it("returns collection when can_write is true (regardless of is_remote_synced)", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-data",
+ can_write: true,
+ is_remote_synced: false,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getAccessibleCollection(
+ rootCollection,
+ "library-data",
+ isInstanceRemoteSyncEnabled,
+ );
+
+ expect(result).toEqual(childCollection);
+ });
+
+ it("returns collection when can_write is false but collection is remote synced", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-data",
+ can_write: false,
+ is_remote_synced: true,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getAccessibleCollection(
+ rootCollection,
+ "library-data",
+ isInstanceRemoteSyncEnabled,
+ );
+
+ expect(result).toEqual(childCollection);
+ });
+
+ it("returns undefined when can_write is false and collection is not remote synced", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-data",
+ can_write: false,
+ is_remote_synced: false,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getAccessibleCollection(
+ rootCollection,
+ "library-data",
+ isInstanceRemoteSyncEnabled,
+ );
+
+ expect(result).toBeUndefined();
+ });
+ });
+
+ it("returns undefined when collection type does not exist", () => {
+ const childCollection = createMockCollection({
+ id: 2,
+ type: "library-metrics",
+ can_write: true,
+ });
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: [childCollection],
+ });
+
+ const result = getAccessibleCollection(
+ rootCollection,
+ "library-data",
+ true,
+ );
+
+ expect(result).toBeUndefined();
+ });
+
+ it("returns undefined when root collection has no children", () => {
+ const rootCollection = createMockCollection({
+ id: 1,
+ children: undefined,
+ });
+
+ const result = getAccessibleCollection(
+ rootCollection,
+ "library-data",
+ true,
+ );
+
+ expect(result).toBeUndefined();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/common/hooks/use-load-table-with-metadata/use-load-table-with-metadata.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/common/hooks/use-load-table-with-metadata/use-load-table-with-metadata.ts
index c309e5377b5..802c8801342 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/common/hooks/use-load-table-with-metadata/use-load-table-with-metadata.ts
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/common/hooks/use-load-table-with-metadata/use-load-table-with-metadata.ts
@@ -23,18 +23,12 @@ export function useLoadTableWithMetadata(
isLoading,
error,
} = useGetTableQueryMetadataQuery(
- tableId != null
- ? { id: tableId, include_editable_data_model: true }
- : skipToken,
+ tableId != null ? { id: tableId } : skipToken,
);
useEffect(() => {
if (includeForeignTables && table) {
- dispatch(
- fetchForeignTablesMetadata(table, {
- include_editable_data_model: true,
- }),
- );
+ dispatch(fetchForeignTablesMetadata(table));
}
}, [dispatch, table, includeForeignTables]);
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/components/SearchNew.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/components/SearchNew.tsx
index 5751caa3fa2..68ba7e8692c 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/components/SearchNew.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/components/SearchNew.tsx
@@ -114,12 +114,10 @@ export function SearchNew({
"unused-only": filters.unusedOnly === true ? true : undefined,
});
const { data: databases, isLoading: isLoadingDatabases } =
- useListDatabasesQuery({ include_editable_data_model: true });
+ useListDatabasesQuery();
const { isExpanded, toggle } = useExpandedState(
{},
- {
- defaultExpanded: true,
- },
+ { defaultExpanded: true },
);
const allowedDatabaseIds = useMemo(
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/hooks/useTableLoader.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/hooks/useTableLoader.ts
index dc9b0e8f10c..a28a9e316f4 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/hooks/useTableLoader.ts
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TablePicker/hooks/useTableLoader.ts
@@ -43,10 +43,7 @@ export function useTableLoader() {
const [isInitialLoading, setIsInitialLoading] = useState(true);
const getDatabases = useCallback(async () => {
- const response = await fetchDatabases(
- { include_editable_data_model: true },
- true,
- );
+ const response = await fetchDatabases({}, true);
if (databasesRef.current.isError) {
// Do not refetch when this call failed previously.
// This is to prevent infinite data-loading loop as RTK query does not cache error responses.
@@ -77,7 +74,6 @@ export function useTableLoader() {
id: databaseId,
schema: schemaName,
include_hidden: true,
- include_editable_data_model: true,
};
if (
@@ -115,7 +111,6 @@ export function useTableLoader() {
const newArgs = {
id: databaseId,
include_hidden: true,
- include_editable_data_model: true,
};
if (
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/pages/DataModel/DataModel.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/pages/DataModel/DataModel.tsx
index b70d7d59320..9c294db5d4a 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/pages/DataModel/DataModel.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/pages/DataModel/DataModel.tsx
@@ -83,7 +83,7 @@ function DataModelContent({ params }: Props) {
data: databasesData,
error: databasesError,
isLoading: isLoadingDatabases,
- } = useListDatabasesQuery({ include_editable_data_model: true });
+ } = useListDatabasesQuery();
const databaseExists = databasesData?.data?.some(
(database) => database.id === databaseId,
);
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.tsx
index 62cd1a7b3ab..a7b58bc5094 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.tsx
@@ -40,6 +40,7 @@ export function NewSegmentPage({
const [name, setName] = useState("");
const [description, setDescription] = useState("");
const [definition, setDefinition] = useState<DatasetQuery | null>(null);
+ const [savedSegment, setSavedSegment] = useState<Segment | null>(null);
const isInitialized = useRef(false);
@@ -53,7 +54,8 @@ export function NewSegmentPage({
const { query, filters } = useSegmentQuery(definition, metadata);
const isDirty =
- name.trim().length > 0 || description.length > 0 || filters.length > 0;
+ !savedSegment &&
+ (name.trim().length > 0 || description.length > 0 || filters.length > 0);
const isValid = name.trim().length > 0 && filters.length > 0;
const previewUrl =
filters.length > 0 ? getDatasetQueryPreviewUrl(definition) : undefined;
@@ -78,8 +80,8 @@ export function NewSegmentPage({
if (error) {
sendErrorToast(t`Failed to create segment`);
} else if (segment) {
+ setSavedSegment(segment);
sendSuccessToast(t`Segment created`);
- dispatch(push(getSuccessUrl(segment)));
}
}, [
table,
@@ -88,12 +90,16 @@ export function NewSegmentPage({
description,
isValid,
createSegment,
- dispatch,
- getSuccessUrl,
sendSuccessToast,
sendErrorToast,
]);
+ useEffect(() => {
+ if (savedSegment) {
+ dispatch(push(getSuccessUrl(savedSegment)));
+ }
+ }, [savedSegment, dispatch, getSuccessUrl]);
+
return (
<PageContainer data-testid="new-segment-page" gap="xl">
<NewSegmentHeader
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.unit.spec.tsx
index c69539ab724..71f7642895d 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/segments/pages/NewSegmentPage/NewSegmentPage.unit.spec.tsx
@@ -252,4 +252,35 @@ describe("NewSegmentPage", () => {
screen.queryByText("Add filters to narrow your answer"),
).not.toBeInTheDocument();
});
+
+ it("does not show discard changes modal after successful save", async () => {
+ const createdSegment = createMockSegment({
+ id: 123,
+ name: "High Value Orders",
+ table_id: TEST_TABLE.id,
+ });
+
+ fetchMock.post("path:/api/segment", createdSegment);
+
+ const { history, successUrl } = setup();
+
+ await userEvent.type(
+ screen.getByPlaceholderText("New segment"),
+ "High Value Orders",
+ );
+
+ await addFilter();
+
+ const saveButton = screen.getByRole("button", { name: "Save" });
+ await userEvent.click(saveButton);
+
+ await waitFor(() => {
+ expect(history?.getCurrentLocation().pathname).toBe(
+ `${successUrl}/${createdSegment.id}`,
+ );
+ });
+
+ // The "Discard your changes?" modal should NOT appear after a successful save
+ expect(screen.queryByText("Discard your changes?")).not.toBeInTheDocument();
+ });
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
index 99db5fc6418..488de7bbcb6 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
@@ -19,12 +19,21 @@ import {
getCollectionPathSegments,
getSyncStatusColor,
getSyncStatusIcon,
+ isTableChildModel,
} from "metabase-enterprise/remote_sync/utils";
import type { RemoteSyncEntity } from "metabase-types/api";
import { CollectionPath } from "./CollectionPath";
import { EntityLink } from "./EntityLink";
+/** A table group with optional entity (when table itself is dirty) and its nested children */
+type TableGroup = {
+ tableId: number;
+ tableName: string;
+ table?: RemoteSyncEntity;
+ children: RemoteSyncEntity[];
+};
+
const SYNC_STATUS_ORDER: RemoteSyncEntity["sync_status"][] = [
"create",
"update",
@@ -63,17 +72,66 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {
const groupedData = useMemo(() => {
const byCollection = _.groupBy(entities, (e) => e.collection_id || 0);
+ const sortByStatus = (items: RemoteSyncEntity[]) =>
+ items.sort((a, b) => {
+ const statusOrderA = SYNC_STATUS_ORDER.indexOf(a.sync_status);
+ const statusOrderB = SYNC_STATUS_ORDER.indexOf(b.sync_status);
+ if (statusOrderA !== statusOrderB) {
+ return statusOrderA - statusOrderB;
+ }
+ return a.name.localeCompare(b.name);
+ });
+
return Object.entries(byCollection)
.map(([collectionId, items]) => {
const collectionEntity = items.find(
(item) =>
item.model === "collection" && item.id === Number(collectionId),
);
- const nonCollectionItems = items.filter(
+
+ // Separate tables, table children, and other items
+ const tables = items.filter((item) => item.model === "table");
+ const tableChildren = items.filter((item) =>
+ isTableChildModel(item.model),
+ );
+ const otherItems = items.filter(
(item) =>
+ item.model !== "table" &&
+ !isTableChildModel(item.model) &&
!(item.model === "collection" && item.id === Number(collectionId)),
);
+ // Group table children by their parent table_id
+ const childrenByTableId = _.groupBy(
+ tableChildren,
+ (e) => e.table_id || 0,
+ );
+
+ // Create table groups for dirty tables
+ const tableIds = new Set(tables.map((t) => t.id));
+ const dirtyTableGroups: TableGroup[] = tables.map((table) => ({
+ tableId: table.id,
+ tableName: table.name,
+ table,
+ children: sortByStatus(childrenByTableId[table.id] || []),
+ }));
+
+ // Create table groups for orphan children (children whose parent table is not dirty)
+ const orphanTableGroups: TableGroup[] = Object.entries(
+ childrenByTableId,
+ )
+ .filter(([tableId]) => !tableIds.has(Number(tableId)))
+ .map(([tableId, children]) => ({
+ tableId: Number(tableId),
+ tableName: children[0]?.table_name ?? t`Unknown table`,
+ children: sortByStatus(children),
+ }));
+
+ // Combine and sort all table groups
+ const tableGroups = [...dirtyTableGroups, ...orphanTableGroups].sort(
+ (a, b) => a.tableName.localeCompare(b.tableName),
+ );
+
return {
pathSegments: getCollectionPathSegments(
Number(collectionId) || undefined,
@@ -81,14 +139,8 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {
),
collectionId: Number(collectionId) || undefined,
collectionEntity,
- items: nonCollectionItems.sort((a, b) => {
- const statusOrderA = SYNC_STATUS_ORDER.indexOf(a.sync_status);
- const statusOrderB = SYNC_STATUS_ORDER.indexOf(b.sync_status);
- if (statusOrderA !== statusOrderB) {
- return statusOrderA - statusOrderB;
- }
- return a.name.localeCompare(b.name);
- }),
+ tableGroups,
+ items: sortByStatus(otherItems),
};
})
.sort((a, b) =>
@@ -118,50 +170,97 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {
}}
>
<Stack gap={0}>
- {groupedData.map((group, groupIndex) => (
- <Fragment key={group.collectionId}>
- {groupIndex > 0 && <Divider />}
- <Box p="md">
- <Group
- p="sm"
- gap="sm"
- mb={group.items.length > 0 ? "0.75rem" : 0}
- bg="background-secondary"
- bdrs="md"
- >
- <Icon name="synced_collection" size={16} c="text-secondary" />
- <CollectionPath segments={group.pathSegments} />
- {group.collectionEntity && (
+ {groupedData.map((group, groupIndex) => {
+ const hasItems =
+ group.items.length > 0 || group.tableGroups.length > 0;
+
+ return (
+ <Fragment key={group.collectionId}>
+ {groupIndex > 0 && <Divider />}
+ <Box p="md">
+ <Group
+ p="sm"
+ gap="sm"
+ mb={hasItems ? "0.75rem" : 0}
+ bg="background-secondary"
+ bdrs="md"
+ >
<Icon
- name={getSyncStatusIcon(
- group.collectionEntity.sync_status,
- )}
+ name="synced_collection"
size={16}
- c={getSyncStatusColor(group.collectionEntity.sync_status)}
- ml="auto"
+ c="text-secondary"
/>
- )}
- </Group>
- {group.items.length > 0 && (
- <Stack
- gap="0.75rem"
- ml="md"
- pl="xs"
- style={{
- borderLeft: "2px solid var(--mb-color-border)",
- }}
- >
- {group.items.map((entity) => (
- <EntityLink
- key={`${entity.model}-${entity.id}`}
- entity={entity}
+ <CollectionPath segments={group.pathSegments} />
+ {group.collectionEntity && (
+ <Icon
+ name={getSyncStatusIcon(
+ group.collectionEntity.sync_status,
+ )}
+ size={16}
+ c={getSyncStatusColor(
+ group.collectionEntity.sync_status,
+ )}
+ ml="auto"
/>
- ))}
- </Stack>
- )}
- </Box>
- </Fragment>
- ))}
+ )}
+ </Group>
+ {hasItems && (
+ <Stack
+ gap="0.75rem"
+ ml="md"
+ pl="xs"
+ style={{
+ borderLeft: "2px solid var(--mb-color-border)",
+ }}
+ >
+ {/* Render table groups (both dirty tables and orphan children) */}
+ {group.tableGroups.map((tableGroup) => (
+ <Box key={`table-${tableGroup.tableId}`}>
+ {tableGroup.table ? (
+ <EntityLink entity={tableGroup.table} />
+ ) : (
+ <Group gap="sm" wrap="nowrap" px="sm">
+ <Icon name="table" size={16} c="text-secondary" />
+ <Text size="sm" c="text-secondary">
+ {tableGroup.tableName}
+ </Text>
+ </Group>
+ )}
+ {tableGroup.children.length > 0 && (
+ <Stack
+ gap="0.75rem"
+ ml="md"
+ pl="xs"
+ mt="0.75rem"
+ style={{
+ borderLeft:
+ "2px solid var(--mb-color-border-subtle)",
+ }}
+ >
+ {tableGroup.children.map((child) => (
+ <EntityLink
+ key={`${child.model}-${child.id}`}
+ entity={child}
+ />
+ ))}
+ </Stack>
+ )}
+ </Box>
+ ))}
+
+ {/* Render other non-table items */}
+ {group.items.map((entity) => (
+ <EntityLink
+ key={`${entity.model}-${entity.id}`}
+ entity={entity}
+ />
+ ))}
+ </Stack>
+ )}
+ </Box>
+ </Fragment>
+ );
+ })}
</Stack>
</Paper>
{hasRemovals && (
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx
index 0b78bb5aa96..9c56fea98ba 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/EntityLink.tsx
@@ -1,6 +1,6 @@
import { useMemo } from "react";
-import { getIcon } from "metabase/lib/icon";
+import { type IconData, type IconModel, getIcon } from "metabase/lib/icon";
import { modelToUrl } from "metabase/lib/urls";
import { Anchor, Group, Icon } from "metabase/ui";
import type { RemoteSyncEntity } from "metabase-types/api";
@@ -14,11 +14,17 @@ interface EntityLinkProps {
}
export const EntityLink = ({ entity }: EntityLinkProps) => {
- const entityIcon = getIcon({
- model: entity.model,
- id: entity.id,
- display: entity.display,
- });
+ const entityIcon = useMemo((): IconData => {
+ if (entity.model === "field") {
+ return { name: "field" };
+ }
+
+ return getIcon({
+ model: entity.model as IconModel,
+ id: entity.id,
+ display: entity.display,
+ });
+ }, [entity]);
const url = useMemo(() => modelToUrl(entity), [entity]);
if (url == null) {
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/GitSyncControls/BranchDropdown.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/GitSyncControls/BranchDropdown.unit.spec.tsx
index 5c60a8c2afd..a858b20eee8 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/GitSyncControls/BranchDropdown.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/GitSyncControls/BranchDropdown.unit.spec.tsx
@@ -1,6 +1,7 @@
import userEvent from "@testing-library/user-event";
import fetchMock from "fetch-mock";
+import { setupRemoteSyncBranchesEndpoint } from "__support__/server-mocks";
import { renderWithProviders, screen, waitFor } from "__support__/ui";
import { Combobox, useCombobox } from "metabase/ui";
@@ -33,9 +34,7 @@ function WrapperComponent({
}
const setupEndpoints = () => {
- fetchMock.get("path:/api/ee/remote-sync/branches", {
- items: ["main", "develop", "feature-1"],
- });
+ setupRemoteSyncBranchesEndpoint(["main", "develop", "feature-1"]);
fetchMock.post("path:/api/ee/remote-sync/create-branch", {});
};
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.ts
index 10597a470cb..fe72c300217 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.ts
@@ -10,7 +10,8 @@ import { useRemoteSyncDirtyState } from "./use-remote-sync-dirty-state";
export function useHasLibraryDirtyChanges(): boolean {
const { isVisible: isGitSyncVisible } = useGitSyncVisible();
- const { hasDirtyInCollectionTree, isDirty } = useRemoteSyncDirtyState();
+ const { hasDirtyInCollectionTree, isDirty, hasRemovedItems } =
+ useRemoteSyncDirtyState();
const { data: collections = [] } = useListCollectionsTreeQuery(
{
@@ -22,6 +23,11 @@ export function useHasLibraryDirtyChanges(): boolean {
);
return useMemo(() => {
+ // Always show dirty if there are removed items
+ if (hasRemovedItems) {
+ return true;
+ }
+
if (!isDirty) {
return false;
}
@@ -43,5 +49,5 @@ export function useHasLibraryDirtyChanges(): boolean {
);
return hasDirtyInCollectionTree(numericIds);
- }, [collections, isDirty, hasDirtyInCollectionTree]);
+ }, [collections, isDirty, hasRemovedItems, hasDirtyInCollectionTree]);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.ts
index 3432be6bf1c..45621c08832 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.ts
@@ -1,39 +1,20 @@
-import fetchMock from "fetch-mock";
-
+import { setupEnterprisePlugins } from "__support__/enterprise";
+import {
+ setupCollectionsEndpoints,
+ setupPropertiesEndpoints,
+ setupRemoteSyncDirtyEndpoint,
+ setupSettingsEndpoints,
+} from "__support__/server-mocks";
+import { mockSettings } from "__support__/settings";
import { renderHookWithProviders, waitFor } from "__support__/ui";
-import type { Collection } from "metabase-types/api";
-import { createMockCollection } from "metabase-types/api/mocks";
-
-// Mock isLibraryCollection to return true for collections with type "library"
-jest.mock("metabase/collections/utils", () => ({
- ...jest.requireActual("metabase/collections/utils"),
- isLibraryCollection: (collection: { type: string | null }) =>
- collection?.type === "library",
-}));
-
-// Mock useRemoteSyncDirtyState
-const mockHasDirtyInCollectionTree = jest.fn(
- (_collectionIds: Set<number>) => false,
-);
-const mockIsDirty = jest.fn(() => false);
-jest.mock("./use-remote-sync-dirty-state", () => ({
- useRemoteSyncDirtyState: () => ({
- hasDirtyInCollectionTree: mockHasDirtyInCollectionTree,
- isDirty: mockIsDirty(),
- }),
-}));
-
-// Mock useGitSyncVisible
-const mockUseGitSyncVisible = jest.fn<
- { isVisible: boolean; currentBranch: string | null },
- []
->(() => ({
- isVisible: true,
- currentBranch: "main",
-}));
-jest.mock("./use-git-sync-visible", () => ({
- useGitSyncVisible: () => mockUseGitSyncVisible(),
-}));
+import type { Collection, RemoteSyncEntity } from "metabase-types/api";
+import {
+ createMockCollection,
+ createMockSettings,
+ createMockTokenFeatures,
+ createMockUser,
+} from "metabase-types/api/mocks";
+import { createMockState } from "metabase-types/store/mocks";
import { useHasLibraryDirtyChanges } from "./use-has-library-dirty-changes";
@@ -57,31 +38,82 @@ const createRegularCollection = (
...overrides,
});
-const setupCollectionsEndpoint = (collections: Collection[]) => {
- fetchMock.get("path:/api/collection/tree", collections);
-};
+const createMockDirtyEntity = (
+ overrides: Partial<RemoteSyncEntity> = {},
+): RemoteSyncEntity => ({
+ id: 1,
+ name: "Test Entity",
+ model: "card",
+ sync_status: "update",
+ collection_id: 1,
+ ...overrides,
+});
-const setup = () => {
- return renderHookWithProviders(() => useHasLibraryDirtyChanges(), {});
+interface SetupOptions {
+ isGitSyncVisible?: boolean;
+ collections?: Collection[];
+ dirty?: RemoteSyncEntity[];
+}
+
+/**
+ * Compute changedCollections map from dirty entities.
+ * This mirrors what the backend does - marking collections that contain dirty entities.
+ */
+const computeChangedCollections = (
+ dirty: RemoteSyncEntity[],
+): Record<number, boolean> => {
+ const changedCollections: Record<number, boolean> = {};
+ for (const entity of dirty) {
+ if (entity.collection_id != null) {
+ changedCollections[entity.collection_id] = true;
+ }
+ }
+ return changedCollections;
};
-describe("useHasLibraryDirtyChanges", () => {
- beforeEach(() => {
- fetchMock.removeRoutes();
- fetchMock.clearHistory();
- mockUseGitSyncVisible.mockReturnValue({
- isVisible: true,
- currentBranch: "main",
- });
- mockIsDirty.mockReturnValue(false);
- mockHasDirtyInCollectionTree.mockReturnValue(false);
+const setup = ({
+ isGitSyncVisible = true,
+ collections = [],
+ dirty = [],
+}: SetupOptions = {}) => {
+ setupEnterprisePlugins();
+
+ const tokenFeatures = createMockTokenFeatures({ data_studio: true });
+ const settings = createMockSettings({
+ "remote-sync-enabled": isGitSyncVisible,
+ "remote-sync-branch": isGitSyncVisible ? "main" : null,
+ "remote-sync-type": "read-write",
+ "token-features": tokenFeatures,
});
- it("returns false when no dirty changes exist", async () => {
- setupCollectionsEndpoint([createLibraryCollection()]);
- mockIsDirty.mockReturnValue(false);
+ const changedCollections = computeChangedCollections(dirty);
+
+ setupPropertiesEndpoints(settings);
+ setupSettingsEndpoints([]);
+ setupRemoteSyncDirtyEndpoint({ dirty, changedCollections });
+ setupCollectionsEndpoints({ collections });
+
+ const storeInitialState = createMockState({
+ currentUser: createMockUser({ is_superuser: true }),
+ settings: mockSettings({
+ "remote-sync-enabled": isGitSyncVisible,
+ "remote-sync-branch": isGitSyncVisible ? "main" : null,
+ "remote-sync-type": "read-write",
+ "token-features": tokenFeatures,
+ }),
+ });
+
+ return renderHookWithProviders(() => useHasLibraryDirtyChanges(), {
+ storeInitialState,
+ });
+};
- const { result } = setup();
+describe("useHasLibraryDirtyChanges", () => {
+ it("returns false when no dirty changes exist", async () => {
+ const { result } = setup({
+ collections: [createLibraryCollection()],
+ dirty: [],
+ });
await waitFor(() => {
expect(result.current).toBe(false);
@@ -89,14 +121,13 @@ describe("useHasLibraryDirtyChanges", () => {
});
it("returns false when dirty changes exist but not in Library collections", async () => {
- setupCollectionsEndpoint([
- createLibraryCollection({ id: 1 }),
- createRegularCollection({ id: 2 }),
- ]);
- mockIsDirty.mockReturnValue(true);
- mockHasDirtyInCollectionTree.mockReturnValue(false);
-
- const { result } = setup();
+ const { result } = setup({
+ collections: [
+ createLibraryCollection({ id: 1 }),
+ createRegularCollection({ id: 2 }),
+ ],
+ dirty: [createMockDirtyEntity({ collection_id: 2 })],
+ });
await waitFor(() => {
expect(result.current).toBe(false);
@@ -104,62 +135,56 @@ describe("useHasLibraryDirtyChanges", () => {
});
it("returns true when Library collection has dirty items", async () => {
- setupCollectionsEndpoint([createLibraryCollection({ id: 1 })]);
- mockIsDirty.mockReturnValue(true);
- // When hasDirtyInCollectionTree is called with the library IDs, return true
- mockHasDirtyInCollectionTree.mockReturnValue(true);
-
- const { result } = setup();
+ const { result } = setup({
+ collections: [createLibraryCollection({ id: 1 })],
+ dirty: [createMockDirtyEntity({ collection_id: 1 })],
+ });
await waitFor(() => {
expect(result.current).toBe(true);
});
});
- it("calls hasDirtyInCollectionTree when Library exists and isDirty", async () => {
- setupCollectionsEndpoint([createLibraryCollection({ id: 42 })]);
- mockIsDirty.mockReturnValue(true);
- mockHasDirtyInCollectionTree.mockReturnValue(false);
-
- const { result } = setup();
+ it("returns false when no Library collection exists", async () => {
+ const { result } = setup({
+ collections: [createRegularCollection()],
+ dirty: [createMockDirtyEntity({ collection_id: 2 })],
+ });
await waitFor(() => {
expect(result.current).toBe(false);
});
-
- // Verify hasDirtyInCollectionTree was called with a Set
- expect(mockHasDirtyInCollectionTree).toHaveBeenCalled();
- const calledWith = mockHasDirtyInCollectionTree.mock.calls[0]?.[0];
- expect(calledWith).toBeInstanceOf(Set);
- // The Set should contain numeric IDs (exact IDs depend on buildCollectionTree transformation)
- expect(calledWith?.size).toBeGreaterThan(0);
});
- it("returns false when no Library collection exists", async () => {
- setupCollectionsEndpoint([createRegularCollection()]);
- mockIsDirty.mockReturnValue(true);
- mockHasDirtyInCollectionTree.mockReturnValue(true);
+ it("returns true when there are removed items even if no other dirty changes", async () => {
+ const { result } = setup({
+ collections: [createLibraryCollection()],
+ dirty: [createMockDirtyEntity({ sync_status: "removed" })],
+ });
+
+ await waitFor(() => {
+ expect(result.current).toBe(true);
+ });
+ });
- const { result } = setup();
+ it("returns true when there are removed items even without Library collection", async () => {
+ const { result } = setup({
+ collections: [createRegularCollection()],
+ dirty: [createMockDirtyEntity({ sync_status: "removed" })],
+ });
await waitFor(() => {
- expect(result.current).toBe(false);
+ expect(result.current).toBe(true);
});
});
it("returns false when git sync is not visible", async () => {
- setupCollectionsEndpoint([createLibraryCollection()]);
- mockUseGitSyncVisible.mockReturnValue({
- isVisible: false,
- currentBranch: null,
+ const { result } = setup({
+ isGitSyncVisible: false,
+ collections: [createLibraryCollection()],
+ dirty: [createMockDirtyEntity({ collection_id: 1 })],
});
- mockIsDirty.mockReturnValue(true);
- mockHasDirtyInCollectionTree.mockReturnValue(true);
-
- const { result } = setup();
- // When git sync is not visible, collections query is skipped,
- // so no library collection will be found
await waitFor(() => {
expect(result.current).toBe(false);
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.ts
index 2b317d5ea0d..8b0a520bdb0 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.ts
@@ -13,6 +13,8 @@ export interface RemoteSyncDirtyState {
changedCollections: Record<number, boolean>;
/** Whether any dirty changes exist globally */
isDirty: boolean;
+ /** Whether any entities have "removed" status */
+ hasRemovedItems: boolean;
/** Whether data is loading */
isLoading: boolean;
/** Check if a specific collection has dirty items */
@@ -45,6 +47,10 @@ export function useRemoteSyncDirtyState(): RemoteSyncDirtyState {
[dirtyData?.changedCollections],
);
const isDirty = dirty.length > 0;
+ const hasRemovedItems = useMemo(
+ () => dirty.some((entity) => entity.sync_status === "removed"),
+ [dirty],
+ );
const isCollectionDirty = useCallback(
(collectionId: number | string | undefined) => {
@@ -93,6 +99,7 @@ export function useRemoteSyncDirtyState(): RemoteSyncDirtyState {
dirty,
changedCollections,
isDirty,
+ hasRemovedItems,
isLoading,
isCollectionDirty,
hasAnyCollectionDirty,
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.unit.spec.ts
index 6eea6d98a06..134811bdcb1 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-remote-sync-dirty-state.unit.spec.ts
@@ -1,67 +1,77 @@
-import fetchMock from "fetch-mock";
-
+import {
+ setupPropertiesEndpoints,
+ setupRemoteSyncDirtyEndpoint,
+ setupSettingsEndpoints,
+} from "__support__/server-mocks";
+import { mockSettings } from "__support__/settings";
import { renderHookWithProviders, waitFor } from "__support__/ui";
-
-// Mock the useGitSyncVisible hook before importing useRemoteSyncDirtyState
-const mockUseGitSyncVisible = jest.fn(() => ({
- isVisible: true,
- currentBranch: "main",
-}));
-jest.mock("./use-git-sync-visible", () => ({
- useGitSyncVisible: () => mockUseGitSyncVisible(),
-}));
+import type { RemoteSyncEntity } from "metabase-types/api";
+import { createMockSettings, createMockUser } from "metabase-types/api/mocks";
+import { createMockState } from "metabase-types/store/mocks";
import { useRemoteSyncDirtyState } from "./use-remote-sync-dirty-state";
-const createMockDirtyEntity = (overrides: Record<string, unknown> = {}) => ({
+const createMockDirtyEntity = (
+ overrides: Partial<RemoteSyncEntity> = {},
+): RemoteSyncEntity => ({
id: 1,
name: "Test Entity",
- description: null,
- created_at: "2024-01-01T00:00:00Z",
- updated_at: null,
- model: "card" as const,
- sync_status: "update" as const,
+ model: "card",
+ sync_status: "update",
collection_id: 1,
...overrides,
});
-const setupEndpoints = ({
- dirty = [],
- changedCollections = {},
-}: {
- dirty?: Array<ReturnType<typeof createMockDirtyEntity>>;
- changedCollections?: Record<number, boolean>;
-} = {}) => {
- fetchMock.get("path:/api/ee/remote-sync/dirty", {
- dirty,
- changedCollections,
- });
+/**
+ * Compute changedCollections map from dirty entities.
+ * This mirrors what the backend does - marking collections that contain dirty entities.
+ */
+const computeChangedCollections = (
+ dirty: RemoteSyncEntity[],
+): Record<number, boolean> => {
+ const changedCollections: Record<number, boolean> = {};
+ for (const entity of dirty) {
+ if (entity.collection_id != null) {
+ changedCollections[entity.collection_id] = true;
+ }
+ }
+ return changedCollections;
};
const setup = ({
isGitSyncVisible = true,
dirty = [],
- changedCollections = {},
}: {
isGitSyncVisible?: boolean;
- dirty?: Array<ReturnType<typeof createMockDirtyEntity>>;
- changedCollections?: Record<number, boolean>;
+ dirty?: RemoteSyncEntity[];
} = {}) => {
- mockUseGitSyncVisible.mockReturnValue({
- isVisible: isGitSyncVisible,
- currentBranch: "main",
+ const settings = createMockSettings({
+ "remote-sync-enabled": isGitSyncVisible,
+ "remote-sync-branch": isGitSyncVisible ? "main" : null,
+ "remote-sync-type": "read-write",
});
- setupEndpoints({ dirty, changedCollections });
- return renderHookWithProviders(() => useRemoteSyncDirtyState(), {});
-};
+ const changedCollections = computeChangedCollections(dirty);
-describe("useRemoteSyncDirtyState", () => {
- beforeEach(() => {
- fetchMock.removeRoutes();
- fetchMock.clearHistory();
+ setupPropertiesEndpoints(settings);
+ setupSettingsEndpoints([]);
+ setupRemoteSyncDirtyEndpoint({ dirty, changedCollections });
+
+ const storeInitialState = createMockState({
+ currentUser: createMockUser({ is_superuser: true }),
+ settings: mockSettings({
+ "remote-sync-enabled": isGitSyncVisible,
+ "remote-sync-branch": isGitSyncVisible ? "main" : null,
+ "remote-sync-type": "read-write",
+ }),
+ });
+
+ return renderHookWithProviders(() => useRemoteSyncDirtyState(), {
+ storeInitialState,
});
+};
+describe("useRemoteSyncDirtyState", () => {
describe("isDirty", () => {
it("returns false when no dirty entities exist", async () => {
const { result } = setup({ dirty: [] });
@@ -84,7 +94,6 @@ describe("useRemoteSyncDirtyState", () => {
describe("isCollectionDirty", () => {
it("returns false for collection without dirty items", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [createMockDirtyEntity({ collection_id: 1 })],
});
@@ -97,7 +106,6 @@ describe("useRemoteSyncDirtyState", () => {
});
it("returns true for collection with dirty items", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [createMockDirtyEntity({ collection_id: 1 })],
});
@@ -110,7 +118,6 @@ describe("useRemoteSyncDirtyState", () => {
});
it("returns false for non-numeric collection id", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [createMockDirtyEntity({ collection_id: 1 })],
});
@@ -126,12 +133,10 @@ describe("useRemoteSyncDirtyState", () => {
describe("hasAnyCollectionDirty", () => {
it("returns false when no collections in set are dirty", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [createMockDirtyEntity({ collection_id: 1 })],
});
- // Wait for data to be populated
await waitFor(() => {
expect(result.current.dirty.length).toBeGreaterThan(0);
});
@@ -140,7 +145,6 @@ describe("useRemoteSyncDirtyState", () => {
});
it("returns true when any collection in set is dirty", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [
createMockDirtyEntity({ id: 1, collection_id: 1 }),
@@ -148,7 +152,6 @@ describe("useRemoteSyncDirtyState", () => {
],
});
- // Wait for data to be populated
await waitFor(() => {
expect(result.current.dirty.length).toBeGreaterThan(0);
});
@@ -157,12 +160,10 @@ describe("useRemoteSyncDirtyState", () => {
});
it("works with array input", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [createMockDirtyEntity({ collection_id: 1 })],
});
- // Wait for data to be populated
await waitFor(() => {
expect(result.current.dirty.length).toBeGreaterThan(0);
});
@@ -173,12 +174,10 @@ describe("useRemoteSyncDirtyState", () => {
describe("hasDirtyInCollectionTree", () => {
it("returns false when no collections in tree are dirty", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [createMockDirtyEntity({ collection_id: 100 })],
});
- // Wait for data to be populated
await waitFor(() => {
expect(result.current.dirty.length).toBeGreaterThan(0);
});
@@ -189,12 +188,10 @@ describe("useRemoteSyncDirtyState", () => {
});
it("returns true when a collection in tree has dirty children", async () => {
- // changedCollections is built from dirty entities' collection_id
const { result } = setup({
dirty: [createMockDirtyEntity({ collection_id: 2 })],
});
- // Wait for data to be populated
await waitFor(() => {
expect(result.current.dirty.length).toBeGreaterThan(0);
});
@@ -210,12 +207,11 @@ describe("useRemoteSyncDirtyState", () => {
createMockDirtyEntity({
id: 2,
model: "collection",
- collection_id: null,
+ collection_id: undefined,
}),
],
});
- // Wait for dirty to be populated
await waitFor(() => {
expect(result.current.dirty.length).toBeGreaterThan(0);
});
@@ -231,12 +227,11 @@ describe("useRemoteSyncDirtyState", () => {
createMockDirtyEntity({
id: 100,
model: "collection",
- collection_id: null,
+ collection_id: undefined,
}),
],
});
- // Wait for dirty to be populated
await waitFor(() => {
expect(result.current.dirty.length).toBeGreaterThan(0);
});
@@ -247,6 +242,48 @@ describe("useRemoteSyncDirtyState", () => {
});
});
+ describe("hasRemovedItems", () => {
+ it("returns false when no entities have removed status", async () => {
+ const { result } = setup({
+ dirty: [
+ createMockDirtyEntity({ sync_status: "update" }),
+ createMockDirtyEntity({ id: 2, sync_status: "create" }),
+ ],
+ });
+
+ await waitFor(() => {
+ expect(result.current.dirty.length).toBeGreaterThan(0);
+ });
+
+ expect(result.current.hasRemovedItems).toBe(false);
+ });
+
+ it("returns true when any entity has removed status", async () => {
+ const { result } = setup({
+ dirty: [
+ createMockDirtyEntity({ sync_status: "update" }),
+ createMockDirtyEntity({ id: 2, sync_status: "removed" }),
+ ],
+ });
+
+ await waitFor(() => {
+ expect(result.current.dirty.length).toBeGreaterThan(0);
+ });
+
+ expect(result.current.hasRemovedItems).toBe(true);
+ });
+
+ it("returns false when dirty list is empty", async () => {
+ const { result } = setup({ dirty: [] });
+
+ await waitFor(() => {
+ expect(result.current.isLoading).toBe(false);
+ });
+
+ expect(result.current.hasRemovedItems).toBe(false);
+ });
+ });
+
describe("when git sync is not visible", () => {
it("returns empty dirty state", async () => {
const { result } = setup({
@@ -258,6 +295,7 @@ describe("useRemoteSyncDirtyState", () => {
expect(result.current.dirty).toEqual([]);
});
expect(result.current.isDirty).toBe(false);
+ expect(result.current.hasRemovedItems).toBe(false);
});
});
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts
index e2897d4d083..32504079337 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts
@@ -6,11 +6,15 @@ import { cardApi } from "metabase/api/card";
import { collectionApi } from "metabase/api/collection";
import { dashboardApi } from "metabase/api/dashboard";
import { documentApi } from "metabase/api/document";
+import { fieldApi } from "metabase/api/field";
+import { segmentApi } from "metabase/api/segment";
+import { tableApi as coreTableApi } from "metabase/api/table";
import { tag } from "metabase/api/tags";
import { timelineApi } from "metabase/api/timeline";
import { timelineEventApi } from "metabase/api/timeline-event";
import { getCollectionFromCollectionsTree } from "metabase/selectors/collection";
import { remoteSyncApi } from "metabase-enterprise/api/remote-sync";
+import { tableApi as enterpriseTableApi } from "metabase-enterprise/api/table";
import type {
Card,
CardId,
@@ -301,6 +305,52 @@ remoteSyncListenerMiddleware.startListening({
},
});
+// Enterprise table mutations (publish/unpublish/edit)
+remoteSyncListenerMiddleware.startListening({
+ matcher: isAnyOf(
+ enterpriseTableApi.endpoints.publishTables.matchFulfilled,
+ enterpriseTableApi.endpoints.unpublishTables.matchFulfilled,
+ enterpriseTableApi.endpoints.editTables.matchFulfilled,
+ ),
+ effect: async (_action, { dispatch }) => {
+ invalidateRemoteSyncTags(dispatch);
+ },
+});
+
+// Core table mutations (metadata edits)
+remoteSyncListenerMiddleware.startListening({
+ matcher: isAnyOf(
+ coreTableApi.endpoints.updateTable.matchFulfilled,
+ coreTableApi.endpoints.updateTableFieldsOrder.matchFulfilled,
+ ),
+ effect: async (_action, { dispatch }) => {
+ invalidateRemoteSyncTags(dispatch);
+ },
+});
+
+// Field mutations
+remoteSyncListenerMiddleware.startListening({
+ matcher: isAnyOf(
+ fieldApi.endpoints.updateField.matchFulfilled,
+ fieldApi.endpoints.createFieldDimension.matchFulfilled,
+ fieldApi.endpoints.deleteFieldDimension.matchFulfilled,
+ ),
+ effect: async (_action, { dispatch }) => {
+ invalidateRemoteSyncTags(dispatch);
+ },
+});
+
+// Segment mutations
+remoteSyncListenerMiddleware.startListening({
+ matcher: isAnyOf(
+ segmentApi.endpoints.createSegment.matchFulfilled,
+ segmentApi.endpoints.updateSegment.matchFulfilled,
+ ),
+ effect: async (_action, { dispatch }) => {
+ invalidateRemoteSyncTags(dispatch);
+ },
+});
+
remoteSyncListenerMiddleware.startListening({
matcher: remoteSyncApi.endpoints.exportChanges.matchPending,
effect: async (_action, { dispatch }) => {
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
index e5909aafdfd..4fa5eaeaac6 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
@@ -5,6 +5,7 @@ import type { IconName } from "metabase/ui";
import type {
Collection,
CollectionId,
+ RemoteSyncEntityModel,
RemoteSyncEntityStatus,
} from "metabase-types/api";
@@ -176,3 +177,18 @@ export const getCollectionPathSegments = (
segments.push({ id: collection.id, name: collection.name });
return segments;
};
+
+/**
+ * Models that are children of tables (get their collection from a parent table)
+ */
+const TABLE_CHILD_MODELS: Set<RemoteSyncEntityModel> = new Set([
+ "field",
+ "segment",
+]);
+
+/**
+ * Check if a model type is a child of a table (field or segment)
+ */
+export const isTableChildModel = (model: RemoteSyncEntityModel): boolean => {
+ return TABLE_CHILD_MODELS.has(model);
+};
diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.ts
index a915c246c80..85631645f2a 100644
--- a/frontend/src/metabase-types/api/remote-sync.ts
+++ b/frontend/src/metabase-types/api/remote-sync.ts
@@ -9,7 +9,10 @@ export type RemoteSyncEntityModel =
| "dashboard"
| "collection"
| "document"
- | "snippet";
+ | "snippet"
+ | "table"
+ | "field"
+ | "segment";
export type RemoteSyncEntityStatus =
| "create"
@@ -25,6 +28,11 @@ export type RemoteSyncEntity = {
collection_id?: number;
display?: CardDisplayType;
sync_status: RemoteSyncEntityStatus;
+ authority_level?: string | null;
+ /** Parent table ID for field and segment models */
+ table_id?: number;
+ /** Parent table name for field and segment models */
+ table_name?: string;
};
export type RemoteSyncChangesResponse = {
diff --git a/frontend/test/__support__/server-mocks/remote-sync.ts b/frontend/test/__support__/server-mocks/remote-sync.ts
index 75fa8b354ca..c22c15cf736 100644
--- a/frontend/test/__support__/server-mocks/remote-sync.ts
+++ b/frontend/test/__support__/server-mocks/remote-sync.ts
@@ -2,6 +2,57 @@ import fetchMock from "fetch-mock";
import type { RemoteSyncEntity } from "metabase-types/api";
+export interface RemoteSyncDirtyResponse {
+ dirty: RemoteSyncEntity[];
+ changedCollections: Record<number, boolean>;
+}
+
+/**
+ * Setup the remote-sync dirty endpoint
+ */
+export const setupRemoteSyncDirtyEndpoint = ({
+ dirty = [],
+ changedCollections = {},
+}: Partial<RemoteSyncDirtyResponse> = {}) => {
+ fetchMock.removeRoute("remote-sync-dirty");
+ fetchMock.get(
+ "path:/api/ee/remote-sync/dirty",
+ { dirty, changedCollections },
+ { name: "remote-sync-dirty" },
+ );
+};
+
+/**
+ * Setup the remote-sync branches endpoint
+ */
+export const setupRemoteSyncBranchesEndpoint = (
+ branches: string[] = ["main", "develop"],
+) => {
+ fetchMock.removeRoute("remote-sync-branches");
+ fetchMock.get(
+ "path:/api/ee/remote-sync/branches",
+ { items: branches },
+ { name: "remote-sync-branches" },
+ );
+};
+
+/**
+ * Setup the remote-sync current-task endpoint
+ */
+const setupRemoteSyncCurrentTaskEndpoint = (
+ status: "idle" | "running" | "completed" = "idle",
+) => {
+ fetchMock.removeRoute("remote-sync-current-task");
+ fetchMock.get(
+ "path:/api/ee/remote-sync/current-task",
+ { status },
+ { name: "remote-sync-current-task" },
+ );
+};
+
+/**
+ * Setup all remote-sync endpoints at once
+ */
export const setupRemoteSyncEndpoints = ({
branches = ["main", "develop"],
dirty = [],
@@ -11,14 +62,10 @@ export const setupRemoteSyncEndpoints = ({
dirty?: RemoteSyncEntity[];
changedCollections?: Record<number, boolean>;
} = {}) => {
- fetchMock.get("path:/api/ee/remote-sync/branches", { items: branches });
- fetchMock.get("path:/api/ee/remote-sync/dirty", {
- dirty,
- changedCollections,
- });
+ setupRemoteSyncBranchesEndpoint(branches);
+ setupRemoteSyncDirtyEndpoint({ dirty, changedCollections });
+ setupRemoteSyncCurrentTaskEndpoint("idle");
fetchMock.post("path:/api/ee/remote-sync/import", {});
fetchMock.post("path:/api/ee/remote-sync/create-branch", {});
- fetchMock.get("path:/api/ee/remote-sync/current-task", { status: "idle" });
-
fetchMock.put("path:/api/ee/remote-sync/settings", { success: true });
};
diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yaml
index 32e48684323..534aaf88920 100644
--- a/resources/migrations/058_update_migrations.yaml
+++ b/resources/migrations/058_update_migrations.yaml
@@ -1235,8 +1235,6 @@ databaseChangeLog:
path: instance_analytics_views/tables/v1/h2-tables.sql
relativeToChangelogFile: true
-# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
-
- changeSet:
id: v58.2025-12-18T16:14:03
author: winlost
@@ -1257,6 +1255,36 @@ databaseChangeLog:
tableName: pulse
columnName: disable_links
+ - changeSet:
+ id: v58.2025-12-30T22:48:02
+ author: edpaget
+ comment: Add remote_sync_object.model_table_id for parent table reference
+ changes:
+ - addColumn:
+ tableName: remote_sync_object
+ columns:
+ - column:
+ name: model_table_id
+ type: int
+ remarks: Parent table ID for Field/Segment/Table models
+ constraints:
+ nullable: true
+
+ - changeSet:
+ id: v58.2025-12-30T22:48:03
+ author: edpaget
+ comment: Add remote_sync_object.model_table_name for parent table name
+ changes:
+ - addColumn:
+ tableName: remote_sync_object
+ columns:
+ - column:
+ name: model_table_name
+ type: varchar(255)
+ remarks: Parent table name for Field/Segment/Table models
+ constraints:
+ nullable: true
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/events/schema.clj b/src/metabase/events/schema.clj
index 273545f6081..f84fa6605ca 100644
--- a/src/metabase/events/schema.clj
+++ b/src/metabase/events/schema.clj
@@ -185,6 +185,17 @@
[:user-id pos-int?]
[:object [:fn #(t2/instance-of? :model/Table %)]]])
+;; table write events
+
+(mr/def ::table
+ [:map {:closed true}
+ [:user-id [:maybe pos-int?]]
+ [:object [:fn #(t2/instance-of? :model/Table %)]]])
+
+(mr/def :event/table-create ::table)
+(mr/def :event/table-update ::table)
+(mr/def :event/table-delete ::table)
+
(mr/def ::permission-failure
[:map {:closed true}
[:user-id [:maybe pos-int?]]
@@ -223,3 +234,14 @@
(mr/def :event/snippet-create ::snippet)
(mr/def :event/snippet-update ::snippet)
(mr/def :event/snippet-delete ::snippet)
+
+;; field events
+
+(mr/def ::field
+ [:map {:closed true}
+ [:user-id [:maybe pos-int?]]
+ [:object [:fn #(t2/instance-of? :model/Field %)]]])
+
+(mr/def :event/field-create ::field)
+(mr/def :event/field-update ::field)
+(mr/def :event/field-delete ::field)
diff --git a/src/metabase/remote_sync/core.clj b/src/metabase/remote_sync/core.clj
index cc358abece6..6ed9e889e40 100644
--- a/src/metabase/remote_sync/core.clj
+++ b/src/metabase/remote_sync/core.clj
@@ -9,3 +9,12 @@
metabase-enterprise.remote-sync.core
[_collection]
true)
+
+(defenterprise table-editable?
+ "Returns if a table's metadata can be edited. Takes a table to check.
+
+ Returns false if the table is published, in a remote-synced collection, and remote-sync-type is :read-only.
+ Always true on OSS."
+ metabase-enterprise.remote-sync.core
+ [_table]
+ true)
diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.clj
index e3c73cf7981..a66ca0d4ad4 100644
--- a/src/metabase/segments/models/segment.clj
+++ b/src/metabase/segments/models/segment.clj
@@ -10,6 +10,7 @@
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
+ [metabase.remote-sync.core :as remote-sync]
[metabase.search.core :as search]
[metabase.segments.schema :as segments.schema]
[metabase.util :as u]
@@ -70,9 +71,7 @@
(doto :model/Segment
(derive :metabase/model)
(derive :hook/timestamped?)
- (derive :hook/entity-id)
- (derive ::mi/write-policy.superuser)
- (derive ::mi/create-policy.superuser))
+ (derive :hook/entity-id))
(defmethod mi/can-read? :model/Segment
([instance]
@@ -86,6 +85,57 @@
([model pk]
(mi/can-read? (t2/select-one model pk))))
+;; Segments can be written by superusers, but only if the parent table is editable
+;; (not in a remote-synced collection in read-only mode).
+(defmethod mi/can-write? :model/Segment
+ ([instance]
+ (let [table (or (:table instance)
+ (t2/select-one :model/Table :id (:table_id instance)))]
+ (and (mi/superuser?)
+ (remote-sync/table-editable? table))))
+ ([model pk]
+ (mi/can-write? (t2/select-one model pk))))
+
+;; Segments can be created by superusers, but only if the parent table is editable
+;; (not in a remote-synced collection in read-only mode).
+(defmethod mi/can-create? :model/Segment
+ [_model instance]
+ (let [table (or (:table instance)
+ (t2/select-one :model/Table :id (:table_id instance)))]
+ (and (mi/superuser?)
+ (remote-sync/table-editable? table))))
+
+(methodical/defmethod t2/batched-hydrate [:model/Segment :can_write]
+ "Batched hydration for :can_write on segments. First hydrates :table for all segments,
+ then pre-fetches collection is_remote_synced values for those tables, and calls can-write?
+ on each segment. This avoids N+1 queries when checking permissions for multiple segments."
+ [_model k segments]
+ (let [segments-with-tables (t2/hydrate (remove nil? segments) :table)
+ ;; Get all unique collection IDs from the hydrated tables
+ collection-ids (->> segments-with-tables
+ (keep (comp :collection_id :table))
+ distinct)
+ ;; Batch fetch is_remote_synced for all collections
+ collection-synced-map (if (seq collection-ids)
+ (into {}
+ (map (juxt :id :is_remote_synced))
+ (t2/select :model/Collection :id [:in collection-ids]))
+ {})
+ ;; Associate collection info with each segment's table
+ segments-with-collection (for [segment segments-with-tables
+ :let [table (:table segment)
+ coll-id (:collection_id table)]]
+ (if (and table coll-id)
+ (assoc-in segment [:table :collection]
+ {:id coll-id
+ :is_remote_synced (get collection-synced-map coll-id false)})
+ segment))]
+ (mi/instances-with-hydrated-data
+ segments k
+ #(u/index-by :id mi/can-write? segments-with-collection)
+ :id
+ {:default false})))
+
(defn- migrated-segment-definition
[{:keys [definition], table-id :table_id}]
(let [database-id (t2/select-one-fn :db_id :model/Table :id table-id)]
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 4f118de3643..fd1168e0a13 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -12,6 +12,7 @@
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.premium-features.core :refer [defenterprise]]
+ [metabase.remote-sync.core :as remote-sync]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
@@ -192,10 +193,15 @@
(mi/can-query? (t2/select-one model pk))))
(defenterprise current-user-can-write-field?
- "OSS implementation. Returns a boolean whether the current user can write the given field."
+ "OSS implementation. Returns a boolean whether the current user can write the given field.
+ Checks both that the user is a superuser and that the parent table is editable (not in a
+ remote-synced collection in read-only mode)."
metabase-enterprise.advanced-permissions.common
- [_instance]
- (mi/superuser?))
+ [instance]
+ (let [table (or (:table instance)
+ (t2/select-one :model/Table :id (:table_id instance)))]
+ (and (remote-sync/table-editable? table)
+ (mi/superuser?))))
(defmethod mi/can-write? :model/Field
([instance]
@@ -203,6 +209,37 @@
([model pk]
(mi/can-write? (t2/select-one model pk))))
+(methodical/defmethod t2/batched-hydrate [:model/Field :can_write]
+ "Batched hydration for :can_write on fields. First hydrates :table for all fields,
+ then pre-fetches collection is_remote_synced values for those tables, and calls can-write?
+ on each field. This avoids N+1 queries when checking permissions for multiple fields."
+ [_model k fields]
+ (let [fields-with-tables (t2/hydrate (remove nil? fields) :table)
+ ;; Get all unique collection IDs from the hydrated tables
+ collection-ids (->> fields-with-tables
+ (keep (comp :collection_id :table))
+ distinct)
+ ;; Batch fetch is_remote_synced for all collections
+ collection-synced-map (if (seq collection-ids)
+ (into {}
+ (map (juxt :id :is_remote_synced))
+ (t2/select :model/Collection :id [:in collection-ids]))
+ {})
+ ;; Associate collection info with each field's table
+ fields-with-collection (for [field fields-with-tables
+ :let [table (:table field)
+ coll-id (:collection_id table)]]
+ (if (and table coll-id)
+ (assoc-in field [:table :collection]
+ {:id coll-id
+ :is_remote_synced (get collection-synced-map coll-id false)})
+ field))]
+ (mi/instances-with-hydrated-data
+ fields k
+ #(u/index-by :id mi/can-write? fields-with-collection)
+ :id
+ {:default false})))
+
(defmethod serdes/hash-fields :model/Field
[_field]
[:name (serdes/hydrated-hash :table :table_id) (serdes/hydrated-hash :parent :parent_id)])
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index b6d2cea1c3c..9f2f45dd4f4 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -10,6 +10,7 @@
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise]]
+ [metabase.remote-sync.core :as remote-sync]
[metabase.search.spec :as search.spec]
[metabase.util :as u]
[metabase.util.log :as log]
@@ -297,10 +298,13 @@
(mi/can-query? (t2/select-one :model/Table pk))))
(defenterprise current-user-can-write-table?
- "OSS implementation. Returns a boolean whether the current user can write the given field."
+ "OSS implementation. Returns a boolean whether the current user can write the given table.
+ Checks both that the user is a superuser and that the table is editable (not in a remote-synced
+ collection in read-only mode)."
metabase-enterprise.advanced-permissions.common
- [_instance]
- (mi/superuser?))
+ [instance]
+ (and (remote-sync/table-editable? instance)
+ (mi/superuser?)))
(defmethod mi/can-write? :model/Table
;; Check if user can modify this table's metadata.
@@ -310,6 +314,32 @@
([_ pk]
(mi/can-write? (t2/select-one :model/Table pk))))
+(methodical/defmethod t2/batched-hydrate [:model/Table :can_write]
+ "Batched hydration for :can_write on tables. Pre-fetches collection is_remote_synced values
+ to avoid N+1 queries when checking remote-sync permissions for multiple tables."
+ [_model k tables]
+ (let [;; Get all unique collection IDs (excluding nil)
+ collection-ids (->> tables
+ (keep :collection_id)
+ distinct)
+ ;; Batch fetch is_remote_synced for all collections
+ collection-synced-map (if (seq collection-ids)
+ (into {}
+ (map (juxt :id :is_remote_synced))
+ (t2/select :model/Collection :id [:in collection-ids]))
+ {})
+ ;; Associate collection info with each table so table-editable? doesn't need to query
+ tables-with-collection (for [table tables]
+ (if-let [coll-id (:collection_id table)]
+ (assoc table :collection {:id coll-id
+ :is_remote_synced (get collection-synced-map coll-id false)})
+ table))]
+ (mi/instances-with-hydrated-data
+ tables k
+ #(u/index-by :id mi/can-write? tables-with-collection)
+ :id
+ {:default false})))
+
;;; ------------------------------------------------ SQL Permissions ------------------------------------------------
(mu/defmethod mi/visible-filter-clause :model/Table
@@ -487,6 +517,16 @@
(cond-> [[{:model "Database" :id db_id}]]
collection_id (conj [{:model "Collection" :id collection_id}])))
+(defmethod serdes/descendants "Table" [_model-name id {:keys [skip-archived]}]
+ (let [fields (into {} (for [field-id (t2/select-pks-set :model/Field {:where [:= :table_id id]})]
+ {["Field" field-id] {"Table" id}}))
+ segments (into {} (for [segment-id (t2/select-pks-set :model/Segment
+ {:where [:and
+ [:= :table_id id]
+ (when skip-archived [:not :archived])]})]
+ {["Segment" segment-id] {"Table" id}}))]
+ (merge fields segments)))
+
(defmethod serdes/generate-path "Table" [_ table]
(let [db-name (t2/select-one-fn :name :model/Database :id (:db_id table))]
(filterv some? [{:model "Database" :id db-name}
diff --git a/src/metabase/warehouse_schema_rest/api/field.clj b/src/metabase/warehouse_schema_rest/api/field.clj
index 4413965784c..1bb594602ec 100644
--- a/src/metabase/warehouse_schema_rest/api/field.clj
+++ b/src/metabase/warehouse_schema_rest/api/field.clj
@@ -4,6 +4,7 @@
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.app-db.core :as app-db]
+ [metabase.events.core :as events]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.models.interface :as mi]
[metabase.parameters.field :as parameters.field]
@@ -185,6 +186,7 @@
(u/prog1 (-> (t2/select-one :model/Field :id id)
(t2/hydrate :dimensions :has_field_values)
(field/hydrate-target-with-write-perms))
+ (events/publish-event! :event/field-update {:object <> :user-id api/*current-user-id*})
(when (not= effective-type (:effective_type field))
(analytics/track-event! :snowplow/simple_event {:event "field_effective_type_change" :target_id id})
(quick-task/submit-task! (fn [] (sync/refingerprint-field! <>)))))))
diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.clj
index bfc3732e482..2255abc4ac7 100644
--- a/src/metabase/warehouse_schema_rest/api/table.clj
+++ b/src/metabase/warehouse_schema_rest/api/table.clj
@@ -6,6 +6,7 @@
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.app-db.core :as app-db]
+ [metabase.config.core :as config]
[metabase.database-routing.core :as database-routing]
[metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
@@ -233,6 +234,12 @@
newly-unhidden (when (and (contains? body :visibility_type) (nil? visibility_type))
(into [] (filter (comp some? :visibility_type)) existing-tables))]
(sync-unhidden-tables newly-unhidden)
+ ;; Publish update events for remote sync tracking, only when ee extensions are available to provide
+ ;; a handler for these events
+ (when config/ee-available?
+ (doseq [table updated-tables]
+ (events/publish-event! :event/table-update {:object table
+ :user-id api/*current-user-id*})))
updated-tables)))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
diff --git a/test/metabase/warehouse_schema/models/table_test.clj b/test/metabase/warehouse_schema/models/table_test.clj
index 1552e25bea9..1eadf755906 100644
--- a/test/metabase/warehouse_schema/models/table_test.clj
+++ b/test/metabase/warehouse_schema/models/table_test.clj
@@ -664,3 +664,45 @@
(let [hydrated (t2/hydrate table :can_query)]
(is (contains? hydrated :can_query))
(is (boolean? (:can_query hydrated))))))))
+
+(deftest serdes-descendants-includes-fields-and-segments-test
+ (testing "Table descendants includes Fields and Segments"
+ (mt/with-temp [:model/Database {db-id :id} {:name "Test DB"}
+ :model/Table {table-id :id} {:name "Test Table" :db_id db-id}
+ :model/Field {field1-id :id} {:name "Field 1" :table_id table-id :base_type :type/Integer}
+ :model/Field {field2-id :id} {:name "Field 2" :table_id table-id :base_type :type/Text}
+ :model/Segment {segment-id :id} {:name "Test Segment" :table_id table-id :definition {}}]
+ (let [descendants (serdes/descendants "Table" table-id {})]
+ (testing "Fields are included"
+ (is (contains? descendants ["Field" field1-id]))
+ (is (contains? descendants ["Field" field2-id])))
+ (testing "Segments are included"
+ (is (contains? descendants ["Segment" segment-id]))))))
+ (testing "Table with no fields or segments returns empty map"
+ (mt/with-temp [:model/Database {db-id :id} {:name "Test DB"}
+ :model/Table {table-id :id} {:name "Empty Table" :db_id db-id}]
+ (is (= {} (serdes/descendants "Table" table-id {}))))))
+
+(deftest serdes-descendants-skip-archived-segments-test
+ (testing "Table descendants respects skip-archived option for Segments"
+ (mt/with-temp [:model/Database {db-id :id} {:name "Test DB"}
+ :model/Table {table-id :id} {:name "Test Table" :db_id db-id}
+ :model/Field {field-id :id} {:name "Test Field" :table_id table-id :base_type :type/Integer}
+ :model/Segment {active-seg-id :id} {:name "Active Segment" :table_id table-id :definition {} :archived false}
+ :model/Segment {archived-seg-id :id} {:name "Archived Segment" :table_id table-id :definition {} :archived true}]
+ (testing "archived segments are excluded when skip-archived: true"
+ (let [descendants (serdes/descendants "Table" table-id {:skip-archived true})]
+ (is (contains? descendants ["Field" field-id])
+ "Fields are still included")
+ (is (contains? descendants ["Segment" active-seg-id])
+ "Active segments are included")
+ (is (not (contains? descendants ["Segment" archived-seg-id]))
+ "Archived segments are excluded")))
+ (testing "archived segments are included when skip-archived: false"
+ (let [descendants (serdes/descendants "Table" table-id {:skip-archived false})]
+ (is (contains? descendants ["Segment" active-seg-id]))
+ (is (contains? descendants ["Segment" archived-seg-id]))))
+ (testing "archived segments are included when skip-archived is not specified"
+ (let [descendants (serdes/descendants "Table" table-id {})]
+ (is (contains? descendants ["Segment" active-seg-id]))
+ (is (contains? descendants ["Segment" archived-seg-id])))))))