Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: better display for transforms sync changes (#68919)

Edward Pagetf0abb34fe4c3
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 04f9ba5cef0..d7b0d7ce827 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -72,7 +72,7 @@
[:not-in entity-ids]
:entity_id)
scope-key (get-in model-spec [:removal :scope-key])
- extra-conditions (into [] cat (get-in model-spec [:removal :conditions]))]]
+ extra-conditions (into [] cat (:conditions model-spec))]]
(if scope-key
;; Collection-scoped: delete only within synced collections
(when (seq synced-collection-ids)
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
index 74675d564d0..7039012f2a1 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
@@ -29,9 +29,11 @@
;;; ------------------------------------------- Dirty Item Schemas -------------------------------------------
(def DirtyItem
- "Schema for a dirty sync item."
+ "Schema for a dirty sync item.
+ Note: id is :int instead of pos-int? because the Transforms root collection uses
+ a sentinel value of -1 as its model_id."
[:map
- [:id pos-int?]
+ [:id :int]
[:name [:maybe :string]]
[:model :string]
[:sync_status :string]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index 88cebef4481..326b76ae742 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -88,37 +88,53 @@
:encryption :no
:default (* 1000 60 5))
+(def ^:const transforms-root-id
+ "Sentinel value for the virtual Transforms root collection.
+ Used to represent the entire transforms feature being enabled/disabled."
+ -1)
+
(defn sync-transform-tracking!
"Called when remote-sync-transforms setting changes.
- When enabled: mark all existing transforms, transform tags, and transforms-namespace collections
- as 'create' for initial sync.
- When disabled: remove all transform-related tracking entries."
+ Creates a single 'Transforms' RSO entry with model_id=-1 as a sentinel value.
+ When enabled: status is 'create' to indicate transforms should be synced.
+ When disabled: status is 'delete' to indicate transforms should be removed.
+ When disabling and there's no existing Transforms RSO, does nothing (avoids creating
+ spurious 'delete' entries when going from default false to explicitly false)."
[enabled?]
(let [timestamp (t/offset-date-time)
- transform-coll-ids (t2/select-pks-set :model/Collection :namespace (name collection/transforms-ns))]
- (t2/delete! :model/RemoteSyncObject
- :model_type [:in ["Transform" "TransformTag"]])
- (when (seq transform-coll-ids)
- (t2/delete! :model/RemoteSyncObject
- :model_type "Collection"
- :model_id [:in transform-coll-ids]))
- (when enabled?
- (doseq [coll (t2/select [:model/Collection :id :name] :namespace (name collection/transforms-ns))]
+ existing-rso (t2/select-one :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id transforms-root-id)]
+ (cond
+ ;; When enabling, always create/update to 'create' status
+ enabled?
+ (do
+ (when existing-rso
+ (t2/delete! :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id transforms-root-id))
(t2/insert! :model/RemoteSyncObject
{:model_type "Collection"
- :model_id (:id coll)
- :model_name (:name coll)
+ :model_id transforms-root-id
+ :model_name "Transforms"
:status "create"
:status_changed_at timestamp}))
- (doseq [[model name-key] [[:model/Transform :name]
- [:model/TransformTag :name]]]
- (doseq [entity (t2/select [model :id name-key])]
- (t2/insert! :model/RemoteSyncObject
- {:model_type (name (last (str/split (name model) #"/")))
- :model_id (:id entity)
- :model_name (get entity name-key)
- :status "create"
- :status_changed_at timestamp}))))))
+ ;; When disabling and there's an existing RSO, update to 'delete' status
+ existing-rso
+ (do
+ (t2/delete! :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id transforms-root-id)
+ (t2/insert! :model/RemoteSyncObject
+ {:model_type "Collection"
+ :model_id transforms-root-id
+ :model_name "Transforms"
+ :status "delete"
+ :status_changed_at timestamp}))
+ ;; When disabling and there's no existing RSO, do nothing
+ ;; (this avoids creating spurious 'delete' entries when going from default false to explicitly false)
+ :else
+ nil)))
(defn- sync-transform-tracking-on-change
"Called when remote-sync-transforms setting changes."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index b6d1a5e49c2..2b4e140114c 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -51,12 +51,16 @@
:select-fields - Fields to select for hydration
:hydrate-query - Optional custom query for joins (overrides select-fields)
:field-mappings - Map of RemoteSyncObject column -> source field or [field transform-fn]
+ - :conditions - Optional map of conditions for filtering syncable entities.
+ Only entities matching these conditions are eligible for sync
+ (export, import cleanup, removal). Example: {:built_in_type nil}
+ to exclude built-in items from sync.
- :removal - Removal/cleanup configuration:
:statuses - Set of statuses to check for removal (e.g., #{\"removed\" \"delete\"})
:scope-key - Optional key for scoping deletions (e.g., :collection_id, :id).
If nil, deletions are global (by entity_id only).
- :conditions - Optional map of extra conditions for filtering removals
- (e.g., {:built_in_type nil} to protect built-in items)
+ :all-on-setting-disable - Optional setting keyword; when this setting's sentinel
+ RSO exists with 'delete' status, remove ALL entities of this type
- :export-scope - Export scope for query-export-roots:
:root-collections - Query root-level remote-synced + namespace collections (Collection)
:root-only - Query root instances with collection_id = nil (Transform)
@@ -267,7 +271,8 @@
:tracking {:select-fields [:name :collection_id]
:field-mappings {:model_name :name
:model_collection_id :collection_id}}
- :removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion
+ :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
+ :all-on-setting-disable :remote-sync-transforms}
:export-scope :root-only ; query for root transforms (collection_id = nil)
:enabled? :remote-sync-transforms}
@@ -282,8 +287,9 @@
:archived-key nil ; no archived field
:tracking {:select-fields [:name]
:field-mappings {:model_name :name}}
- :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
- :conditions {:built_in_type nil}} ; protect built-in tags from removal
+ :conditions {:built_in_type nil} ; exclude built-in tags from sync
+ :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
+ :all-on-setting-disable :remote-sync-transforms}
:export-scope :all ; query for all instances
:enabled? :remote-sync-transforms}
@@ -298,7 +304,8 @@
:archived-key nil ; no archived field
:tracking {:select-fields [:path]
:field-mappings {:model_name :path}}
- :removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion
+ :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
+ :all-on-setting-disable :remote-sync-transforms}
:export-scope :all ; query for all instances
:enabled? :remote-sync-transforms}})
@@ -759,6 +766,30 @@
;; Select all columns since different models need different fields for serdes paths
(t2/select model-key :id [:in removed-ids]))))
+(defn- setting-sentinel-delete?
+ "Returns true if the setting's sentinel RSO exists with 'delete' status.
+ Used to detect when a setting (like :remote-sync-transforms) has been disabled
+ and all entities of the controlled types should be removed."
+ [setting-key]
+ (when (= setting-key :remote-sync-transforms)
+ (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id rs-settings/transforms-root-id
+ :status "delete")))
+
+(defn- build-bulk-removal-paths
+ "Builds removal paths for ALL entities of a spec's model type.
+ Used when the controlling setting is disabled (sentinel RSO has 'delete' status).
+ Respects :conditions from the spec to exclude ineligible entities."
+ [spec ctx]
+ (let [{:keys [model-type model-key conditions]} spec]
+ (for [entity (if conditions
+ (apply t2/select model-key (into [] cat conditions))
+ (t2/select model-key))
+ :let [path (entity->serdes-path model-type entity ctx)]
+ :when path]
+ path)))
+
(defn build-all-removal-paths
"Builds full file paths for all entities marked for removal in RemoteSyncObject.
Uses serdes/storage-path to generate paths that exactly match the file structure.
@@ -768,19 +799,31 @@
2. Query actual entities by those IDs
3. Use serdes/storage-path to build the exact file path
+ Also handles bulk removal for specs with :all-on-setting-disable when the
+ controlling setting's sentinel RSO has 'delete' status.
+
Returns paths without file extensions - the git source adds .yaml as needed."
[]
- (let [ctx (serdes/storage-base-context)]
- (into []
- (for [[_model-key spec] (enabled-specs)
- :when (spec-enabled? spec)
- :let [{:keys [statuses]} (:removal spec)
- model-type (:model-type spec)]
- :when (seq statuses)
- entity (query-removed-entities spec)
- :let [path (entity->serdes-path model-type entity ctx)]
- :when path]
- path))))
+ (let [ctx (serdes/storage-base-context)
+ ;; Standard removal paths from enabled specs
+ standard-paths (into []
+ (for [[_model-key spec] (enabled-specs)
+ :when (spec-enabled? spec)
+ :let [{:keys [statuses]} (:removal spec)
+ model-type (:model-type spec)]
+ :when (seq statuses)
+ entity (query-removed-entities spec)
+ :let [path (entity->serdes-path model-type entity ctx)]
+ :when path]
+ path))
+ ;; Bulk removal paths for specs with :all-on-setting-disable
+ bulk-paths (into []
+ (for [[_model-key spec] remote-sync-specs
+ :let [setting-key (get-in spec [:removal :all-on-setting-disable])]
+ :when (and setting-key (setting-sentinel-delete? setting-key))
+ path (build-bulk-removal-paths spec ctx)]
+ path))]
+ (into standard-paths bulk-paths)))
;;; ----------------------------------------- Sync Object Query Functions --------------------------------------------
@@ -958,13 +1001,16 @@
nil))
(defmethod query-export-roots :setting
- [{:keys [export-scope model-key model-type] :as spec}]
+ [{:keys [export-scope model-key model-type conditions] :as spec}]
(when (spec-enabled? spec)
(case export-scope
:root-only
- (t2/select-fn-set (juxt (constantly model-type) :id) model-key :collection_id nil)
+ (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key
+ :collection_id nil
+ (into [] cat conditions))
:all
- (t2/select-fn-set (juxt (constantly model-type) :id) model-key)
+ (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key
+ (into [] cat conditions))
nil)))
(defmethod query-export-roots :published-table [_] nil)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index fd0d7123377..41a208ae69b 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -1017,3 +1017,51 @@
(is (=? {:success true} resp))
(is (true? (settings/remote-sync-transforms))
"Transforms setting should be preserved when not included in request")))))))
+
+;;; ------------------------------------------- Dirty Endpoint with Transforms Root Tests -------------------------------------------
+
+(deftest dirty-returns-transforms-root-collection-test
+ (testing "GET /api/ee/remote-sync/dirty returns the Transforms root collection (id=-1) when present"
+ (test-helpers/with-clean-object
+ (mt/with-temp [:model/RemoteSyncObject _ {:model_type "Collection"
+ :model_id settings/transforms-root-id
+ :model_name "Transforms"
+ :status "create"
+ :status_changed_at (java.time.OffsetDateTime/now)}]
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/dirty")
+ dirty-items (:dirty response)]
+ (is (= 1 (count dirty-items)))
+ (is (= "Transforms" (:name (first dirty-items))))
+ (is (= "collection" (:model (first dirty-items))))
+ (is (= settings/transforms-root-id (:id (first dirty-items)))))))))
+
+(deftest dirty-returns-transforms-root-collection-with-delete-status-test
+ (testing "GET /api/ee/remote-sync/dirty returns the Transforms root collection (id=-1) with delete status"
+ (test-helpers/with-clean-object
+ (mt/with-temp [:model/RemoteSyncObject _ {:model_type "Collection"
+ :model_id settings/transforms-root-id
+ :model_name "Transforms"
+ :status "delete"
+ :status_changed_at (java.time.OffsetDateTime/now)}]
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/dirty")
+ dirty-items (:dirty response)]
+ (is (= 1 (count dirty-items)))
+ (is (= "Transforms" (:name (first dirty-items))))
+ (is (= "collection" (:model (first dirty-items))))
+ (is (= "delete" (:sync_status (first dirty-items))))
+ (is (= settings/transforms-root-id (:id (first dirty-items)))))))))
+
+(deftest dirty-returns-transforms-root-collection-when-setting-disabled-test
+ (testing "GET /api/ee/remote-sync/dirty returns the Transforms root collection when transforms setting is disabled"
+ (test-helpers/with-clean-object
+ (mt/with-temporary-setting-values [remote-sync-transforms false]
+ (mt/with-temp [:model/RemoteSyncObject _ {:model_type "Collection"
+ :model_id settings/transforms-root-id
+ :model_name "Transforms"
+ :status "delete"
+ :status_changed_at (java.time.OffsetDateTime/now)}]
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/dirty")
+ dirty-items (:dirty response)]
+ (is (= 1 (count dirty-items)) "Transforms root should be returned even when setting is disabled")
+ (is (= "Transforms" (:name (first dirty-items))))
+ (is (= "delete" (:sync_status (first dirty-items))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index 1cf885f511e..3bf94ff11f4 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -120,49 +120,52 @@
"Transforms-namespace collection should NOT be tracked when setting is disabled"))))))
(deftest enable-transform-sync-marks-all-transforms-test
- (testing "Enabling transform sync marks all existing transforms, tags, and collections for initial sync"
+ (testing "Enabling transform sync creates a single Transforms RSO with model_id=-1 and status 'create'"
(mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms false
- remote-sync-enabled true]
+ (mt/with-temporary-setting-values [remote-sync-enabled true]
+ ;; Clean up any existing Transforms root RSO first
+ (t2/delete! :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id)
(mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns}
- :model/Transform transform {:name "Existing Transform" :collection_id coll-id}
- :model/TransformTag tag {:name "Existing Tag"}]
- (is (zero? (t2/count :model/RemoteSyncObject :model_type [:in ["Transform" "TransformTag"]]))
- "Should have no transform tracking entries initially")
+ :model/Transform _transform {:name "Existing Transform" :collection_id coll-id}
+ :model/TransformTag _tag {:name "Existing Tag"}]
+ (is (not (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id))
+ "Should not have Transforms root RSO initially")
(settings/sync-transform-tracking! true)
(is (t2/exists? :model/RemoteSyncObject
:model_type "Collection"
- :model_id coll-id
- :status "create")
- "Transforms-namespace collection should be marked for initial sync")
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Transform"
- :model_id (:id transform)
+ :model_id settings/transforms-root-id
:status "create")
- "Transform should be marked for initial sync")
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "TransformTag"
- :model_id (:id tag)
- :status "create")
- "TransformTag should be marked for initial sync"))))))
+ "Transforms root should be created with status 'create'")
+ (let [rso (t2/select-one :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id)]
+ (is (= "Transforms" (:model_name rso))
+ "Transforms root should have name 'Transforms'")))))))
(deftest disable-transform-sync-removes-all-tracking-test
- (testing "Disabling transform sync removes all transform-related tracking entries"
+ (testing "Disabling transform sync creates a Transforms RSO with status 'delete'"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms false
remote-sync-enabled true]
(mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns}
- :model/Transform transform {:name "Test Transform" :collection_id coll-id}
- :model/TransformTag tag {:name "Test Tag"}]
+ :model/Transform _transform {:name "Test Transform" :collection_id coll-id}
+ :model/TransformTag _tag {:name "Test Tag"}]
(mt/with-temporary-setting-values [remote-sync-transforms true]
- (is (= 3 (t2/count :model/RemoteSyncObject :model_type [:in ["Collection" "Transform" "TransformTag"]]
- :model_id [:in [coll-id (:id transform) (:id tag)]]))
- "Should have 3 tracking entries")
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id
+ :status "create")
+ "Should have Transforms root RSO with 'create' status")
(settings/sync-transform-tracking! false)
- (is (zero? (t2/count :model/RemoteSyncObject :model_type [:in ["Transform" "TransformTag"]]))
- "Transform and TransformTag tracking entries should be removed")
- (is (zero? (t2/count :model/RemoteSyncObject :model_type "Collection" :model_id coll-id))
- "Transforms-namespace collection tracking entry should be removed")))))))
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id
+ :status "delete")
+ "Transforms root should now have 'delete' status")))))))
(deftest transforms-included-in-dirty-check-when-enabled-test
(testing "Transforms are included in dirty check when setting is enabled"
@@ -811,3 +814,85 @@ serdes/meta:
"Remote transform should be imported")
(is (not (t2/exists? :model/Transform :id local-transform-id))
"Local transform should be removed because it's not on the remote")))))))))
+
+;;; ------------------------------------------- build-all-removal-paths Tests -------------------------------------------
+
+(deftest build-all-removal-paths-includes-all-transforms-on-setting-disable-test
+ (testing "build-all-removal-paths returns paths for all transforms content when sentinel RSO has 'delete' status"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
+ :namespace collection/transforms-ns
+ :location "/"}
+ :model/Transform {transform-id :id transform-eid :entity_id} {:name "Test Transform"
+ :collection_id coll-id}
+ :model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"
+ :built_in_type nil}]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})
+ lib-eid (:entity_id library)]
+ (is (t2/exists? :model/Transform :id transform-id))
+ (is (t2/exists? :model/TransformTag :id tag-id))
+ (is (t2/exists? :model/PythonLibrary :id (:id library)))
+ (let [paths-before (spec/build-all-removal-paths)]
+ (is (not (some #(str/includes? % transform-eid) paths-before))
+ "Transform should not be in removal paths before setting is disabled"))
+ (settings/sync-transform-tracking! true)
+ (settings/sync-transform-tracking! false)
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id
+ :status "delete")
+ "Sentinel RSO should exist with 'delete' status")
+ (let [paths-after (spec/build-all-removal-paths)]
+ (is (some #(str/includes? % transform-eid) paths-after)
+ "Transform should be in removal paths after setting is disabled")
+ (is (some #(str/includes? % tag-eid) paths-after)
+ "TransformTag should be in removal paths after setting is disabled")
+ (is (some #(str/includes? % lib-eid) paths-after)
+ "PythonLibrary should be in removal paths after setting is disabled"))))))))))
+
+(deftest build-all-removal-paths-excludes-builtin-transform-tags-test
+ (testing "build-all-removal-paths respects :conditions and excludes built-in tags"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-temp [:model/TransformTag {custom-tag-id :id custom-tag-eid :entity_id} {:name "Custom Tag"
+ :built_in_type nil}
+ :model/TransformTag {builtin-tag-id :id builtin-tag-eid :entity_id} {:name "Built-in Tag"
+ :built_in_type "target"}]
+ (is (t2/exists? :model/TransformTag :id custom-tag-id))
+ (is (t2/exists? :model/TransformTag :id builtin-tag-id))
+ (settings/sync-transform-tracking! true)
+ (settings/sync-transform-tracking! false)
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id
+ :status "delete")
+ "Sentinel RSO should exist with 'delete' status")
+ (let [paths (spec/build-all-removal-paths)]
+ (is (some #(str/includes? % custom-tag-eid) paths)
+ "Custom tag (built_in_type nil) should be in removal paths")
+ (is (not (some #(str/includes? % builtin-tag-eid) paths))
+ "Built-in tag should NOT be in removal paths")))))))
+
+(deftest export-excludes-builtin-transform-tags-test
+ (testing "Export excludes built-in transform tags based on :conditions"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-temp [:model/TransformTag {custom-tag-id :id} {:name "Custom Tag"
+ :built_in_type nil}
+ :model/TransformTag {builtin-tag-id :id} {:name "Built-in Tag"
+ :built_in_type "target"}]
+ (is (t2/exists? :model/TransformTag :id custom-tag-id))
+ (is (t2/exists? :model/TransformTag :id builtin-tag-id))
+ (let [transform-tag-spec (spec/spec-for-model-key :model/TransformTag)
+ export-roots (spec/query-export-roots transform-tag-spec)
+ exported-ids (set (map second export-roots))]
+ (is (contains? exported-ids custom-tag-id)
+ "Custom tag (built_in_type nil) should be in export roots")
+ (is (not (contains? exported-ids builtin-tag-id))
+ "Built-in tag should NOT be in export roots")))))))
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
index f6476ec9f79..32b2266ee93 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
@@ -1,9 +1,7 @@
import { Fragment, useMemo } from "react";
import { t } from "ttag";
-import _ from "underscore";
import { useListCollectionsTreeQuery } from "metabase/api";
-import { isLibraryCollection } from "metabase/collections/utils";
import { useSetting } from "metabase/common/hooks";
import {
Box,
@@ -15,33 +13,25 @@ import {
Text,
Title,
} from "metabase/ui";
+import {
+ type CollectionGroup,
+ TRANSFORMS_ROOT_ID,
+ buildNamespaceCollectionMap,
+ findLibraryCollectionId,
+ getGroupIcon,
+ groupEntitiesByCollection,
+} from "metabase-enterprise/remote_sync/displayGroups";
import {
buildCollectionMap,
getCollectionPathSegments,
getSyncStatusColor,
getSyncStatusIcon,
- isTableChildModel,
} from "metabase-enterprise/remote_sync/utils";
-import type { Collection, RemoteSyncEntity } from "metabase-types/api";
+import type { RemoteSyncEntity } from "metabase-types/api";
import { CollectionPath } from "./CollectionPath";
import { EntityLink } from "./EntityLink";
-/** A table group with optional entity (when table itself is dirty) and its nested children */
-type TableGroup = {
- tableId: number;
- tableName: string;
- table?: RemoteSyncEntity;
- children: RemoteSyncEntity[];
-};
-
-const SYNC_STATUS_ORDER: RemoteSyncEntity["sync_status"][] = [
- "create",
- "update",
- "touch",
- "delete",
-];
-
interface AllChangesViewProps {
entities: RemoteSyncEntity[];
title?: string;
@@ -65,45 +55,28 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {
namespace: "snippets",
});
- // Build a set of snippet collection IDs for icon selection
- // Only include collections with namespace "snippets" to avoid false positives
- const snippetCollectionIds = useMemo(() => {
- const ids = new Set<number>();
- const collectIds = (collections: typeof snippetCollectionTree) => {
- for (const collection of collections) {
- if (
- typeof collection.id === "number" &&
- collection.namespace === "snippets"
- ) {
- ids.add(collection.id);
- }
- if (collection.children) {
- collectIds(collection.children);
- }
- }
- };
- collectIds(snippetCollectionTree);
- return ids;
- }, [snippetCollectionTree]);
+ // Build namespace-to-collection-ids map in a single pass
+ const namespaceCollectionMap = useMemo(
+ () =>
+ buildNamespaceCollectionMap([
+ ...collectionTree,
+ ...snippetCollectionTree,
+ ]),
+ [collectionTree, snippetCollectionTree],
+ );
+
+ // Find the Transforms root entity (id=-1) if it exists
+ const transformsRootEntity = useMemo(() => {
+ return entities.find(
+ (e) => e.model === "collection" && e.id === TRANSFORMS_ROOT_ID,
+ );
+ }, [entities]);
// Find the library collection ID for placing snippets without a collection_id
- const libraryCollectionId = useMemo(() => {
- const findLibrary = (collections: Collection[]): number | null => {
- for (const col of collections) {
- if (isLibraryCollection(col)) {
- return typeof col.id === "number" ? col.id : null;
- }
- if (col.children?.length) {
- const found = findLibrary(col.children);
- if (found !== null) {
- return found;
- }
- }
- }
- return null;
- };
- return findLibrary(collectionTree);
- }, [collectionTree]);
+ const libraryCollectionId = useMemo(
+ () => findLibraryCollectionId(collectionTree),
+ [collectionTree],
+ );
const collectionMap = useMemo(() => {
// Merge regular collections with snippet collections
@@ -118,118 +91,24 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {
);
}, [entities]);
- const groupedData = useMemo(() => {
- const byCollection = _.groupBy(entities, (e) => {
- if (e.collection_id != null) {
- return e.collection_id;
- }
- // Snippets without a collection_id should go under the Library collection
- if (e.model === "nativequerysnippet" && libraryCollectionId != null) {
- return libraryCollectionId;
- }
- // Default to Root for other entities without a collection_id
- return 0;
- });
-
- const sortByStatus = (items: RemoteSyncEntity[]) =>
- items.sort((a, b) => {
- const statusOrderA = SYNC_STATUS_ORDER.indexOf(a.sync_status);
- const statusOrderB = SYNC_STATUS_ORDER.indexOf(b.sync_status);
- if (statusOrderA !== statusOrderB) {
- return statusOrderA - statusOrderB;
- }
- return a.name.localeCompare(b.name);
- });
-
- return Object.entries(byCollection)
- .map(([collectionId, items]) => {
- const collectionEntity = items.find(
- (item) =>
- item.model === "collection" && item.id === Number(collectionId),
- );
-
- // Separate tables, table children, and other items
- const tables = items.filter((item) => item.model === "table");
- const tableChildren = items.filter((item) =>
- isTableChildModel(item.model),
- );
- const otherItems = items.filter(
- (item) =>
- item.model !== "table" &&
- !isTableChildModel(item.model) &&
- !(item.model === "collection" && item.id === Number(collectionId)),
- );
-
- // Group table children by their parent table_id
- const childrenByTableId = _.groupBy(
- tableChildren,
- (e) => e.table_id || 0,
- );
-
- // Create table groups for dirty tables
- const tableIds = new Set(tables.map((t) => t.id));
- const dirtyTableGroups: TableGroup[] = tables.map((table) => ({
- tableId: table.id,
- tableName: table.name,
- table,
- children: sortByStatus(childrenByTableId[table.id] || []),
- }));
-
- // Create table groups for orphan children (children whose parent table is not dirty)
- const orphanTableGroups: TableGroup[] = Object.entries(
- childrenByTableId,
- )
- .filter(([tableId]) => !tableIds.has(Number(tableId)))
- .map(([tableId, children]) => ({
- tableId: Number(tableId),
- tableName: children[0]?.table_name ?? t`Unknown table`,
- children: sortByStatus(children),
- }));
-
- // Combine and sort all table groups
- const tableGroups = [...dirtyTableGroups, ...orphanTableGroups].sort(
- (a, b) => a.tableName.localeCompare(b.tableName),
- );
-
- const numericCollectionId = Number(collectionId) || undefined;
- const isSnippetCollection =
- numericCollectionId !== undefined &&
- snippetCollectionIds.has(numericCollectionId);
-
- // Get base path segments for this collection
- let pathSegments = getCollectionPathSegments(
- numericCollectionId,
- collectionMap,
- );
-
- // For snippet collections, prepend the Library collection path
- // This makes them appear as descendants of the Library collection
- if (isSnippetCollection && libraryCollectionId != null) {
- const libraryCollection = collectionMap.get(libraryCollectionId);
- if (libraryCollection) {
- pathSegments = [
- { id: libraryCollection.id, name: libraryCollection.name },
- ...pathSegments,
- ];
- }
- }
-
- return {
- pathSegments,
- collectionId: numericCollectionId,
- collectionEntity,
- tableGroups,
- items: sortByStatus(otherItems),
- isSnippetCollection,
- };
- })
- .sort((a, b) =>
- a.pathSegments
- .map((s) => s.name)
- .join(" / ")
- .localeCompare(b.pathSegments.map((s) => s.name).join(" / ")),
- );
- }, [entities, collectionMap, snippetCollectionIds, libraryCollectionId]);
+ const groupedData: CollectionGroup[] = useMemo(
+ () =>
+ groupEntitiesByCollection({
+ entities,
+ transformsRootEntity,
+ namespaceCollectionMap,
+ collectionMap,
+ libraryCollectionId,
+ getCollectionPathSegments,
+ }),
+ [
+ entities,
+ collectionMap,
+ namespaceCollectionMap,
+ libraryCollectionId,
+ transformsRootEntity,
+ ],
+ );
return (
<Box>
@@ -266,11 +145,7 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {
bdrs="md"
>
<Icon
- name={
- group.isSnippetCollection
- ? "snippet"
- : "synced_collection"
- }
+ name={getGroupIcon(group.spec)}
size={16}
c="text-secondary"
/>
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
index a6359d71d27..d72ba1ba298 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
@@ -1,8 +1,11 @@
import { setupCollectionTreeEndpoint } from "__support__/server-mocks";
+import { mockSettings } from "__support__/settings";
import { renderWithProviders, screen } from "__support__/ui";
+import { TRANSFORMS_ROOT_ID } from "metabase-enterprise/remote_sync/displayGroups";
import type { Collection, RemoteSyncEntity } from "metabase-types/api";
import { createMockCollection } from "metabase-types/api/mocks";
import { createMockRemoteSyncEntity } from "metabase-types/api/mocks/remote-sync";
+import { createMockState } from "metabase-types/store/mocks";
import { AllChangesView } from "./AllChangesView";
@@ -30,13 +33,23 @@ const deletedEntity = createMockRemoteSyncEntity({
const setup = ({
entities = [updatedEntity],
collections = [defaultCollection],
+ isTransformsSyncEnabled = false,
}: {
entities: RemoteSyncEntity[];
collections?: Collection[];
+ isTransformsSyncEnabled?: boolean;
}) => {
setupCollectionTreeEndpoint(collections);
- renderWithProviders(<AllChangesView entities={entities} />);
+ const storeInitialState = createMockState({
+ settings: mockSettings({
+ "remote-sync-transforms": isTransformsSyncEnabled,
+ }),
+ });
+
+ renderWithProviders(<AllChangesView entities={entities} />, {
+ storeInitialState,
+ });
};
describe("AllChangesView", () => {
@@ -224,6 +237,70 @@ describe("AllChangesView", () => {
await screen.findByText("New Transforms Collection"),
).toBeInTheDocument();
});
+
+ it("should display transforms collections as descendants of Transforms virtual root", async () => {
+ const transformsCollection = createMockCollection({
+ id: 100,
+ name: "My Transforms Collection",
+ namespace: "transforms",
+ effective_ancestors: [],
+ });
+ const transformEntity = createMockRemoteSyncEntity({
+ id: 200,
+ name: "Transform in Collection",
+ model: "transform",
+ collection_id: 100,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformEntity],
+ collections: [transformsCollection],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(
+ await screen.findByText("My Transforms Collection"),
+ ).toBeInTheDocument();
+ expect(screen.getByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("Transform in Collection")).toBeInTheDocument();
+ });
+
+ it("should display nested transforms collections with Transforms virtual root in path", async () => {
+ const childTransformsCollection = createMockCollection({
+ id: 101,
+ name: "Child Transforms Collection",
+ namespace: "transforms",
+ });
+ const parentTransformsCollection = createMockCollection({
+ id: 100,
+ name: "Parent Transforms Collection",
+ namespace: "transforms",
+ children: [childTransformsCollection],
+ });
+ const transformEntity = createMockRemoteSyncEntity({
+ id: 200,
+ name: "Transform in Child",
+ model: "transform",
+ collection_id: 101,
+ sync_status: "update",
+ });
+
+ setup({
+ entities: [transformEntity],
+ collections: [parentTransformsCollection],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(
+ await screen.findByText("Child Transforms Collection"),
+ ).toBeInTheDocument();
+ expect(screen.getByText("Transforms")).toBeInTheDocument();
+ expect(
+ screen.getByText("Parent Transforms Collection"),
+ ).toBeInTheDocument();
+ expect(screen.getByText("Transform in Child")).toBeInTheDocument();
+ });
});
describe("table child models", () => {
@@ -530,21 +607,21 @@ describe("AllChangesView", () => {
expect(screen.queryByText("Library")).not.toBeInTheDocument();
});
- it("should find Library collection in nested tree structure", async () => {
+ it("should find Library collection alongside other root collections", async () => {
const libraryCollection = createMockCollection({
id: 99,
name: "Library",
type: "library",
effective_ancestors: [],
});
- const parentCollection = createMockCollection({
+ const otherCollection = createMockCollection({
id: 1,
- name: "Parent Collection",
- children: [libraryCollection],
+ name: "Other Collection",
+ effective_ancestors: [],
});
const snippetEntity = createMockRemoteSyncEntity({
id: 300,
- name: "Nested Library Snippet",
+ name: "Root Library Snippet",
model: "nativequerysnippet",
collection_id: undefined,
sync_status: "create",
@@ -552,11 +629,11 @@ describe("AllChangesView", () => {
setup({
entities: [snippetEntity],
- collections: [parentCollection],
+ collections: [libraryCollection, otherCollection],
});
expect(await screen.findByText("Library")).toBeInTheDocument();
- expect(screen.getByText("Nested Library Snippet")).toBeInTheDocument();
+ expect(screen.getByText("Root Library Snippet")).toBeInTheDocument();
});
it("should display snippet collections as descendants of Library", async () => {
@@ -627,4 +704,204 @@ describe("AllChangesView", () => {
expect(screen.getByText("SELECT Query")).toBeInTheDocument();
});
});
+
+ describe("Transforms root collection", () => {
+ it("should display Transforms root as a parent group when present", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformsRootEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ });
+
+ it("should group transform entities under Transforms root", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+ const transformEntity = createMockRemoteSyncEntity({
+ id: 200,
+ name: "My Transform",
+ model: "transform",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformsRootEntity, transformEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("My Transform")).toBeInTheDocument();
+ });
+
+ it("should group transform tags under Transforms root", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+ const transformTagEntity = createMockRemoteSyncEntity({
+ id: 300,
+ name: "My Tag",
+ model: "transformtag",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformsRootEntity, transformTagEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("My Tag")).toBeInTheDocument();
+ });
+
+ it("should group pythonlibrary under Transforms root", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+ const pythonLibEntity = createMockRemoteSyncEntity({
+ id: 400,
+ name: "common.py",
+ model: "pythonlibrary",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformsRootEntity, pythonLibEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("common.py")).toBeInTheDocument();
+ });
+
+ it("should group transforms-namespace collections under Transforms root", async () => {
+ const transformsCollection = createMockCollection({
+ id: 100,
+ name: "My Transforms Collection",
+ namespace: "transforms",
+ });
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+ const collectionEntity = createMockRemoteSyncEntity({
+ id: 100,
+ name: "My Transforms Collection",
+ model: "collection",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformsRootEntity, collectionEntity],
+ collections: [transformsCollection],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("My Transforms Collection")).toBeInTheDocument();
+ });
+
+ it("should show delete icon when Transforms root has delete status", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ collection_id: undefined,
+ sync_status: "delete",
+ });
+
+ setup({
+ entities: [transformsRootEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByRole("img", { name: /trash/i })).toBeInTheDocument();
+ });
+
+ it("should group transform entities under Transforms even without transforms root entity", async () => {
+ const transformEntity = createMockRemoteSyncEntity({
+ id: 200,
+ name: "My Transform",
+ model: "transform",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("My Transform")).toBeInTheDocument();
+ expect(screen.queryByText("Root")).not.toBeInTheDocument();
+ });
+
+ it("should group transform tags under Transforms even without transforms root entity", async () => {
+ const transformTagEntity = createMockRemoteSyncEntity({
+ id: 300,
+ name: "My Tag",
+ model: "transformtag",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [transformTagEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("My Tag")).toBeInTheDocument();
+ expect(screen.queryByText("Root")).not.toBeInTheDocument();
+ });
+
+ it("should group pythonlibrary under Transforms even without transforms root entity", async () => {
+ const pythonLibEntity = createMockRemoteSyncEntity({
+ id: 400,
+ name: "common.py",
+ model: "pythonlibrary",
+ collection_id: undefined,
+ sync_status: "create",
+ });
+
+ setup({
+ entities: [pythonLibEntity],
+ isTransformsSyncEnabled: true,
+ });
+
+ expect(await screen.findByText("Transforms")).toBeInTheDocument();
+ expect(screen.getByText("common.py")).toBeInTheDocument();
+ expect(screen.queryByText("Root")).not.toBeInTheDocument();
+ });
+ });
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts
new file mode 100644
index 00000000000..96934c029cd
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/displayGroups.ts
@@ -0,0 +1,501 @@
+import { t } from "ttag";
+import _ from "underscore";
+
+import { isLibraryCollection } from "metabase/collections/utils";
+import type { IconName } from "metabase/ui";
+import type {
+ Collection,
+ CollectionId,
+ RemoteSyncEntity,
+ RemoteSyncEntityModel,
+ RemoteSyncEntityStatus,
+} from "metabase-types/api";
+
+/**
+ * Sentinel value for the virtual Transforms root collection.
+ * Used to represent the entire transforms feature being enabled/disabled.
+ */
+export const TRANSFORMS_ROOT_ID = -1;
+
+/**
+ * Configuration for how entities are grouped and displayed in the changes view.
+ * Similar to the backend remote-sync-specs pattern.
+ */
+export type DisplayGroupSpec = {
+ /** Unique identifier for this group */
+ id: string;
+ /** Collection namespace to match (e.g., "transforms", "snippets") */
+ namespace?: string;
+ /** Model types that belong to this group */
+ models?: Set<RemoteSyncEntityModel>;
+ /** Virtual root ID for groups that have a synthetic root (e.g., -1 for Transforms) */
+ virtualRootId?: number;
+ /** i18n function for virtual root name */
+ virtualRootName?: () => string;
+ /** Icon to display for this group's collections */
+ icon: IconName;
+ /** ID of another group whose path should be prepended */
+ pathPrefixGroupId?: string;
+ /** Priority for matching (higher values checked first) */
+ priority: number;
+};
+
+/**
+ * Display group specifications ordered by priority (highest first).
+ * Each spec defines how a category of entities should be grouped and displayed.
+ */
+const displayGroupSpecs: DisplayGroupSpec[] = [
+ {
+ id: "transforms",
+ namespace: "transforms",
+ models: new Set(["transform", "transformtag", "pythonlibrary"]),
+ virtualRootId: TRANSFORMS_ROOT_ID,
+ virtualRootName: () => t`Transforms`,
+ icon: "transform",
+ priority: 100,
+ },
+ {
+ id: "snippets",
+ namespace: "snippets",
+ models: new Set(["nativequerysnippet"]),
+ icon: "snippet",
+ pathPrefixGroupId: "library",
+ priority: 90,
+ },
+ {
+ id: "tables",
+ models: new Set(["table", "field", "segment", "measure"]),
+ icon: "synced_collection",
+ priority: 50,
+ },
+ {
+ id: "default",
+ icon: "synced_collection",
+ priority: 0,
+ },
+];
+
+/**
+ * Map of namespace string to set of collection IDs in that namespace.
+ */
+export type NamespaceCollectionMap = Map<string, Set<number>>;
+
+/**
+ * Build a map from namespace to collection IDs in a single pass.
+ */
+export const buildNamespaceCollectionMap = (
+ collectionTree: Collection[],
+): NamespaceCollectionMap => {
+ const map = new Map<string, Set<number>>();
+
+ const collectIds = (collections: Collection[]) => {
+ for (const collection of collections) {
+ if (
+ typeof collection.id === "number" &&
+ collection.namespace &&
+ typeof collection.namespace === "string"
+ ) {
+ const existing = map.get(collection.namespace);
+ if (existing) {
+ existing.add(collection.id);
+ } else {
+ map.set(collection.namespace, new Set([collection.id]));
+ }
+ }
+ if (collection.children) {
+ collectIds(collection.children);
+ }
+ }
+ };
+
+ collectIds(collectionTree);
+ return map;
+};
+
+/**
+ * Find the display group spec that matches an entity.
+ * Checks in priority order: model match, then namespace match (for collections).
+ */
+const getSpecForEntity = (
+ entity: RemoteSyncEntity,
+ namespaceCollectionMap: NamespaceCollectionMap,
+): DisplayGroupSpec => {
+ for (const spec of displayGroupSpecs) {
+ if (spec.models?.has(entity.model)) {
+ return spec;
+ }
+ if (entity.model === "collection" && spec.namespace) {
+ const namespaceIds = namespaceCollectionMap.get(spec.namespace);
+ if (namespaceIds?.has(entity.id)) {
+ return spec;
+ }
+ }
+ if (entity.collection_id != null && spec.namespace) {
+ const namespaceIds = namespaceCollectionMap.get(spec.namespace);
+ if (namespaceIds?.has(entity.collection_id)) {
+ return spec;
+ }
+ }
+ }
+ return displayGroupSpecs[displayGroupSpecs.length - 1];
+};
+
+/**
+ * Result of getGroupKeyInfo containing both the group key and the matched spec.
+ */
+type GroupKeyInfo = {
+ groupKey: string | number;
+ spec: DisplayGroupSpec;
+};
+
+/**
+ * Determine the group key for an entity based on its spec.
+ * Returns both the key and the spec for use in rendering.
+ */
+const getGroupKeyInfo = (
+ entity: RemoteSyncEntity,
+ spec: DisplayGroupSpec,
+ libraryCollectionId: number | null,
+ transformsRootExists: boolean,
+): GroupKeyInfo => {
+ if (entity.model === "collection" && entity.id === TRANSFORMS_ROOT_ID) {
+ return { groupKey: "transforms-root", spec };
+ }
+ if (spec.id === "transforms") {
+ if (transformsRootExists || entity.collection_id == null) {
+ return { groupKey: "transforms-root", spec };
+ }
+ return { groupKey: entity.collection_id, spec };
+ }
+ if (entity.collection_id != null) {
+ return { groupKey: entity.collection_id, spec };
+ }
+ if (spec.id === "snippets" && libraryCollectionId != null) {
+ return { groupKey: libraryCollectionId, spec };
+ }
+ return { groupKey: 0, spec };
+};
+
+/**
+ * Get the icon name for a display group.
+ */
+export const getGroupIcon = (spec: DisplayGroupSpec): IconName => {
+ return spec.icon;
+};
+
+/**
+ * Check if a collection is in a specific namespace.
+ */
+const isCollectionInNamespace = (
+ collectionId: number,
+ namespace: string,
+ namespaceCollectionMap: NamespaceCollectionMap,
+): boolean => {
+ const namespaceIds = namespaceCollectionMap.get(namespace);
+ return namespaceIds?.has(collectionId) ?? false;
+};
+
+/**
+ * Find the Library collection in the collection tree.
+ */
+export const findLibraryCollectionId = (
+ collectionTree: Collection[],
+): number | null => {
+ const findLibrary = (collections: Collection[]): number | null => {
+ for (const col of collections) {
+ if (isLibraryCollection(col)) {
+ return typeof col.id === "number" ? col.id : null;
+ }
+ if (col.children?.length) {
+ const found = findLibrary(col.children);
+ if (found !== null) {
+ return found;
+ }
+ }
+ }
+ return null;
+ };
+ return findLibrary(collectionTree);
+};
+
+export type CollectionPathSegment = {
+ id: CollectionId;
+ name: string;
+};
+
+/**
+ * Get path prefix segments for entities that need a virtual parent path.
+ * For example, snippet collections get "Library" prepended, transforms collections get "Transforms" prepended.
+ */
+const getPathPrefixSegments = (
+ spec: DisplayGroupSpec,
+ collectionId: number | undefined,
+ namespaceCollectionMap: NamespaceCollectionMap,
+ collectionMap: Map<number, Collection>,
+ libraryCollectionId: number | null,
+): CollectionPathSegment[] => {
+ if (spec.id === "transforms" && spec.virtualRootId != null) {
+ return [{ id: spec.virtualRootId, name: spec.virtualRootName?.() ?? "" }];
+ }
+ if (
+ spec.pathPrefixGroupId === "library" &&
+ libraryCollectionId != null &&
+ collectionId !== libraryCollectionId &&
+ collectionId != null &&
+ isCollectionInNamespace(collectionId, "snippets", namespaceCollectionMap)
+ ) {
+ const libraryCollection = collectionMap.get(libraryCollectionId);
+ if (libraryCollection) {
+ return [{ id: libraryCollection.id, name: libraryCollection.name }];
+ }
+ }
+ return [];
+};
+
+/**
+ * Get the spec by ID.
+ */
+const getSpecById = (id: string): DisplayGroupSpec | undefined => {
+ return displayGroupSpecs.find((spec) => spec.id === id);
+};
+
+/**
+ * Models that are children of tables (get their collection from a parent table)
+ */
+const TABLE_CHILD_MODELS: Set<RemoteSyncEntityModel> = new Set([
+ "field",
+ "segment",
+ "measure",
+]);
+
+/**
+ * Check if a model type is a child of a table (field, segment, or measure)
+ */
+export const isTableChildModel = (model: RemoteSyncEntityModel): boolean => {
+ return TABLE_CHILD_MODELS.has(model);
+};
+
+/** A table group with optional entity (when table itself is dirty) and its nested children */
+export type TableGroup = {
+ tableId: number;
+ tableName: string;
+ table?: RemoteSyncEntity;
+ children: RemoteSyncEntity[];
+};
+
+/** A collection group with display metadata */
+export type CollectionGroup = {
+ pathSegments: CollectionPathSegment[];
+ collectionId: number | undefined;
+ collectionEntity: RemoteSyncEntity | undefined;
+ tableGroups: TableGroup[];
+ items: RemoteSyncEntity[];
+ spec: DisplayGroupSpec;
+ isTransformsRoot: boolean;
+};
+
+/** Order for sorting entities by sync status */
+const SYNC_STATUS_ORDER: RemoteSyncEntityStatus[] = [
+ "create",
+ "update",
+ "touch",
+ "delete",
+];
+
+/**
+ * Sort entities by sync status (create, update, touch, delete) then by name.
+ * Returns a new sorted array.
+ */
+const sortEntitiesByStatus = (
+ items: RemoteSyncEntity[],
+): RemoteSyncEntity[] => {
+ return [...items].sort((a, b) => {
+ const statusOrderA = SYNC_STATUS_ORDER.indexOf(a.sync_status);
+ const statusOrderB = SYNC_STATUS_ORDER.indexOf(b.sync_status);
+ if (statusOrderA !== statusOrderB) {
+ return statusOrderA - statusOrderB;
+ }
+ return a.name.localeCompare(b.name);
+ });
+};
+
+/**
+ * Build table groups from a list of entities.
+ * Groups table children under their parent tables, creating groups for both
+ * dirty tables and orphan children (children whose parent table is not dirty).
+ */
+const buildTableGroups = (items: RemoteSyncEntity[]): TableGroup[] => {
+ const tables = items.filter((item) => item.model === "table");
+ const tableChildren = items.filter((item) => isTableChildModel(item.model));
+ const childrenByTableId = _.groupBy(tableChildren, (e) => e.table_id || 0);
+ const tableIds = new Set(tables.map((tbl) => tbl.id));
+
+ const dirtyTableGroups: TableGroup[] = tables.map((table) => ({
+ tableId: table.id,
+ tableName: table.name,
+ table,
+ children: sortEntitiesByStatus(childrenByTableId[table.id] || []),
+ }));
+
+ const orphanTableGroups: TableGroup[] = Object.entries(childrenByTableId)
+ .filter(([tableId]) => !tableIds.has(Number(tableId)))
+ .map(([tableId, children]) => ({
+ tableId: Number(tableId),
+ tableName: children[0]?.table_name ?? t`Unknown table`,
+ children: sortEntitiesByStatus(children),
+ }));
+
+ return [...dirtyTableGroups, ...orphanTableGroups].sort((a, b) =>
+ a.tableName.localeCompare(b.tableName),
+ );
+};
+
+/**
+ * Filter items to get non-table, non-table-child entities.
+ * Excludes the collection entity itself and the transforms root if applicable.
+ */
+const filterOtherItems = (
+ items: RemoteSyncEntity[],
+ collectionId: string,
+ isTransformsRoot: boolean,
+): RemoteSyncEntity[] => {
+ return items.filter((item) => {
+ if (
+ isTransformsRoot &&
+ item.model === "collection" &&
+ item.id === TRANSFORMS_ROOT_ID
+ ) {
+ return false;
+ }
+ return (
+ item.model !== "table" &&
+ !isTableChildModel(item.model) &&
+ !(item.model === "collection" && item.id === Number(collectionId))
+ );
+ });
+};
+
+/** Parameters for building a collection group */
+type BuildCollectionGroupParams = {
+ collectionId: string;
+ items: RemoteSyncEntity[];
+ transformsRootEntity: RemoteSyncEntity | undefined;
+ namespaceCollectionMap: NamespaceCollectionMap;
+ collectionMap: Map<number, Collection>;
+ libraryCollectionId: number | null;
+ getCollectionPathSegments: (
+ collectionId: number | undefined,
+ collectionMap: Map<number, Collection>,
+ ) => CollectionPathSegment[];
+};
+
+/**
+ * Build a single collection group from a collection ID and its items.
+ */
+const buildCollectionGroup = ({
+ collectionId,
+ items,
+ transformsRootEntity,
+ namespaceCollectionMap,
+ collectionMap,
+ libraryCollectionId,
+ getCollectionPathSegments,
+}: BuildCollectionGroupParams): CollectionGroup => {
+ const isTransformsRoot = collectionId === "transforms-root";
+ const collectionEntity = isTransformsRoot
+ ? transformsRootEntity
+ : items.find(
+ (item) =>
+ item.model === "collection" && item.id === Number(collectionId),
+ );
+ const firstItem = items[0];
+ const groupSpec = firstItem
+ ? getSpecForEntity(firstItem, namespaceCollectionMap)
+ : getSpecById("default")!;
+ const tableGroups = buildTableGroups(items);
+ const otherItems = filterOtherItems(items, collectionId, isTransformsRoot);
+ const numericCollectionId = isTransformsRoot
+ ? TRANSFORMS_ROOT_ID
+ : Number(collectionId) || undefined;
+
+ let pathSegments = isTransformsRoot
+ ? [{ id: TRANSFORMS_ROOT_ID, name: t`Transforms` }]
+ : getCollectionPathSegments(numericCollectionId, collectionMap);
+ const prefixSegments = getPathPrefixSegments(
+ groupSpec,
+ numericCollectionId,
+ namespaceCollectionMap,
+ collectionMap,
+ libraryCollectionId,
+ );
+ if (prefixSegments.length > 0 && !isTransformsRoot) {
+ pathSegments = [...prefixSegments, ...pathSegments];
+ }
+
+ return {
+ pathSegments,
+ collectionId: numericCollectionId,
+ collectionEntity,
+ tableGroups,
+ items: sortEntitiesByStatus(otherItems),
+ spec: groupSpec,
+ isTransformsRoot,
+ };
+};
+
+/** Parameters for grouping entities into collection groups */
+type GroupEntitiesParams = {
+ entities: RemoteSyncEntity[];
+ transformsRootEntity: RemoteSyncEntity | undefined;
+ namespaceCollectionMap: NamespaceCollectionMap;
+ collectionMap: Map<number, Collection>;
+ libraryCollectionId: number | null;
+ getCollectionPathSegments: (
+ collectionId: number | undefined,
+ collectionMap: Map<number, Collection>,
+ ) => CollectionPathSegment[];
+};
+
+/**
+ * Group entities by collection and build collection groups.
+ * This is the main orchestration function for the changes view grouping logic.
+ */
+export const groupEntitiesByCollection = ({
+ entities,
+ transformsRootEntity,
+ namespaceCollectionMap,
+ collectionMap,
+ libraryCollectionId,
+ getCollectionPathSegments,
+}: GroupEntitiesParams): CollectionGroup[] => {
+ const transformsRootExists = transformsRootEntity != null;
+ const byCollection = _.groupBy(entities, (e) => {
+ const spec = getSpecForEntity(e, namespaceCollectionMap);
+ const { groupKey } = getGroupKeyInfo(
+ e,
+ spec,
+ libraryCollectionId,
+ transformsRootExists,
+ );
+ return groupKey;
+ });
+
+ return Object.entries(byCollection)
+ .map(([collectionId, items]) =>
+ buildCollectionGroup({
+ collectionId,
+ items,
+ transformsRootEntity,
+ namespaceCollectionMap,
+ collectionMap,
+ libraryCollectionId,
+ getCollectionPathSegments,
+ }),
+ )
+ .sort((a, b) =>
+ a.pathSegments
+ .map((s) => s.name)
+ .join(" / ")
+ .localeCompare(b.pathSegments.map((s) => s.name).join(" / ")),
+ );
+};
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
index fe4ed106413..7ed8c49a847 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
@@ -4,6 +4,7 @@ import { useListCollectionsTreeQuery } from "metabase/api";
import { getAllDescendantIds } from "metabase/common/components/tree/utils";
import { useSetting } from "metabase/common/hooks";
import { buildCollectionTree } from "metabase/entities/collections";
+import { TRANSFORMS_ROOT_ID } from "metabase-enterprise/remote_sync/utils";
import { useGitSyncVisible } from "./use-git-sync-visible";
import { useRemoteSyncDirtyState } from "./use-remote-sync-dirty-state";
@@ -20,8 +21,24 @@ export function useHasTransformDirtyChanges(): boolean {
);
return useMemo(() => {
- // Only show if git sync is visible and transforms setting is enabled
- if (!isGitSyncVisible || !transformsSetting || !isDirty) {
+ // Only show if git sync is visible and there are dirty entities
+ if (!isGitSyncVisible || !isDirty) {
+ return false;
+ }
+
+ // Check for Transforms root collection (sentinel value id=-1)
+ // This needs to show even when transforms setting is off (to show pending delete)
+ const hasTransformsRootChange = dirty.some(
+ (entity) =>
+ entity.model === "collection" && entity.id === TRANSFORMS_ROOT_ID,
+ );
+
+ if (hasTransformsRootChange) {
+ return true;
+ }
+
+ // For other transform-related changes, only show if transforms setting is enabled
+ if (!transformsSetting) {
return false;
}
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
index 6fa5c3d9caa..9b10248d049 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
@@ -7,6 +7,7 @@ import {
} from "__support__/server-mocks";
import { mockSettings } from "__support__/settings";
import { renderHookWithProviders, waitFor } from "__support__/ui";
+import { TRANSFORMS_ROOT_ID } from "metabase-enterprise/remote_sync/utils";
import type { Collection, RemoteSyncEntity } from "metabase-types/api";
import {
createMockCollection,
@@ -191,4 +192,56 @@ describe("useHasTransformDirtyChanges", () => {
expect(result.current).toBe(false);
});
});
+
+ it("returns true when Transforms root collection (id=-1) is dirty with create status", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ sync_status: "create",
+ });
+ const { result } = setup({
+ collections: [createMockTransformsCollection()],
+ dirty: [transformsRootEntity],
+ });
+
+ await waitFor(() => {
+ expect(result.current).toBe(true);
+ });
+ });
+
+ it("returns true when Transforms root collection (id=-1) is dirty with delete status", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ sync_status: "delete",
+ });
+ const { result } = setup({
+ collections: [createMockTransformsCollection()],
+ dirty: [transformsRootEntity],
+ });
+
+ await waitFor(() => {
+ expect(result.current).toBe(true);
+ });
+ });
+
+ it("returns true when Transforms root collection (id=-1) is dirty even when transforms setting is disabled", async () => {
+ const transformsRootEntity = createMockRemoteSyncEntity({
+ id: TRANSFORMS_ROOT_ID,
+ name: "Transforms",
+ model: "collection",
+ sync_status: "delete",
+ });
+ const { result } = setup({
+ isTransformsSyncEnabled: false,
+ collections: [],
+ dirty: [transformsRootEntity],
+ });
+
+ await waitFor(() => {
+ expect(result.current).toBe(true);
+ });
+ });
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
index 56833cb08d3..97bc54da993 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
@@ -2,17 +2,16 @@ import { t } from "ttag";
import type { ColorName } from "metabase/lib/colors/types";
import type { IconName } from "metabase/ui";
-import type {
- Collection,
- CollectionId,
- RemoteSyncEntityModel,
- RemoteSyncEntityStatus,
-} from "metabase-types/api";
-
-export type CollectionPathSegment = {
- id: CollectionId;
- name: string;
-};
+import type { Collection, RemoteSyncEntityStatus } from "metabase-types/api";
+
+import type { CollectionPathSegment } from "./displayGroups";
+
+// Re-export from displayGroups for backwards compatibility
+export {
+ TRANSFORMS_ROOT_ID,
+ isTableChildModel,
+ type CollectionPathSegment,
+} from "./displayGroups";
type ErrorData = {
message?: string;
@@ -177,19 +176,3 @@ export const getCollectionPathSegments = (
segments.push({ id: collection.id, name: collection.name });
return segments;
};
-
-/**
- * Models that are children of tables (get their collection from a parent table)
- */
-const TABLE_CHILD_MODELS: Set<RemoteSyncEntityModel> = new Set([
- "field",
- "segment",
- "measure",
-]);
-
-/**
- * Check if a model type is a child of a table (field, segment, or measure)
- */
-export const isTableChildModel = (model: RemoteSyncEntityModel): boolean => {
- return TABLE_CHILD_MODELS.has(model);
-};