Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: transforms collections set setting on import (#69852)

Edward Paget7d2d6571cb0f
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 50593fcc584..4f1fcb41dcb 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -24,14 +24,29 @@
(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
(defn- snapshot-has-transforms?
- "Checks if the snapshot contains any Transform, TransformTag, or PythonLibrary entities.
- Used to auto-enable remote-sync-transforms setting during import.
+ "Checks if the snapshot contains any Transform, PythonLibrary entities,
+ non-built-in TransformTags, or transforms-namespace collections.
+ Used to auto-enable/disable remote-sync-transforms setting during import.
- Uses the ingestable to list all entities and checks their :model metadata."
+ Uses the ingestable to list all entities and checks their :model metadata,
+ then also checks if any Collection entities have namespace=transforms.
+ Built-in TransformTags are excluded since they are system-created and always present."
[ingestable]
(let [serdes-paths (serialization/ingest-list ingestable)
models-present (spec/models-in-import serdes-paths)]
- (some spec/transform-models models-present)))
+ (or (some spec/transform-models models-present)
+ ;; Check for non-built-in TransformTags
+ (when (contains? models-present "TransformTag")
+ (some (fn [path]
+ (when (= "TransformTag" (:model (last path)))
+ (let [entity (serialization/ingest-one ingestable path)]
+ (nil? (:built_in_type entity)))))
+ serdes-paths))
+ (some (fn [path]
+ (when (= "Collection" (:model (last path)))
+ (let [entity (serialization/ingest-one ingestable path)]
+ (spec/transforms-namespace-collection? entity))))
+ serdes-paths))))
(defn- sync-objects!
"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and
@@ -170,8 +185,25 @@
(let [ingest-list (serialization/ingest-list ingestable)
imported-data (spec/extract-imported-entities ingest-list)
models-present (spec/models-in-import ingest-list)
+ ;; Extract namespace info from imported Collection entities
+ import-ns-info
+ (reduce (fn [acc path]
+ (if (= "Collection" (:model (last path)))
+ (let [entity (serialization/ingest-one ingestable path)]
+ (if-let [ns (some-> (:namespace entity) keyword name)]
+ (-> acc
+ (update :namespaces conj ns)
+ (update-in [:entity-ids ns] (fnil conj #{}) (:entity_id entity)))
+ acc))
+ acc))
+ {:namespaces #{} :entity-ids {}}
+ ingest-list)
+ import-namespace-collections (:namespaces import-ns-info)
;; TODO (epaget 2026-02-02) -- entity-id conflict checking (detect unsynced local entities with matching entity_ids)
- feature-conflicts (spec/check-feature-conflicts models-present)
+ feature-conflicts (spec/check-feature-conflicts models-present import-namespace-collections)
+ ns-coll-conflicts (spec/check-namespace-collection-conflicts
+ import-namespace-collections
+ (:entity-ids import-ns-info))
library-conflict (when-let [local-library (t2/select-one :model/Collection :type collection/library-collection-type)]
(when (and first-import?
(contains? (get-in imported-data [:by-entity-id "Collection"] #{})
@@ -184,6 +216,7 @@
:message "Import contains Library but local instance has an unsynced Library collection"}))
all-conflicts (concat
feature-conflicts
+ ns-coll-conflicts
(when library-conflict [library-conflict]))]
{:conflicts (vec all-conflicts)
:summary (into #{} (map :category) all-conflicts)}))
@@ -243,6 +276,10 @@
(t2/with-transaction [_conn]
(remove-unsynced! (spec/all-syncable-collection-ids) imported-data)
(sync-objects! sync-timestamp imported-data))
+ (when (and (not has-transforms?)
+ (settings/remote-sync-transforms))
+ (log/info "No transforms in remote source, disabling remote-sync-transforms setting")
+ (settings/remote-sync-transforms! false))
(remote-sync.task/update-progress! task-id 0.95)
(remote-sync.task/set-version!
task-id
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index b8705b2f526..4201d68b85f 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -444,6 +444,14 @@
:library-synced "Snippets"
(str/capitalize (name setting-kw))))
+(defn- setting->namespace
+ "Converts a setting keyword to the corresponding collection namespace keyword, or nil."
+ [setting-kw]
+ (case setting-kw
+ :remote-sync-transforms :transforms
+ :library-synced :snippets
+ nil))
+
(defn models-for-setting
"Returns set of model-type strings for specs with the given `:enabled?` setting."
[setting-kw]
@@ -454,8 +462,10 @@
(def transform-models
"Models that indicate transforms content in a snapshot.
- Derived from specs with `:enabled? :remote-sync-transforms`."
- (disj (models-for-setting :remote-sync-transforms) :model/TransformTag))
+ Derived from specs with `:enabled? :remote-sync-transforms`.
+ Excludes TransformTag since built-in tags are always present and don't indicate
+ user transform content. Non-built-in TransformTags are checked separately."
+ (disj (models-for-setting :remote-sync-transforms) "TransformTag"))
(defn models-in-import
"Returns set of model-type strings present in the import.
@@ -496,7 +506,9 @@
(defn- has-unsynced-entities-for-feature?
"Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.
- Excludes built-in TransformTags from the count since they are system-created and not user data."
+ Excludes built-in TransformTags and the built-in PythonLibrary from the count since they are
+ system-created and not user data. Namespace collections are not checked here because they are
+ organizational containers, not user data that would be lost on import."
[specs-for-feature]
(some (fn [[_ spec]]
(let [model-key (:model-key spec)
@@ -515,13 +527,19 @@
"Checks if import contains models that conflict with existing local entities that are NOT already remote synced.
Derives conflict categories from specs with keyword `:enabled?` values (optional features).
Only triggers conflict when local has unsynced entities AND import has matching entities.
- If local entities are already synced, dirty tracking handles conflicts instead."
- [models-present]
+ If local entities are already synced, dirty tracking handles conflicts instead.
+
+ `import-namespace-collections` is a set of namespace strings (e.g., #{\"transforms\" \"snippets\"})
+ found in Collection entities in the import."
+ [models-present import-namespace-collections]
(let [feature-groups (optional-feature-specs)]
(into []
(for [[setting-kw specs-for-feature] feature-groups
- :let [feature-model-types (into #{} (map (fn [[_ s]] (:model-type s))) specs-for-feature)]
- :when (some feature-model-types models-present)
+ :let [feature-model-types (into #{} (map (fn [[_ s]] (:model-type s))) specs-for-feature)
+ feature-namespace (setting->namespace setting-kw)]
+ :when (or (some feature-model-types models-present)
+ (and feature-namespace
+ (contains? import-namespace-collections (name feature-namespace))))
:when (has-unsynced-entities-for-feature? specs-for-feature)
:let [category (setting->category setting-kw)]]
{:type (keyword (str (u/lower-case-en category) "-conflict"))
@@ -529,6 +547,41 @@
:message (format "Import contains %s but local instance has unsynced %s"
category category)}))))
+(defn check-namespace-collection-conflicts
+ "Checks if import contains namespace collections (transforms/snippets) that conflict with local
+ unsynced namespace collections. Only flags a conflict when the local collections were NOT created
+ by the sync system (i.e., have no matching entity_id in the import).
+
+ `import-namespace-collections` is a set of namespace strings found in the import.
+ `import-ns-collection-entity-ids` is a map of namespace string -> set of entity_ids from the import."
+ [import-namespace-collections import-ns-collection-entity-ids]
+ (let [ns-configs [{:ns-name "transforms" :setting-kw :remote-sync-transforms :category "Transforms"}
+ {:ns-name "snippets" :setting-kw :library-synced :category "Snippets"}]]
+ (into []
+ (for [{:keys [ns-name setting-kw category]} ns-configs
+ :when (contains? import-namespace-collections ns-name)
+ :let [setting-enabled? (cond
+ (= setting-kw :library-synced) (rs-settings/library-is-remote-synced?)
+ (keyword? setting-kw) (boolean (setting/get-value-of-type :boolean setting-kw))
+ :else false)]
+ :when (not setting-enabled?)
+ :let [local-ns-colls (t2/select [:model/Collection :id :entity_id] :namespace ns-name)
+ import-eids (get import-ns-collection-entity-ids ns-name #{})
+ ;; Only consider local collections that are NOT in the import (truly local-only)
+ ;; and NOT tracked in RemoteSyncObject
+ unsynced-local (remove
+ (fn [coll]
+ (or (contains? import-eids (:entity_id coll))
+ (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id (:id coll))))
+ local-ns-colls)]
+ :when (seq unsynced-local)]
+ {:type (keyword (str (u/lower-case-en category) "-conflict"))
+ :category category
+ :message (format "Import contains %s but local instance has unsynced %s namespace collections"
+ category category)}))))
+
;;; ------------------------------------------------ Eligibility Checking ----------------------------------------------
(defn transforms-namespace-collection?
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 7f0ec11050e..b73a1a7c72c 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -1115,8 +1115,8 @@ serdes/meta:
(is (false? (remote-sync.settings/remote-sync-transforms))
"remote-sync-transforms should remain disabled when no transforms in remote")))))))))
-(deftest import!-does-not-disable-transforms-setting-when-already-enabled-test
- (testing "import! does not modify remote-sync-transforms setting when it's already enabled"
+(deftest import!-disables-transforms-setting-when-no-transforms-in-branch-test
+ (testing "import! disables remote-sync-transforms setting when importing a branch without transforms"
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temporary-setting-values [remote-sync-transforms true]
@@ -1128,8 +1128,8 @@ serdes/meta:
"remote-sync-transforms should be initially enabled")
(let [result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- (is (true? (remote-sync.settings/remote-sync-transforms))
- "remote-sync-transforms should remain enabled even when no transforms in remote")))))))))
+ (is (false? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should be disabled when no transforms in remote")))))))))
(deftest import!-includes-all-optional-paths-regardless-of-settings-test
(testing "import! always includes all optional paths (transforms, python-libraries, snippets)"
@@ -1318,3 +1318,90 @@ serdes/meta:
(is (= :conflict (:status result)))
(is (contains? (:conflicts result) "Snippets"))
(is (some #(= :snippets-conflict (:type %)) (:conflict-details result))))))))
+
+(deftest transforms-namespace-collection-conflict-test
+ (testing "import with transforms-namespace Collection + local unsynced transforms-namespace Collection = conflict"
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection _ {:name "Local Transforms" :namespace "transforms"}]
+ (let [test-files {"main" {"collections/txns-coll-_/txns-coll.yaml"
+ (test-helpers/generate-collection-yaml "txns-coll-xxxxxxxxx" "Txns Coll")
+ "collections/txns-ns-coll-_/txns-ns-coll.yaml"
+ (test-helpers/generate-collection-yaml "txns-ns-coll-xxxxxxx" "Remote Transforms"
+ :namespace "transforms")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (contains? (:conflicts result) "Transforms"))
+ (is (some #(= :transforms-conflict (:type %)) (:conflict-details result)))))))))
+
+(deftest snippets-namespace-collection-conflict-test
+ (testing "import with snippets-namespace Collection + local unsynced snippets-namespace Collection = conflict"
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection _ {:name "Local Snippets" :namespace "snippets"}]
+ (let [test-files {"main" {"collections/snip-coll-_/snip-coll.yaml"
+ (test-helpers/generate-collection-yaml "snip-coll-xxxxxxxxx" "Snip Coll")
+ "collections/snip-ns-coll-_/snip-ns-coll.yaml"
+ (test-helpers/generate-collection-yaml "snip-ns-coll-xxxxxxx" "Remote Snippets"
+ :namespace "snippets")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (contains? (:conflicts result) "Snippets"))
+ (is (some #(= :snippets-conflict (:type %)) (:conflict-details result)))))))))
+
+(deftest no-conflict-when-namespace-collections-synced-test
+ (testing "no conflict when local namespace collections are fully tracked in RemoteSyncObject"
+ (mt/with-model-cleanup [:model/Collection :model/RemoteSyncObject :model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection coll {:name "Local Transforms" :namespace "transforms"
+ :entity_id "syncd-ns-coll-xxxxxxx"}
+ :model/RemoteSyncObject _ {:model_type "Collection"
+ :model_id (:id coll)
+ :model_name "Local Transforms"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)}]
+ (doseq [coll-id (t2/select-pks-vec :model/Collection :namespace [:in ["transforms" "snippets"]])
+ :when (not= coll-id (:id coll))]
+ (when-not (t2/exists? :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)
+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection"
+ :model_id coll-id
+ :model_name "synced"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)})))
+ (let [test-files {"main" {"collections/syncd-coll-_/syncd-coll.yaml"
+ (test-helpers/generate-collection-yaml "syncd-coll-xxxxxxxxx" "Syncd Coll")
+ "collections/syncd-ns-coll-_/syncd-ns-coll.yaml"
+ (test-helpers/generate-collection-yaml "syncd-ns-coll-xxxxxxx" "Remote Transforms"
+ :namespace "transforms")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (not= :conflict (:status result))
+ "Should not detect a conflict when all local namespace collections are synced")
+ (is (nil? (seq (:conflict-details result))))))))))
+
+(deftest no-conflict-when-no-local-namespace-collections-test
+ (testing "no conflict when import has namespace collections but local has none (or all are synced)"
+ (mt/with-model-cleanup [:model/Collection :model/RemoteSyncObject :model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (doseq [coll-id (t2/select-pks-vec :model/Collection :namespace [:in ["transforms" "snippets"]])]
+ (when-not (t2/exists? :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)
+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection"
+ :model_id coll-id
+ :model_name "pre-existing"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)})))
+ (let [test-files {"main" {"collections/noloc-coll-_/noloc-coll.yaml"
+ (test-helpers/generate-collection-yaml "noloc-coll-xxxxxxxxx" "Noloc Coll")
+ "collections/noloc-tx-coll-_/noloc-tx-coll.yaml"
+ (test-helpers/generate-collection-yaml "noloc-tx-coll-xxxxxx" "Remote Transforms"
+ :namespace "transforms")
+ "collections/noloc-sn-coll-_/noloc-sn-coll.yaml"
+ (test-helpers/generate-collection-yaml "noloc-sn-coll-xxxxxx" "Remote Snippets"
+ :namespace "snippets")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (not= :conflict (:status result))
+ "Should not detect a conflict when local has no unsynced namespace collections")
+ (is (nil? (seq (:conflict-details result)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 059e1f440e8..0a69cfa3eac 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -10,8 +10,9 @@
(defn generate-collection-yaml
"Generate YAML content for a collection with the given `entity-id` and `name`.
- Optionally accepts `:parent-id` for nested collections."
- [entity-id name & {:keys [parent-id]}]
+ Optionally accepts `:parent-id` for nested collections and `:namespace` for
+ namespace collections (e.g., \"transforms\" or \"snippets\")."
+ [entity-id name & {:keys [parent-id namespace]}]
(format "name: %s
description: null
entity_id: %s
@@ -21,7 +22,7 @@ archived: false
type: null
parent_id: %s
personal_owner_id: null
-namespace: null
+namespace: %s
authority_level: null
serdes/meta:
- id: %s
@@ -32,7 +33,7 @@ archived_directly: null
is_sample: false
"
name entity-id (str/replace (u/lower-case-en name) #"\s+" "_")
- (or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
+ (or parent-id "null") (or namespace "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
(defn generate-v57-collection-yaml
"Generate YAML content for a collection in v57 format. In v57, remote-synced collections
@@ -296,25 +297,29 @@ width: fixed
(t2/insert! :model/PythonLibrary builtin-python-library)))
(defn clean-optional-feature-models
- "Test fixture that cleans Transform, TransformTag, and PythonLibrary tables to prevent
- conflict detection during first-import tests. Preserves built-in TransformTags and
- recreates the built-in common.py PythonLibrary after cleanup."
+ "Test fixture that cleans Transform, TransformTag, PythonLibrary, and namespace collection
+ tables to prevent conflict detection during first-import tests. Preserves built-in TransformTags
+ and recreates the built-in common.py PythonLibrary after cleanup."
[f]
(let [old-transforms (t2/select :model/Transform)
old-tags (t2/select :model/TransformTag :built_in_type nil)
- old-libs (t2/select :model/PythonLibrary)]
+ old-libs (t2/select :model/PythonLibrary)
+ old-ns-colls (t2/select :model/Collection :namespace [:in ["transforms" "snippets"]])]
(try
(t2/delete! :model/TransformTag :built_in_type nil)
(t2/delete! :model/Transform)
(t2/delete! :model/PythonLibrary)
+ (t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])
(f)
(finally
(t2/delete! :model/TransformTag :built_in_type nil)
(t2/delete! :model/Transform)
(t2/delete! :model/PythonLibrary)
+ (t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])
(when (seq old-transforms) (t2/insert! :model/Transform old-transforms))
(when (seq old-tags) (t2/insert! :model/TransformTag old-tags))
(when (seq old-libs) (t2/insert! :model/PythonLibrary old-libs))
+ (when (seq old-ns-colls) (t2/insert! :model/Collection old-ns-colls))
(ensure-builtin-python-library!)))))
(def clean-remote-sync-state
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index e5dfa8ff6df..fb59edec0e9 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -16,6 +16,7 @@
[environ.core :refer [env]]
[java-time.api :as t]
[metabase.analytics.prometheus :as analytics]
+ [metabase.app-db.core :as app-db]
[metabase.config.core :as config]
[metabase.events.core :as events]
[metabase.internal-stats.core :as internal-stats]
@@ -368,16 +369,23 @@
(defn- write-cache-to-db!
"Upsert a token status hash into the premium_features_token_cache table.
- Uses insert-first to avoid TOCTOU race between instances."
+ Uses update-first to avoid poisoning PostgreSQL transactions on duplicate key.
+ Catches constraint violations from concurrent inserters and retries with an update."
[token-hash result-hash]
- (let [now (t/offset-date-time)]
- (try
- (t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash
- :token_status_hash result-hash
- :updated_at now})
- (catch Exception _
- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash
- {:token_status_hash result-hash :updated_at now})))))
+ (t2/with-connection [_conn (app-db/app-db)]
+ (let [now (t/offset-date-time)
+ updated (t2/update! :model/PremiumFeaturesCache :token_hash token-hash
+ {:token_status_hash result-hash :updated_at now})]
+ (when (zero? updated) ;; even though toucan2 returns 0 if we match a row but don't update it
+ ;; we should always be updating this row with the timestamp if it's there.
+ (try
+ (t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash
+ :token_status_hash result-hash
+ :updated_at now})
+ (catch Exception _e
+ ;; Another instance inserted first — update instead.
+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash
+ {:token_status_hash result-hash :updated_at now})))))))
(defn- clear-db-cache!
"Delete all rows from the premium_features_token_cache table."