The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix: transforms collections set setting on import (#69852)
Edward Paget7d2d6571cb0f
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 50593fcc584..4f1fcb41dcb 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -24,14 +24,29 @@(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))(defn- snapshot-has-transforms?- "Checks if the snapshot contains any Transform, TransformTag, or PythonLibrary entities.- Used to auto-enable remote-sync-transforms setting during import.+ "Checks if the snapshot contains any Transform, PythonLibrary entities,+ non-built-in TransformTags, or transforms-namespace collections.+ Used to auto-enable/disable remote-sync-transforms setting during import.- Uses the ingestable to list all entities and checks their :model metadata."+ Uses the ingestable to list all entities and checks their :model metadata,+ then also checks if any Collection entities have namespace=transforms.+ Built-in TransformTags are excluded since they are system-created and always present."[ingestable](let [serdes-paths (serialization/ingest-list ingestable)models-present (spec/models-in-import serdes-paths)]- (some spec/transform-models models-present)))+ (or (some spec/transform-models models-present)+ ;; Check for non-built-in TransformTags+ (when (contains? models-present "TransformTag")+ (some (fn [path]+ (when (= "TransformTag" (:model (last path)))+ (let [entity (serialization/ingest-one ingestable path)]+ (nil? (:built_in_type entity)))))+ serdes-paths))+ (some (fn [path]+ (when (= "Collection" (:model (last path)))+ (let [entity (serialization/ingest-one ingestable path)]+ (spec/transforms-namespace-collection? entity))))+ serdes-paths))))(defn- sync-objects!"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and@@ -170,8 +185,25 @@(let [ingest-list (serialization/ingest-list ingestable)imported-data (spec/extract-imported-entities ingest-list)models-present (spec/models-in-import ingest-list)+ ;; Extract namespace info from imported Collection entities+ import-ns-info+ (reduce (fn [acc path]+ (if (= "Collection" (:model (last path)))+ (let [entity (serialization/ingest-one ingestable path)]+ (if-let [ns (some-> (:namespace entity) keyword name)]+ (-> acc+ (update :namespaces conj ns)+ (update-in [:entity-ids ns] (fnil conj #{}) (:entity_id entity)))+ acc))+ acc))+ {:namespaces #{} :entity-ids {}}+ ingest-list)+ import-namespace-collections (:namespaces import-ns-info);; TODO (epaget 2026-02-02) -- entity-id conflict checking (detect unsynced local entities with matching entity_ids)- feature-conflicts (spec/check-feature-conflicts models-present)+ feature-conflicts (spec/check-feature-conflicts models-present import-namespace-collections)+ ns-coll-conflicts (spec/check-namespace-collection-conflicts+ import-namespace-collections+ (:entity-ids import-ns-info))library-conflict (when-let [local-library (t2/select-one :model/Collection :type collection/library-collection-type)](when (and first-import?(contains? (get-in imported-data [:by-entity-id "Collection"] #{})@@ -184,6 +216,7 @@:message "Import contains Library but local instance has an unsynced Library collection"}))all-conflicts (concatfeature-conflicts+ ns-coll-conflicts(when library-conflict [library-conflict]))]{:conflicts (vec all-conflicts):summary (into #{} (map :category) all-conflicts)}))@@ -243,6 +276,10 @@(t2/with-transaction [_conn](remove-unsynced! (spec/all-syncable-collection-ids) imported-data)(sync-objects! sync-timestamp imported-data))+ (when (and (not has-transforms?)+ (settings/remote-sync-transforms))+ (log/info "No transforms in remote source, disabling remote-sync-transforms setting")+ (settings/remote-sync-transforms! false))(remote-sync.task/update-progress! task-id 0.95)(remote-sync.task/set-version!task-iddiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex b8705b2f526..4201d68b85f 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -444,6 +444,14 @@:library-synced "Snippets"(str/capitalize (name setting-kw))))+(defn- setting->namespace+ "Converts a setting keyword to the corresponding collection namespace keyword, or nil."+ [setting-kw]+ (case setting-kw+ :remote-sync-transforms :transforms+ :library-synced :snippets+ nil))+(defn models-for-setting"Returns set of model-type strings for specs with the given `:enabled?` setting."[setting-kw]@@ -454,8 +462,10 @@(def transform-models"Models that indicate transforms content in a snapshot.- Derived from specs with `:enabled? :remote-sync-transforms`."- (disj (models-for-setting :remote-sync-transforms) :model/TransformTag))+ Derived from specs with `:enabled? :remote-sync-transforms`.+ Excludes TransformTag since built-in tags are always present and don't indicate+ user transform content. Non-built-in TransformTags are checked separately."+ (disj (models-for-setting :remote-sync-transforms) "TransformTag"))(defn models-in-import"Returns set of model-type strings present in the import.@@ -496,7 +506,9 @@(defn- has-unsynced-entities-for-feature?"Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.- Excludes built-in TransformTags from the count since they are system-created and not user data."+ Excludes built-in TransformTags and the built-in PythonLibrary from the count since they are+ system-created and not user data. Namespace collections are not checked here because they are+ organizational containers, not user data that would be lost on import."[specs-for-feature](some (fn [[_ spec]](let [model-key (:model-key spec)@@ -515,13 +527,19 @@"Checks if import contains models that conflict with existing local entities that are NOT already remote synced.Derives conflict categories from specs with keyword `:enabled?` values (optional features).Only triggers conflict when local has unsynced entities AND import has matching entities.- If local entities are already synced, dirty tracking handles conflicts instead."- [models-present]+ If local entities are already synced, dirty tracking handles conflicts instead.++ `import-namespace-collections` is a set of namespace strings (e.g., #{\"transforms\" \"snippets\"})+ found in Collection entities in the import."+ [models-present import-namespace-collections](let [feature-groups (optional-feature-specs)](into [](for [[setting-kw specs-for-feature] feature-groups- :let [feature-model-types (into #{} (map (fn [[_ s]] (:model-type s))) specs-for-feature)]- :when (some feature-model-types models-present)+ :let [feature-model-types (into #{} (map (fn [[_ s]] (:model-type s))) specs-for-feature)+ feature-namespace (setting->namespace setting-kw)]+ :when (or (some feature-model-types models-present)+ (and feature-namespace+ (contains? import-namespace-collections (name feature-namespace)))):when (has-unsynced-entities-for-feature? specs-for-feature):let [category (setting->category setting-kw)]]{:type (keyword (str (u/lower-case-en category) "-conflict"))@@ -529,6 +547,41 @@:message (format "Import contains %s but local instance has unsynced %s"category category)}))))+(defn check-namespace-collection-conflicts+ "Checks if import contains namespace collections (transforms/snippets) that conflict with local+ unsynced namespace collections. Only flags a conflict when the local collections were NOT created+ by the sync system (i.e., have no matching entity_id in the import).++ `import-namespace-collections` is a set of namespace strings found in the import.+ `import-ns-collection-entity-ids` is a map of namespace string -> set of entity_ids from the import."+ [import-namespace-collections import-ns-collection-entity-ids]+ (let [ns-configs [{:ns-name "transforms" :setting-kw :remote-sync-transforms :category "Transforms"}+ {:ns-name "snippets" :setting-kw :library-synced :category "Snippets"}]]+ (into []+ (for [{:keys [ns-name setting-kw category]} ns-configs+ :when (contains? import-namespace-collections ns-name)+ :let [setting-enabled? (cond+ (= setting-kw :library-synced) (rs-settings/library-is-remote-synced?)+ (keyword? setting-kw) (boolean (setting/get-value-of-type :boolean setting-kw))+ :else false)]+ :when (not setting-enabled?)+ :let [local-ns-colls (t2/select [:model/Collection :id :entity_id] :namespace ns-name)+ import-eids (get import-ns-collection-entity-ids ns-name #{})+ ;; Only consider local collections that are NOT in the import (truly local-only)+ ;; and NOT tracked in RemoteSyncObject+ unsynced-local (remove+ (fn [coll]+ (or (contains? import-eids (:entity_id coll))+ (t2/exists? :model/RemoteSyncObject+ :model_type "Collection"+ :model_id (:id coll))))+ local-ns-colls)]+ :when (seq unsynced-local)]+ {:type (keyword (str (u/lower-case-en category) "-conflict"))+ :category category+ :message (format "Import contains %s but local instance has unsynced %s namespace collections"+ category category)}))))+;;; ------------------------------------------------ Eligibility Checking ----------------------------------------------(defn transforms-namespace-collection?diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 7f0ec11050e..b73a1a7c72c 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -1115,8 +1115,8 @@ serdes/meta:(is (false? (remote-sync.settings/remote-sync-transforms))"remote-sync-transforms should remain disabled when no transforms in remote")))))))))-(deftest import!-does-not-disable-transforms-setting-when-already-enabled-test- (testing "import! does not modify remote-sync-transforms setting when it's already enabled"+(deftest import!-disables-transforms-setting-when-no-transforms-in-branch-test+ (testing "import! disables remote-sync-transforms setting when importing a branch without transforms"(mt/with-model-cleanup [:model/RemoteSyncTask](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})](mt/with-temporary-setting-values [remote-sync-transforms true]@@ -1128,8 +1128,8 @@ serdes/meta:"remote-sync-transforms should be initially enabled")(let [result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- (is (true? (remote-sync.settings/remote-sync-transforms))- "remote-sync-transforms should remain enabled even when no transforms in remote")))))))))+ (is (false? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should be disabled when no transforms in remote")))))))))(deftest import!-includes-all-optional-paths-regardless-of-settings-test(testing "import! always includes all optional paths (transforms, python-libraries, snippets)"@@ -1318,3 +1318,90 @@ serdes/meta:(is (= :conflict (:status result)))(is (contains? (:conflicts result) "Snippets"))(is (some #(= :snippets-conflict (:type %)) (:conflict-details result))))))))++(deftest transforms-namespace-collection-conflict-test+ (testing "import with transforms-namespace Collection + local unsynced transforms-namespace Collection = conflict"+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection _ {:name "Local Transforms" :namespace "transforms"}]+ (let [test-files {"main" {"collections/txns-coll-_/txns-coll.yaml"+ (test-helpers/generate-collection-yaml "txns-coll-xxxxxxxxx" "Txns Coll")+ "collections/txns-ns-coll-_/txns-ns-coll.yaml"+ (test-helpers/generate-collection-yaml "txns-ns-coll-xxxxxxx" "Remote Transforms"+ :namespace "transforms")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (contains? (:conflicts result) "Transforms"))+ (is (some #(= :transforms-conflict (:type %)) (:conflict-details result)))))))))++(deftest snippets-namespace-collection-conflict-test+ (testing "import with snippets-namespace Collection + local unsynced snippets-namespace Collection = conflict"+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection _ {:name "Local Snippets" :namespace "snippets"}]+ (let [test-files {"main" {"collections/snip-coll-_/snip-coll.yaml"+ (test-helpers/generate-collection-yaml "snip-coll-xxxxxxxxx" "Snip Coll")+ "collections/snip-ns-coll-_/snip-ns-coll.yaml"+ (test-helpers/generate-collection-yaml "snip-ns-coll-xxxxxxx" "Remote Snippets"+ :namespace "snippets")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (contains? (:conflicts result) "Snippets"))+ (is (some #(= :snippets-conflict (:type %)) (:conflict-details result)))))))))++(deftest no-conflict-when-namespace-collections-synced-test+ (testing "no conflict when local namespace collections are fully tracked in RemoteSyncObject"+ (mt/with-model-cleanup [:model/Collection :model/RemoteSyncObject :model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection coll {:name "Local Transforms" :namespace "transforms"+ :entity_id "syncd-ns-coll-xxxxxxx"}+ :model/RemoteSyncObject _ {:model_type "Collection"+ :model_id (:id coll)+ :model_name "Local Transforms"+ :status "synced"+ :status_changed_at (t/offset-date-time)}]+ (doseq [coll-id (t2/select-pks-vec :model/Collection :namespace [:in ["transforms" "snippets"]])+ :when (not= coll-id (:id coll))]+ (when-not (t2/exists? :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection"+ :model_id coll-id+ :model_name "synced"+ :status "synced"+ :status_changed_at (t/offset-date-time)})))+ (let [test-files {"main" {"collections/syncd-coll-_/syncd-coll.yaml"+ (test-helpers/generate-collection-yaml "syncd-coll-xxxxxxxxx" "Syncd Coll")+ "collections/syncd-ns-coll-_/syncd-ns-coll.yaml"+ (test-helpers/generate-collection-yaml "syncd-ns-coll-xxxxxxx" "Remote Transforms"+ :namespace "transforms")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (not= :conflict (:status result))+ "Should not detect a conflict when all local namespace collections are synced")+ (is (nil? (seq (:conflict-details result))))))))))++(deftest no-conflict-when-no-local-namespace-collections-test+ (testing "no conflict when import has namespace collections but local has none (or all are synced)"+ (mt/with-model-cleanup [:model/Collection :model/RemoteSyncObject :model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (doseq [coll-id (t2/select-pks-vec :model/Collection :namespace [:in ["transforms" "snippets"]])]+ (when-not (t2/exists? :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection"+ :model_id coll-id+ :model_name "pre-existing"+ :status "synced"+ :status_changed_at (t/offset-date-time)})))+ (let [test-files {"main" {"collections/noloc-coll-_/noloc-coll.yaml"+ (test-helpers/generate-collection-yaml "noloc-coll-xxxxxxxxx" "Noloc Coll")+ "collections/noloc-tx-coll-_/noloc-tx-coll.yaml"+ (test-helpers/generate-collection-yaml "noloc-tx-coll-xxxxxx" "Remote Transforms"+ :namespace "transforms")+ "collections/noloc-sn-coll-_/noloc-sn-coll.yaml"+ (test-helpers/generate-collection-yaml "noloc-sn-coll-xxxxxx" "Remote Snippets"+ :namespace "snippets")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (not= :conflict (:status result))+ "Should not detect a conflict when local has no unsynced namespace collections")+ (is (nil? (seq (:conflict-details result)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 059e1f440e8..0a69cfa3eac 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -10,8 +10,9 @@(defn generate-collection-yaml"Generate YAML content for a collection with the given `entity-id` and `name`.- Optionally accepts `:parent-id` for nested collections."- [entity-id name & {:keys [parent-id]}]+ Optionally accepts `:parent-id` for nested collections and `:namespace` for+ namespace collections (e.g., \"transforms\" or \"snippets\")."+ [entity-id name & {:keys [parent-id namespace]}](format "name: %sdescription: nullentity_id: %s@@ -21,7 +22,7 @@ archived: falsetype: nullparent_id: %spersonal_owner_id: null-namespace: null+namespace: %sauthority_level: nullserdes/meta:- id: %s@@ -32,7 +33,7 @@ archived_directly: nullis_sample: false"name entity-id (str/replace (u/lower-case-en name) #"\s+" "_")- (or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))+ (or parent-id "null") (or namespace "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))(defn generate-v57-collection-yaml"Generate YAML content for a collection in v57 format. In v57, remote-synced collections@@ -296,25 +297,29 @@ width: fixed(t2/insert! :model/PythonLibrary builtin-python-library)))(defn clean-optional-feature-models- "Test fixture that cleans Transform, TransformTag, and PythonLibrary tables to prevent- conflict detection during first-import tests. Preserves built-in TransformTags and- recreates the built-in common.py PythonLibrary after cleanup."+ "Test fixture that cleans Transform, TransformTag, PythonLibrary, and namespace collection+ tables to prevent conflict detection during first-import tests. Preserves built-in TransformTags+ and recreates the built-in common.py PythonLibrary after cleanup."[f](let [old-transforms (t2/select :model/Transform)old-tags (t2/select :model/TransformTag :built_in_type nil)- old-libs (t2/select :model/PythonLibrary)]+ old-libs (t2/select :model/PythonLibrary)+ old-ns-colls (t2/select :model/Collection :namespace [:in ["transforms" "snippets"]])](try(t2/delete! :model/TransformTag :built_in_type nil)(t2/delete! :model/Transform)(t2/delete! :model/PythonLibrary)+ (t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])(f)(finally(t2/delete! :model/TransformTag :built_in_type nil)(t2/delete! :model/Transform)(t2/delete! :model/PythonLibrary)+ (t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])(when (seq old-transforms) (t2/insert! :model/Transform old-transforms))(when (seq old-tags) (t2/insert! :model/TransformTag old-tags))(when (seq old-libs) (t2/insert! :model/PythonLibrary old-libs))+ (when (seq old-ns-colls) (t2/insert! :model/Collection old-ns-colls))(ensure-builtin-python-library!)))))(def clean-remote-sync-statediff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex e5dfa8ff6df..fb59edec0e9 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -16,6 +16,7 @@[environ.core :refer [env]][java-time.api :as t][metabase.analytics.prometheus :as analytics]+ [metabase.app-db.core :as app-db][metabase.config.core :as config][metabase.events.core :as events][metabase.internal-stats.core :as internal-stats]@@ -368,16 +369,23 @@(defn- write-cache-to-db!"Upsert a token status hash into the premium_features_token_cache table.- Uses insert-first to avoid TOCTOU race between instances."+ Uses update-first to avoid poisoning PostgreSQL transactions on duplicate key.+ Catches constraint violations from concurrent inserters and retries with an update."[token-hash result-hash]- (let [now (t/offset-date-time)]- (try- (t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash- :token_status_hash result-hash- :updated_at now})- (catch Exception _- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash- {:token_status_hash result-hash :updated_at now})))))+ (t2/with-connection [_conn (app-db/app-db)]+ (let [now (t/offset-date-time)+ updated (t2/update! :model/PremiumFeaturesCache :token_hash token-hash+ {:token_status_hash result-hash :updated_at now})]+ (when (zero? updated) ;; even though toucan2 returns 0 if we match a row but don't update it+ ;; we should always be updating this row with the timestamp if it's there.+ (try+ (t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash+ :token_status_hash result-hash+ :updated_at now})+ (catch Exception _e+ ;; Another instance inserted first — update instead.+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash+ {:token_status_hash result-hash :updated_at now})))))))(defn- clear-db-cache!"Delete all rows from the premium_features_token_cache table."