Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Don't delete local transforms when remote has none (GHY-3900) (#75898)

Eric Normand7b274b46ac4d
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 1c58e2792d9..9f46c21d442 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -185,14 +185,15 @@
:details {:error-type (type e)}})))
(defn- get-conflicts
- "Detects conflicts that would prevent or complicate import.
- Returns a map with :conflicts (detailed list) and :summary (set of category names).
+ "Detects conflicts that would prevent or complicate import. Returns a map with two classes:
+ - :first-import-conflicts - feature/namespace/library conflicts that only block on the very first import
+ - :deletion-conflicts - unsynced local entities of all-or-nothing models (transforms) that the import
+ would wholesale-delete; these block on EVERY import, not just the first
Conflict types detected:
- - :entity-id-conflict - Items with existing entity IDs that are NOT already synced
- :library-conflict - First import only, local Library exists, import has Library
- - :transforms-not-enabled - Import has Transform/TransformTag/PythonLibrary, setting disabled
- - :snippets-without-library - Import has NativeQuerySnippet, Library not remote-synced"
+ - :transforms-conflict / :snippets-conflict - import has matching content but local has unsynced entities
+ - :*-conflict (deletion) - import lacks transforms/tags/libraries that exist locally and unsynced (GHY-3900)"
[ingestable first-import?]
(let [ingest-list (serialization/ingest-list ingestable)
imported-data (spec/extract-imported-entities ingest-list)
@@ -226,12 +227,12 @@
{:type :library-conflict
:category "Library"
:message "Import contains Library but local instance has an unsynced Library collection"}))
- all-conflicts (concat
- feature-conflicts
- ns-coll-conflicts
- (when library-conflict [library-conflict]))]
- {:conflicts (vec all-conflicts)
- :summary (into #{} (map :category) all-conflicts)}))
+ first-import-conflicts (concat
+ feature-conflicts
+ ns-coll-conflicts
+ (when library-conflict [library-conflict]))]
+ {:first-import-conflicts (vec first-import-conflicts)
+ :deletion-conflicts (spec/check-deletion-conflicts imported-data)}))
(defn- record-exported-paths!
"Records each exported entity's repo file path on its RemoteSyncObject row, so later renames and
@@ -375,7 +376,10 @@
"Imports and reloads Metabase entities from a remote snapshot.
Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:
- - :force? - forces import even when the snapshot version matches the last imported version
+ - :force? - forces import even when the snapshot version matches the last imported version or has first-import conflicts
+ - :force-deletion? - forces past deletion conflicts (unsynced local transforms the import would delete); defaults
+ to :force?. Kept separate so the setup/enable flow can force past version/dirty guards (force? true) while still
+ surfacing transform-deletion as a conflict the admin must confirm (force-deletion? false) — see GHY-3900.
- :merge? - perform a local-only 3-way merge (keep un-pushed local changes) instead of overwriting local
- :base-snapshot - the merge base (last synced version), required when :merge? is set
- :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs
@@ -387,7 +391,7 @@
Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may be
thrown during import and are caught and converted to error status maps."
- [^SourceSnapshot snapshot task-id & {:keys [force? merge? base-snapshot pre-task-branch]}]
+ [^SourceSnapshot snapshot task-id & {:keys [force? force-deletion? merge? base-snapshot pre-task-branch]}]
(when (branch-changed-since-scheduling? pre-task-branch)
(log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"
pre-task-branch (settings/remote-sync-branch))
@@ -404,15 +408,21 @@
(let [snapshot-version (source.p/version snapshot)
last-imported-version (remote-sync.task/last-version)
first-import? (nil? last-imported-version)
+ ;; force-deletion? is optional; when a caller doesn't pass it, deletion follows force?
+ force-deletion? (if (nil? force-deletion?) force? force-deletion?)
path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)
base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
- {:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)]
+ {:keys [first-import-conflicts deletion-conflicts]} (get-conflicts base-ingestable first-import?)
+ ;; First-import conflicts only block on the first import; deletion conflicts block on every
+ ;; import (an already-configured instance must not silently delete unsynced transforms).
+ blocking-conflicts (vec (concat (when (and first-import? (not force?)) first-import-conflicts)
+ (when (not force-deletion?) deletion-conflicts)))]
(cond
- (and first-import? (not force?) (seq conflicts))
+ (seq blocking-conflicts)
(u/prog1 {:status :conflict
:version (source.p/version snapshot)
- :conflicts summary ; Keep backward compatibility: return set of category names
- :conflict-details conflicts ; New: detailed conflict info
+ :conflicts (into #{} (map :category) blocking-conflicts) ; Backward compat: set of category names
+ :conflict-details blocking-conflicts ; Detailed conflict info
:message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)}
(log/infof (:message <>)))
@@ -1004,7 +1014,7 @@
Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if there
are unsaved changes and neither force? nor merge? is set."
- [branch force? import-args & {:keys [on-success merge?]}]
+ [branch force? import-args & {:keys [on-success merge? force-deletion?]}]
(guards/ensure-no-active-task!)
(let [pre-task-branch (settings/remote-sync-branch)
source (source/source-from-settings branch)
@@ -1028,6 +1038,7 @@
(import! snapshot task-id
(assoc import-args
:force? force?
+ :force-deletion? force-deletion?
:merge? merge?
:base-snapshot base-snapshot
:pre-task-branch pre-task-branch)))
@@ -1129,6 +1140,8 @@
(when (str/blank? (setting/get :remote-sync-branch))
(setting/set! :remote-sync-branch (source.p/default-branch (source/source-from-settings))))
(when (= :read-only (settings/remote-sync-type))
- (:id (async-import! (settings/remote-sync-branch) true {}))))
+ ;; force? true bypasses the version/dirty guards for setup, but force-deletion? false keeps unsynced
+ ;; local transforms from being silently destroyed — they surface as a conflict instead (GHY-3900).
+ (:id (async-import! (settings/remote-sync-branch) true {} :force-deletion? false))))
(u/prog1 nil
(collection/clear-remote-synced-collection!))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 3fbf6bfeebe..744db12c134 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -594,6 +594,51 @@
:message (format "Import contains %s but local instance has unsynced %s namespace collections"
category category)}))))
+(defn- removal-condition-clauses
+ "Converts a spec's removal-conditions map into HoneySQL where-fragments, matching the semantics
+ [[metabase-enterprise.remote-sync.impl/build-entity-id-where-clause]] uses: an :entity_id entry
+ whose value is an [op value] pair becomes [op :entity_id value]; any other entry becomes [:= key value]."
+ [removal-conds]
+ (for [[k v] removal-conds]
+ (if (and (= k :entity_id) (vector? v))
+ (let [[op value] v] [op :entity_id value])
+ [:= k v])))
+
+(defn check-deletion-conflicts
+ "Detects local entities of all-or-nothing models (specs with :all-on-setting-disable) that an import
+ would wholesale-delete because they are absent from it, but which hold unsaved local work — i.e. they
+ have no RemoteSyncObject in 'synced' status. Already-synced entities are excluded: their removal is a
+ normal reconcile against the remote source of truth, not data loss.
+
+ Takes imported-data from [[extract-imported-entities]]. Returns a vector of conflict maps
+ ({:type :category :message}), one per affected model type (categories repeat across the transform models;
+ callers dedupe via the conflict summary)."
+ [{:keys [by-entity-id]}]
+ (into []
+ (for [[model-key spec] (specs-for-deletion)
+ :let [setting-kw (get-in spec [:removal :all-on-setting-disable])]
+ :when setting-kw
+ :let [model-type (:model-type spec)
+ imported-ids (get by-entity-id model-type #{})
+ conds (cond-> []
+ (seq imported-ids) (conj [:not-in :entity_id imported-ids])
+ :always (into (removal-condition-clauses (removal-conditions spec))))
+ candidates (if (seq conds)
+ (t2/select [model-key :id] {:where (into [:and] conds)})
+ (t2/select [model-key :id]))
+ n-unsynced (count (remove (fn [{:keys [id]}]
+ (t2/exists? :model/RemoteSyncObject
+ :model_type model-type
+ :model_id id
+ :status "synced"))
+ candidates))]
+ :when (pos? n-unsynced)
+ :let [category (setting->category setting-kw)]]
+ {:type (keyword (str (u/lower-case-en category) "-conflict"))
+ :category category
+ :message (format "Import would delete %d unsynced local %s %s"
+ n-unsynced category (if (= 1 n-unsynced) "entity" "entities"))})))
+
;;; ------------------------------------------------ Eligibility Checking ----------------------------------------------
(defn transforms-namespace-collection?
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 023c59013f1..80c3edd88b9 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -560,6 +560,23 @@
(is @import-called?
"Should call async-import! in read-only mode")))))))))
+(deftest finish-remote-config!-does-not-force-transform-deletion-test
+ (testing "GHY-3900: the enable-triggered import must not force transform deletion (passes :force-deletion? false)"
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Collection]
+ (let [mock-source (test-helpers/create-mock-source)
+ captured-args (atom nil)]
+ (mt/with-temp [:model/Collection _ {:name "Remote Collection" :is_remote_synced true :location "/"}]
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-branch "main"
+ remote-sync-type :read-only]
+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly mock-source)
+ impl/async-import! (fn [& args] (reset! captured-args args) {:id 123})]
+ (impl/finish-remote-config!)
+ (let [[_branch _force? _import-args & kvs] @captured-args]
+ (is (false? (:force-deletion? (apply hash-map kvs)))
+ "finish-remote-config! should pass :force-deletion? false so deletions surface as conflicts")))))))))
+
(deftest finish-remote-config!-does-nothing-when-collection-exists-in-dev-mode-test
(testing "finish-remote-config! does nothing when collection exists in read-write mode"
(mt/with-model-cleanup [:model/Collection]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index ab39fb8c8b5..0336919a277 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -652,22 +652,27 @@ serdes/meta:
"PythonLibrary source should be imported correctly"))))))))
(deftest import-removes-python-library-not-on-remote-test
- (testing "Import removes local PythonLibrary that doesn't exist on the remote"
+ (testing "Import removes local PythonLibrary that doesn't exist on the remote, but only when forced (GHY-3900)"
(mt/with-premium-features #{:transforms-basic}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
- (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/PythonLibrary]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]
(is (t2/exists? :model/PythonLibrary :id (:id local-library))
"Local PythonLibrary should exist before import")
(let [test-files {"main" {}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result))
- (str "Import should succeed. Result: " result))
- (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))
- "Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))
+ snapshot (source.p/snapshot mock-source)]
+ (testing "a non-forced import reports a conflict and preserves the library"
+ (is (= :conflict (:status (impl/import! snapshot task-id)))
+ "Import should conflict when it would delete an unsynced local PythonLibrary")
+ (is (t2/exists? :model/PythonLibrary :id (:id local-library))
+ "Local PythonLibrary must be preserved when the import is not forced"))
+ (testing "forcing the import discards the local library"
+ (is (= :success (:status (impl/import! snapshot task-id :force? true))))
+ (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))
+ "Local PythonLibrary should be deleted after a forced import since it wasn't on remote")))))))))
(deftest import-preserves-builtin-python-library-test
(testing "Import does NOT delete the built-in PythonLibrary even when it's not on remote"
@@ -798,8 +803,8 @@ serdes/meta:
(is (not (t2/exists? :model/Collection :entity_id coll-eid))
"Local transforms collection should be removed when not on remote")))))))))
-(deftest import-removes-all-local-transforms-when-setting-enabled-and-remote-has-none-test
- (testing "When remote-sync-transforms is enabled and remote has no transforms, all local transforms are removed"
+(deftest import-conflicts-on-local-transforms-when-setting-enabled-and-remote-has-none-test
+ (testing "When remote-sync-transforms is enabled and remote has no transforms, deleting unsynced local transforms requires force"
(mt/with-premium-features #{:transforms-basic}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
@@ -826,15 +831,102 @@ serdes/meta:
test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"
(test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result))
- (str "Import should succeed. Result: " result))
- (is (not (t2/exists? :model/Transform :id local-transform-id))
- "Local transform should be deleted when setting enabled and remote has no transforms")
- (is (not (t2/exists? :model/TransformTag :id local-tag-id))
- "Local tag should be deleted when setting enabled and remote has no transforms")
- (is (not (t2/exists? :model/Collection :entity_id coll-eid))
- "Local transforms collection should be deleted when setting enabled and remote has none")))))))))
+ snapshot (source.p/snapshot mock-source)]
+ (testing "a non-forced import reports a conflict instead of silently deleting (GHY-3900)"
+ (is (= :conflict (:status (impl/import! snapshot task-id)))
+ "Import should conflict when it would delete unsynced local transforms")
+ (is (t2/exists? :model/Transform :id local-transform-id)
+ "Local transform must be preserved when the import is not forced")
+ (is (t2/exists? :model/TransformTag :id local-tag-id)
+ "Local tag must be preserved when the import is not forced")
+ (is (t2/exists? :model/Collection :id coll-id)
+ "Local transforms collection must be preserved when the import is not forced"))
+ (testing "forcing the import discards the local transforms"
+ (is (= :success (:status (impl/import! snapshot task-id :force? true))))
+ (is (not (t2/exists? :model/Transform :id local-transform-id))
+ "Local transform should be deleted when forced")
+ (is (not (t2/exists? :model/TransformTag :id local-tag-id))
+ "Local tag should be deleted when forced")
+ (is (not (t2/exists? :model/Collection :entity_id coll-eid))
+ "Local transforms collection should be deleted when forced"))))))))))
+
+(deftest import-conflicts-on-non-first-import-when-local-transform-would-be-deleted-test
+ (testing "GHY-3900: an already-configured instance (not first import) reports a conflict rather than deleting unsynced local transforms"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Local Transforms Collection"
+ :namespace collection/transforms-ns
+ :entity_id (u/generate-nano-id)
+ :location "/"}
+ :model/Transform {local-transform-id :id} {:name "Local Transform"
+ :entity_id (u/generate-nano-id)
+ :collection_id coll-id}]
+ ;; a prior successful import makes this NOT a first import, so the legacy first-import conflict
+ ;; gate would not fire — the deletion-conflict gate must catch it regardless.
+ (t2/insert! :model/RemoteSyncTask {:sync_task_type "import"
+ :initiated_by (mt/user->id :rasta)
+ :cancelled false
+ :version "prior-version"
+ :ended_at (t/offset-date-time)})
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"
+ (test-helpers/generate-collection-yaml (u/generate-nano-id) "Remote Collection")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result))
+ (str "Import should conflict on a non-first import too. Result: " result))
+ (is (t2/exists? :model/Transform :id local-transform-id)
+ "Local transform must be preserved"))))))))
+
+(deftest import-deletes-synced-transform-without-conflict-test
+ (testing "A previously-synced local transform absent from the remote is reconciled away with no conflict"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/RemoteSyncObject]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Local Transforms Collection"
+ :namespace collection/transforms-ns
+ :entity_id (u/generate-nano-id)
+ :location "/"}
+ :model/Transform {local-transform-id :id} {:name "Synced Transform"
+ :entity_id (u/generate-nano-id)
+ :collection_id coll-id}]
+ ;; mark the transform as already synced — its removal is a normal reconcile, not data loss
+ (t2/insert! :model/RemoteSyncObject {:model_type "Transform"
+ :model_id local-transform-id
+ :model_name "Synced Transform"
+ :status "synced"
+ :status_changed_at (t/offset-date-time)})
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"
+ (test-helpers/generate-collection-yaml (u/generate-nano-id) "Remote Collection")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed without conflict for already-synced transforms. Result: " result))
+ (is (not (t2/exists? :model/Transform :id local-transform-id))
+ "Synced local transform should be reconciled away"))))))))
+
+(deftest import-does-not-conflict-on-builtin-transform-tag-test
+ (testing "Built-in TransformTags are excluded from removal and do not trigger a deletion conflict"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/TransformTag]
+ (mt/with-temp [:model/TransformTag {builtin-tag-id :id} {:name "Built-in Tag"
+ :entity_id (u/generate-nano-id)
+ :built_in_type "hourly"}]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"
+ (test-helpers/generate-collection-yaml (u/generate-nano-id) "Remote Collection")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Built-in tag should not cause a conflict. Result: " result))
+ (is (t2/exists? :model/TransformTag :id builtin-tag-id)
+ "Built-in tag should be preserved"))))))))
(deftest import-auto-enables-setting-when-remote-has-transforms-test
(testing "When setting is disabled and remote has transforms, setting is auto-enabled and transforms are synced"