The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Don't delete local transforms when remote has none (GHY-3900) (#75898)
Eric Normand7b274b46ac4d
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 1c58e2792d9..9f46c21d442 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -185,14 +185,15 @@:details {:error-type (type e)}})))(defn- get-conflicts- "Detects conflicts that would prevent or complicate import.- Returns a map with :conflicts (detailed list) and :summary (set of category names).+ "Detects conflicts that would prevent or complicate import. Returns a map with two classes:+ - :first-import-conflicts - feature/namespace/library conflicts that only block on the very first import+ - :deletion-conflicts - unsynced local entities of all-or-nothing models (transforms) that the import+ would wholesale-delete; these block on EVERY import, not just the firstConflict types detected:- - :entity-id-conflict - Items with existing entity IDs that are NOT already synced- :library-conflict - First import only, local Library exists, import has Library- - :transforms-not-enabled - Import has Transform/TransformTag/PythonLibrary, setting disabled- - :snippets-without-library - Import has NativeQuerySnippet, Library not remote-synced"+ - :transforms-conflict / :snippets-conflict - import has matching content but local has unsynced entities+ - :*-conflict (deletion) - import lacks transforms/tags/libraries that exist locally and unsynced (GHY-3900)"[ingestable first-import?](let [ingest-list (serialization/ingest-list ingestable)imported-data (spec/extract-imported-entities ingest-list)@@ -226,12 +227,12 @@{:type :library-conflict:category "Library":message "Import contains Library but local instance has an unsynced Library collection"}))- all-conflicts (concat- feature-conflicts- ns-coll-conflicts- (when library-conflict [library-conflict]))]- {:conflicts (vec all-conflicts)- :summary (into #{} (map :category) all-conflicts)}))+ first-import-conflicts (concat+ feature-conflicts+ ns-coll-conflicts+ (when library-conflict [library-conflict]))]+ {:first-import-conflicts (vec first-import-conflicts)+ :deletion-conflicts (spec/check-deletion-conflicts imported-data)}))(defn- record-exported-paths!"Records each exported entity's repo file path on its RemoteSyncObject row, so later renames and@@ -375,7 +376,10 @@"Imports and reloads Metabase entities from a remote snapshot.Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:- - :force? - forces import even when the snapshot version matches the last imported version+ - :force? - forces import even when the snapshot version matches the last imported version or has first-import conflicts+ - :force-deletion? - forces past deletion conflicts (unsynced local transforms the import would delete); defaults+ to :force?. Kept separate so the setup/enable flow can force past version/dirty guards (force? true) while still+ surfacing transform-deletion as a conflict the admin must confirm (force-deletion? false) — see GHY-3900.- :merge? - perform a local-only 3-way merge (keep un-pushed local changes) instead of overwriting local- :base-snapshot - the merge base (last synced version), required when :merge? is set- :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs@@ -387,7 +391,7 @@Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may bethrown during import and are caught and converted to error status maps."- [^SourceSnapshot snapshot task-id & {:keys [force? merge? base-snapshot pre-task-branch]}]+ [^SourceSnapshot snapshot task-id & {:keys [force? force-deletion? merge? base-snapshot pre-task-branch]}](when (branch-changed-since-scheduling? pre-task-branch)(log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"pre-task-branch (settings/remote-sync-branch))@@ -404,15 +408,21 @@(let [snapshot-version (source.p/version snapshot)last-imported-version (remote-sync.task/last-version)first-import? (nil? last-imported-version)+ ;; force-deletion? is optional; when a caller doesn't pass it, deletion follows force?+ force-deletion? (if (nil? force-deletion?) force? force-deletion?)path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})- {:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)]+ {:keys [first-import-conflicts deletion-conflicts]} (get-conflicts base-ingestable first-import?)+ ;; First-import conflicts only block on the first import; deletion conflicts block on every+ ;; import (an already-configured instance must not silently delete unsynced transforms).+ blocking-conflicts (vec (concat (when (and first-import? (not force?)) first-import-conflicts)+ (when (not force-deletion?) deletion-conflicts)))](cond- (and first-import? (not force?) (seq conflicts))+ (seq blocking-conflicts)(u/prog1 {:status :conflict:version (source.p/version snapshot)- :conflicts summary ; Keep backward compatibility: return set of category names- :conflict-details conflicts ; New: detailed conflict info+ :conflicts (into #{} (map :category) blocking-conflicts) ; Backward compat: set of category names+ :conflict-details blocking-conflicts ; Detailed conflict info:message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)}(log/infof (:message <>)))@@ -1004,7 +1014,7 @@Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if thereare unsaved changes and neither force? nor merge? is set."- [branch force? import-args & {:keys [on-success merge?]}]+ [branch force? import-args & {:keys [on-success merge? force-deletion?]}](guards/ensure-no-active-task!)(let [pre-task-branch (settings/remote-sync-branch)source (source/source-from-settings branch)@@ -1028,6 +1038,7 @@(import! snapshot task-id(assoc import-args:force? force?+ :force-deletion? force-deletion?:merge? merge?:base-snapshot base-snapshot:pre-task-branch pre-task-branch)))@@ -1129,6 +1140,8 @@(when (str/blank? (setting/get :remote-sync-branch))(setting/set! :remote-sync-branch (source.p/default-branch (source/source-from-settings))))(when (= :read-only (settings/remote-sync-type))- (:id (async-import! (settings/remote-sync-branch) true {}))))+ ;; force? true bypasses the version/dirty guards for setup, but force-deletion? false keeps unsynced+ ;; local transforms from being silently destroyed — they surface as a conflict instead (GHY-3900).+ (:id (async-import! (settings/remote-sync-branch) true {} :force-deletion? false))))(u/prog1 nil(collection/clear-remote-synced-collection!))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 3fbf6bfeebe..744db12c134 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -594,6 +594,51 @@:message (format "Import contains %s but local instance has unsynced %s namespace collections"category category)}))))+(defn- removal-condition-clauses+ "Converts a spec's removal-conditions map into HoneySQL where-fragments, matching the semantics+ [[metabase-enterprise.remote-sync.impl/build-entity-id-where-clause]] uses: an :entity_id entry+ whose value is an [op value] pair becomes [op :entity_id value]; any other entry becomes [:= key value]."+ [removal-conds]+ (for [[k v] removal-conds]+ (if (and (= k :entity_id) (vector? v))+ (let [[op value] v] [op :entity_id value])+ [:= k v])))++(defn check-deletion-conflicts+ "Detects local entities of all-or-nothing models (specs with :all-on-setting-disable) that an import+ would wholesale-delete because they are absent from it, but which hold unsaved local work — i.e. they+ have no RemoteSyncObject in 'synced' status. Already-synced entities are excluded: their removal is a+ normal reconcile against the remote source of truth, not data loss.++ Takes imported-data from [[extract-imported-entities]]. Returns a vector of conflict maps+ ({:type :category :message}), one per affected model type (categories repeat across the transform models;+ callers dedupe via the conflict summary)."+ [{:keys [by-entity-id]}]+ (into []+ (for [[model-key spec] (specs-for-deletion)+ :let [setting-kw (get-in spec [:removal :all-on-setting-disable])]+ :when setting-kw+ :let [model-type (:model-type spec)+ imported-ids (get by-entity-id model-type #{})+ conds (cond-> []+ (seq imported-ids) (conj [:not-in :entity_id imported-ids])+ :always (into (removal-condition-clauses (removal-conditions spec))))+ candidates (if (seq conds)+ (t2/select [model-key :id] {:where (into [:and] conds)})+ (t2/select [model-key :id]))+ n-unsynced (count (remove (fn [{:keys [id]}]+ (t2/exists? :model/RemoteSyncObject+ :model_type model-type+ :model_id id+ :status "synced"))+ candidates))]+ :when (pos? n-unsynced)+ :let [category (setting->category setting-kw)]]+ {:type (keyword (str (u/lower-case-en category) "-conflict"))+ :category category+ :message (format "Import would delete %d unsynced local %s %s"+ n-unsynced category (if (= 1 n-unsynced) "entity" "entities"))})))+;;; ------------------------------------------------ Eligibility Checking ----------------------------------------------(defn transforms-namespace-collection?diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 023c59013f1..80c3edd88b9 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -560,6 +560,23 @@(is @import-called?"Should call async-import! in read-only mode")))))))))+(deftest finish-remote-config!-does-not-force-transform-deletion-test+ (testing "GHY-3900: the enable-triggered import must not force transform deletion (passes :force-deletion? false)"+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Collection]+ (let [mock-source (test-helpers/create-mock-source)+ captured-args (atom nil)]+ (mt/with-temp [:model/Collection _ {:name "Remote Collection" :is_remote_synced true :location "/"}]+ (mt/with-temporary-setting-values [remote-sync-enabled true+ remote-sync-url "https://github.com/test/repo.git"+ remote-sync-branch "main"+ remote-sync-type :read-only]+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly mock-source)+ impl/async-import! (fn [& args] (reset! captured-args args) {:id 123})]+ (impl/finish-remote-config!)+ (let [[_branch _force? _import-args & kvs] @captured-args]+ (is (false? (:force-deletion? (apply hash-map kvs)))+ "finish-remote-config! should pass :force-deletion? false so deletions surface as conflicts")))))))))+(deftest finish-remote-config!-does-nothing-when-collection-exists-in-dev-mode-test(testing "finish-remote-config! does nothing when collection exists in read-write mode"(mt/with-model-cleanup [:model/Collection]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex ab39fb8c8b5..0336919a277 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -652,22 +652,27 @@ serdes/meta:"PythonLibrary source should be imported correctly"))))))))(deftest import-removes-python-library-not-on-remote-test- (testing "Import removes local PythonLibrary that doesn't exist on the remote"+ (testing "Import removes local PythonLibrary that doesn't exist on the remote, but only when forced (GHY-3900)"(mt/with-premium-features #{:transforms-basic}(mt/with-temporary-setting-values [remote-sync-transforms trueremote-sync-enabled true]- (mt/with-model-cleanup [:model/RemoteSyncTask]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/PythonLibrary](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})](is (t2/exists? :model/PythonLibrary :id (:id local-library))"Local PythonLibrary should exist before import")(let [test-files {"main" {}}mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result))- (str "Import should succeed. Result: " result))- (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))- "Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))+ snapshot (source.p/snapshot mock-source)]+ (testing "a non-forced import reports a conflict and preserves the library"+ (is (= :conflict (:status (impl/import! snapshot task-id)))+ "Import should conflict when it would delete an unsynced local PythonLibrary")+ (is (t2/exists? :model/PythonLibrary :id (:id local-library))+ "Local PythonLibrary must be preserved when the import is not forced"))+ (testing "forcing the import discards the local library"+ (is (= :success (:status (impl/import! snapshot task-id :force? true))))+ (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))+ "Local PythonLibrary should be deleted after a forced import since it wasn't on remote")))))))))(deftest import-preserves-builtin-python-library-test(testing "Import does NOT delete the built-in PythonLibrary even when it's not on remote"@@ -798,8 +803,8 @@ serdes/meta:(is (not (t2/exists? :model/Collection :entity_id coll-eid))"Local transforms collection should be removed when not on remote")))))))))-(deftest import-removes-all-local-transforms-when-setting-enabled-and-remote-has-none-test- (testing "When remote-sync-transforms is enabled and remote has no transforms, all local transforms are removed"+(deftest import-conflicts-on-local-transforms-when-setting-enabled-and-remote-has-none-test+ (testing "When remote-sync-transforms is enabled and remote has no transforms, deleting unsynced local transforms requires force"(mt/with-premium-features #{:transforms-basic}(mt/with-temporary-setting-values [remote-sync-transforms trueremote-sync-enabled true]@@ -826,15 +831,102 @@ serdes/meta:test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"(test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result))- (str "Import should succeed. Result: " result))- (is (not (t2/exists? :model/Transform :id local-transform-id))- "Local transform should be deleted when setting enabled and remote has no transforms")- (is (not (t2/exists? :model/TransformTag :id local-tag-id))- "Local tag should be deleted when setting enabled and remote has no transforms")- (is (not (t2/exists? :model/Collection :entity_id coll-eid))- "Local transforms collection should be deleted when setting enabled and remote has none")))))))))+ snapshot (source.p/snapshot mock-source)]+ (testing "a non-forced import reports a conflict instead of silently deleting (GHY-3900)"+ (is (= :conflict (:status (impl/import! snapshot task-id)))+ "Import should conflict when it would delete unsynced local transforms")+ (is (t2/exists? :model/Transform :id local-transform-id)+ "Local transform must be preserved when the import is not forced")+ (is (t2/exists? :model/TransformTag :id local-tag-id)+ "Local tag must be preserved when the import is not forced")+ (is (t2/exists? :model/Collection :id coll-id)+ "Local transforms collection must be preserved when the import is not forced"))+ (testing "forcing the import discards the local transforms"+ (is (= :success (:status (impl/import! snapshot task-id :force? true))))+ (is (not (t2/exists? :model/Transform :id local-transform-id))+ "Local transform should be deleted when forced")+ (is (not (t2/exists? :model/TransformTag :id local-tag-id))+ "Local tag should be deleted when forced")+ (is (not (t2/exists? :model/Collection :entity_id coll-eid))+ "Local transforms collection should be deleted when forced"))))))))))++(deftest import-conflicts-on-non-first-import-when-local-transform-would-be-deleted-test+ (testing "GHY-3900: an already-configured instance (not first import) reports a conflict rather than deleting unsynced local transforms"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Local Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id (u/generate-nano-id)+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id (u/generate-nano-id)+ :collection_id coll-id}]+ ;; a prior successful import makes this NOT a first import, so the legacy first-import conflict+ ;; gate would not fire — the deletion-conflict gate must catch it regardless.+ (t2/insert! :model/RemoteSyncTask {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :cancelled false+ :version "prior-version"+ :ended_at (t/offset-date-time)})+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"+ (test-helpers/generate-collection-yaml (u/generate-nano-id) "Remote Collection")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result))+ (str "Import should conflict on a non-first import too. Result: " result))+ (is (t2/exists? :model/Transform :id local-transform-id)+ "Local transform must be preserved"))))))))++(deftest import-deletes-synced-transform-without-conflict-test+ (testing "A previously-synced local transform absent from the remote is reconciled away with no conflict"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/RemoteSyncObject]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Local Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id (u/generate-nano-id)+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Synced Transform"+ :entity_id (u/generate-nano-id)+ :collection_id coll-id}]+ ;; mark the transform as already synced — its removal is a normal reconcile, not data loss+ (t2/insert! :model/RemoteSyncObject {:model_type "Transform"+ :model_id local-transform-id+ :model_name "Synced Transform"+ :status "synced"+ :status_changed_at (t/offset-date-time)})+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"+ (test-helpers/generate-collection-yaml (u/generate-nano-id) "Remote Collection")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed without conflict for already-synced transforms. Result: " result))+ (is (not (t2/exists? :model/Transform :id local-transform-id))+ "Synced local transform should be reconciled away"))))))))++(deftest import-does-not-conflict-on-builtin-transform-tag-test+ (testing "Built-in TransformTags are excluded from removal and do not trigger a deletion conflict"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/TransformTag]+ (mt/with-temp [:model/TransformTag {builtin-tag-id :id} {:name "Built-in Tag"+ :entity_id (u/generate-nano-id)+ :built_in_type "hourly"}]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"+ (test-helpers/generate-collection-yaml (u/generate-nano-id) "Remote Collection")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Built-in tag should not cause a conflict. Result: " result))+ (is (t2/exists? :model/TransformTag :id builtin-tag-id)+ "Built-in tag should be preserved"))))))))(deftest import-auto-enables-setting-when-remote-has-transforms-test(testing "When setting is disabled and remote has transforms, setting is auto-enabled and transforms are synced"