Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Report conflicts on transforms/snippets/lib (#68506)

John Swanson00c573febe68
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index b76caf6e338..77a5fa49e65 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1589,7 +1589,6 @@
enterprise/transforms-python
enterprise/workspaces}}
-
types
{:team "Querying"
:api #{metabase.types.core
@@ -2252,7 +2251,8 @@
task
task-history
util
- enterprise/serialization}}
+ enterprise/serialization
+ enterprise/transforms-python}}
enterprise/sandbox
{:team "UX West"
@@ -2420,6 +2420,7 @@
enterprise/transforms-python
{:team "Querying"
:api #{metabase-enterprise.transforms-python.api
+ metabase-enterprise.transforms-python.core
metabase-enterprise.transforms-python.init
metabase-enterprise.transforms-python.python-runner}
:uses #{analytics
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index d7b0d7ce827..b4348c41dd5 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -23,20 +23,15 @@
[toucan2.core :as t2])
(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
-(def ^:private transform-models
- "Models that indicate transforms content in a snapshot."
- #{"Transform" "TransformTag" "PythonLibrary"})
-
(defn- snapshot-has-transforms?
"Checks if the snapshot contains any Transform, TransformTag, or PythonLibrary entities.
Used to auto-enable remote-sync-transforms setting during import.
Uses the ingestable to list all entities and checks their :model metadata."
[ingestable]
- (let [serdes-paths (serialization/ingest-list ingestable)]
- (some (fn [path]
- (some #(transform-models (:model %)) path))
- serdes-paths)))
+ (let [serdes-paths (serialization/ingest-list ingestable)
+ models-present (spec/models-in-import serdes-paths)]
+ (some spec/transform-models models-present)))
(defn- sync-objects!
"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and
@@ -51,6 +46,19 @@
(when (seq all-inserts)
(t2/insert! :model/RemoteSyncObject all-inserts))))
+(defn- build-entity-id-where-clause
+ "Builds a HoneySQL WHERE clause for entity_id filtering.
+ Combines the imported entity-ids exclusion with any spec-level entity_id condition."
+ [entity-ids spec-entity-id-condition]
+ (let [imported-condition (when (seq entity-ids)
+ [:not-in :entity_id entity-ids])
+ spec-condition (when spec-entity-id-condition
+ (let [[op value] spec-entity-id-condition]
+ [op :entity_id value]))
+ conditions (filterv some? [imported-condition spec-condition])]
+ (when (pos? (count conditions))
+ (into [:and] conditions))))
+
(defn- remove-unsynced!
"Deletes any remote sync content that was NOT part of the import.
@@ -68,22 +76,32 @@
(doseq [[model-key model-spec] (spec/specs-for-deletion)
:let [serdes-model (:model-type model-spec)
entity-ids (get by-entity-id serdes-model [])
- entity-id-clause (if (seq entity-ids)
- [:not-in entity-ids]
- :entity_id)
+ spec-entity-id-condition (get-in model-spec [:conditions :entity_id])
+ entity-id-where (build-entity-id-where-clause entity-ids spec-entity-id-condition)
scope-key (get-in model-spec [:removal :scope-key])
- extra-conditions (into [] cat (:conditions model-spec))]]
+ ;; Get non-entity_id conditions from spec
+ other-conditions (into [] cat (dissoc (:conditions model-spec) :entity_id))]]
(if scope-key
;; Collection-scoped: delete only within synced collections
(when (seq synced-collection-ids)
+ (if entity-id-where
+ (apply t2/delete! model-key
+ {:where [:and
+ [:in scope-key synced-collection-ids]
+ entity-id-where]}
+ other-conditions)
+ (apply t2/delete! model-key
+ scope-key [:in synced-collection-ids]
+ other-conditions)))
+ ;; Global: delete by entity_id
+ (if entity-id-where
(apply t2/delete! model-key
- scope-key [:in synced-collection-ids]
- :entity_id entity-id-clause
- extra-conditions))
- ;; Global: delete by entity_id only
- (apply t2/delete! model-key
- :entity_id entity-id-clause
- extra-conditions))))
+ {:where entity-id-where}
+ other-conditions)
+ ;; No entity_id conditions - delete all (respecting other spec conditions if any)
+ (if (seq other-conditions)
+ (apply t2/delete! model-key other-conditions)
+ (t2/delete! model-key))))))
(defn source-error-message
"Constructs user-friendly error messages from remote sync source exceptions.
@@ -132,6 +150,37 @@
:details {:error-type (type e)}})))
+(defn- get-conflicts
+ "Detects conflicts that would prevent or complicate import.
+ Returns a map with :conflicts (detailed list) and :summary (set of category names).
+
+ Conflict types detected:
+ - :entity-id-conflict - Items with existing entity IDs that are NOT already synced
+ - :library-conflict - First import only, local Library exists, import has Library
+ - :transforms-not-enabled - Import has Transform/TransformTag/PythonLibrary, setting disabled
+ - :snippets-without-library - Import has NativeQuerySnippet, Library not remote-synced"
+ [ingestable first-import?]
+ (let [ingest-list (serialization/ingest-list ingestable)
+ imported-data (spec/extract-imported-entities ingest-list)
+ models-present (spec/models-in-import ingest-list)
+ ;; TODO (epaget 2026-02-02) -- entity-id conflict checking (detect unsynced local entities with matching entity_ids)
+ feature-conflicts (spec/check-feature-conflicts models-present)
+ library-conflict (when-let [local-library (t2/select-one :model/Collection :type collection/library-collection-type)]
+ (when (and first-import?
+ (contains? (get-in imported-data [:by-entity-id "Collection"] #{})
+ collection/library-entity-id)
+ (not (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id (:id local-library))))
+ {:type :library-conflict
+ :category "Library"
+ :message "Import contains Library but local instance has an unsynced Library collection"}))
+ all-conflicts (concat
+ feature-conflicts
+ (when library-conflict [library-conflict]))]
+ {:conflicts (vec all-conflicts)
+ :summary (into #{} (map :category) all-conflicts)}))
+
(defn import!
"Imports and reloads Metabase entities from a remote snapshot.
@@ -150,18 +199,32 @@
(if snapshot
(try
(let [snapshot-version (source.p/version snapshot)
- last-imported-version (remote-sync.task/last-version)]
- (if (and (not force?) (= last-imported-version snapshot-version))
+ last-imported-version (remote-sync.task/last-version)
+ first-import? (nil? last-imported-version)
+ path-filters [#"collections/.*" #"databases/.*" #"actions/.*"
+ #"transforms/.*" #"python-libraries/.*" #"snippets/.*"]
+ base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
+ has-transforms? (snapshot-has-transforms? base-ingestable)
+ {:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)
+ ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)]
+
+ (cond
+ (and first-import? (not force?) (seq conflicts))
+ (u/prog1 {:status :conflict
+ :version (source.p/version snapshot)
+ :conflicts summary ; Keep backward compatibility: return set of category names
+ :conflict-details conflicts ; New: detailed conflict info
+ :message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)}
+ (log/infof (:message <>)))
+
+ (and (not force?) (= last-imported-version snapshot-version))
(u/prog1 {:status :success
:version (source.p/version snapshot)
:message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}
(log/infof (:message <>)))
- (let [path-filters [#"collections/.*" #"databases/.*" #"actions/.*"
- #"transforms/.*" #"python-libraries/.*" #"snippets/.*"]
- base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
- has-transforms? (snapshot-has-transforms? base-ingestable)
- ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)
- load-result (serdes/with-cache
+
+ :else
+ (let [load-result (serdes/with-cache
(serialization/load-metabase! ingestable-snapshot))
seen-paths (:seen load-result)
imported-data (spec/extract-imported-entities seen-paths)]
@@ -309,10 +372,11 @@
(defn handle-task-result!
"Handles the outcome of running import! or export! by updating the RemoteSyncTask record.
- Takes a result map with a :status key (either :success or :error) and optional :message key, a RemoteSyncTask ID,
- and an optional branch name. On success, updates the remote-sync-branch setting (if branch provided), marks the
- task complete, and invalidates the remote changes cache. On error, marks the task as failed with the error message.
- For any other status, marks the task as failed with 'Unexpected Error'."
+ Takes a result map with a :status key (either :success, :conflict, or :error) and optional :message key, a
+ RemoteSyncTask ID, and an optional branch name. On success, updates the remote-sync-branch setting (if branch
+ provided), marks the task complete, and invalidates the remote changes cache. On conflict, sets the version and
+ stores the conflicts. On error, marks the task as failed with the error message. For any other status, marks the
+ task as failed with 'Unexpected Error'."
[result task-id & [branch]]
(case (:status result)
:success (do
@@ -321,6 +385,9 @@
(settings/remote-sync-branch! branch))
(remote-sync.task/complete-sync-task! task-id))
(invalidate-remote-changes-cache!))
+ :conflict (do
+ (remote-sync.task/set-version! task-id (:version result))
+ (remote-sync.task/conflict-sync-task! task-id (:conflicts result)))
:error (remote-sync.task/fail-sync-task! task-id (:message result))
(remote-sync.task/fail-sync-task! task-id "Unexpected Error")))
@@ -351,8 +418,8 @@
(defn async-import!
"Imports remote-synced collections from a remote source repository asynchronously.
- Takes a branch name to import from, a force? boolean (if true, imports even if there are unsaved changes), and an
- import-args map of additional arguments to pass to the import function. Checks for dirty changes and throws an
+ Takes a branch name to import from, a force? boolean (if true, imports even if there are unsaved changes or conflicts),
+ and an import-args map of additional arguments to pass to the import function. Checks for dirty changes and throws an
exception if force? is false and changes exist.
Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if there
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
index 2d9760dff0e..bf2b6fcb8f4 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
@@ -20,6 +20,9 @@
(derive :model/RemoteSyncTask :metabase/model)
+(t2/deftransforms :model/RemoteSyncTask
+ {:conflicts mi/transform-json})
+
(declare current-task)
(t2/define-before-insert :model/RemoteSyncTask
@@ -208,6 +211,30 @@
(t/< (:last_progress_report_at task)
(t/minus (t/offset-date-time) (t/millis (settings/remote-sync-task-time-limit-ms))))))
+(defn conflict?
+ "Checks if a task ended with conflicts.
+
+ Takes a RemoteSyncTask instance.
+
+ Returns true if the task has conflicts stored, false otherwise."
+ [task]
+ (and (some? (:ended_at task))
+ (false? (:cancelled task))
+ (nil? (:error_message task))
+ (some? (:conflicts task))))
+
+(defn conflict-sync-task!
+ "Marks a sync task as having conflicts.
+
+ Takes the ID of the sync task and a collection of conflicts (vector of strings).
+ Conflicts are automatically serialized to JSON via the model transform.
+
+ Returns the number of rows updated (should be 1 if successful)."
+ [task-id conflicts]
+ (t2/update! :model/RemoteSyncTask task-id
+ {:ended_at (mi/now)
+ :conflicts conflicts}))
+
;;; ------------------------------------------- Hydration -------------------------------------------
(methodical/defmethod t2/batched-hydrate [:model/RemoteSyncTask :initiated_by_user]
@@ -222,6 +249,7 @@
(for [task tasks]
(assoc task :status (cond
(failed? task) :errored
+ (conflict? task) :conflict
(successful? task) :successful
(cancelled? task) :cancelled
(timed-out? task) :timed-out
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
index 7039012f2a1..8cac8bb5229 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
@@ -5,7 +5,7 @@
(def TaskStatus
"Status of a remote sync task."
- [:enum :running :successful :errored :cancelled :timed-out])
+ [:enum :running :successful :errored :cancelled :timed-out :conflict])
(def TaskType
"Type of remote sync task."
@@ -24,8 +24,38 @@
[:version {:optional true} [:maybe :string]]
[:cancelled {:optional true} [:maybe :boolean]]
[:error_message {:optional true} [:maybe :string]]
+ [:conflicts {:optional true} [:maybe [:sequential :string]]]
[:status TaskStatus]])
+;;; ------------------------------------------- Conflict Schemas -------------------------------------------
+
+(def ConflictType
+ "Type of import conflict detected during pre-flight check."
+ [:enum
+ :entity-id-conflict ; Local entity exists with same entity_id but not in RemoteSyncObject
+ :library-conflict ; First import, local Library exists, import has Library
+ :transforms-conflict ; Local has transforms AND import has transforms
+ :snippets-conflict ; Local has snippets AND import has snippets
+ :dirty]) ; RemoteSyncObject has items with status != "synced"
+
+(def ConflictDetail
+ "Schema for detailed conflict information."
+ [:map
+ [:type ConflictType]
+ [:category {:optional true} :string]
+ [:count {:optional true} pos-int?]
+ [:entity-ids {:optional true} [:set :string]]
+ [:message {:optional true} :string]])
+
+(def ConflictResponse
+ "Schema for conflict error response from async-import!."
+ [:map
+ [:status-code [:= 400]]
+ [:conflicts [:= true]]
+ [:conflict-type {:optional true} ConflictType]
+ [:conflict-details {:optional true} [:sequential ConflictDetail]]
+ [:conflict-summary {:optional true} [:set :string]]])
+
;;; ------------------------------------------- Dirty Item Schemas -------------------------------------------
(def DirtyItem
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 2b4e140114c..5ad9be679e0 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -11,10 +11,13 @@
To add a new syncable model, add a spec entry to `remote-sync-specs` and
optionally implement custom multimethods if the default behavior doesn't fit."
(:require
+ [clojure.set :as set]
[clojure.string :as str]
[clojure.walk :as walk]
[metabase-enterprise.remote-sync.settings :as rs-settings]
+ [metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.collections.core :as collections]
+ [metabase.collections.models.collection :as collection]
[metabase.models.serialization :as serdes]
[metabase.settings.core :as setting]
[metabase.util :as u]
@@ -304,7 +307,8 @@
:archived-key nil ; no archived field
:tracking {:select-fields [:path]
:field-mappings {:model_name :path}}
- :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
+ :conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; exclude built-in common.py from sync
+ :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
:all-on-setting-disable :remote-sync-transforms}
:export-scope :all ; query for all instances
:enabled? :remote-sync-transforms}})
@@ -405,6 +409,111 @@
[]
(into #{} (map :model-type (vals remote-sync-specs))))
+;;; ---------------------------------------- Import Conflict Detection -----------------------------------------------
+
+(defn optional-feature-specs
+ "Returns specs for optional features, grouped by their enabling setting keyword.
+ Optional features are those with a keyword `:enabled?` value (not `true`).
+ Returns map of {setting-keyword {model-key spec}}."
+ []
+ (-> remote-sync-specs
+ (->> (filter (fn [[_ spec]] (keyword? (:enabled? spec))))
+ (group-by (fn [[_ spec]] (:enabled? spec))))
+ (update-vals #(into {} %))))
+
+(defn- setting->category
+ "Converts a setting keyword to a human-readable conflict category name."
+ [setting-kw]
+ (case setting-kw
+ :remote-sync-transforms "Transforms"
+ :library-synced "Snippets"
+ (str/capitalize (name setting-kw))))
+
+(defn models-for-setting
+ "Returns set of model-type strings for specs with the given `:enabled?` setting."
+ [setting-kw]
+ (->> remote-sync-specs
+ (filter (fn [[_ spec]] (= (:enabled? spec) setting-kw)))
+ (map (fn [[_ spec]] (:model-type spec)))
+ (into #{})))
+
+(def transform-models
+ "Models that indicate transforms content in a snapshot.
+ Derived from specs with `:enabled? :remote-sync-transforms`."
+ (disj (models-for-setting :remote-sync-transforms) :model/TransformTag))
+
+(defn models-in-import
+ "Returns set of model-type strings present in the import.
+ Takes an ingest-list (sequence of serdes paths from serialization/ingest-list)."
+ [ingest-list]
+ (->> ingest-list
+ (map (fn [path] (:model (last path))))
+ (into #{})))
+
+(defn check-entity-id-conflicts
+ "Checks if imported entity_ids exist locally but are NOT in RemoteSyncObject.
+ Returns map of {model-type #{conflicting-entity-ids}}.
+
+ Excludes the Library collection entity_id since that's handled by the library-conflict check."
+ [imported-entity-ids-by-model]
+ (into {}
+ (for [[model-type entity-ids] imported-entity-ids-by-model
+ :when (seq entity-ids)
+ :let [spec (spec-for-model-type model-type)
+ model-key (:model-key spec)]
+ :when (and spec model-key (#{:entity-id :hybrid} (:identity spec)))
+ :let [local-entity-ids (t2/select-fn-set :entity_id model-key :entity_id [:in entity-ids])
+ tracked-entity-ids (when (seq local-entity-ids)
+ (let [pks (t2/select-pks-vec model-key
+ :entity_id [:in local-entity-ids])]
+ (t2/select-fn-set
+ (fn [rso]
+ (:entity_id (t2/select-one model-key :id (:model_id rso))))
+ :model/RemoteSyncObject
+ :model_type model-type
+ :model_id [:in pks])))
+ conflicting-entity-ids (set/difference local-entity-ids (or tracked-entity-ids #{}))
+ conflicting-entity-ids (if (= model-type "Collection")
+ (disj conflicting-entity-ids collection/library-entity-id)
+ conflicting-entity-ids)]
+ :when (seq conflicting-entity-ids)]
+ [model-type conflicting-entity-ids])))
+
+(defn- has-unsynced-entities-for-feature?
+ "Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.
+ Excludes built-in TransformTags from the count since they are system-created and not user data."
+ [specs-for-feature]
+ (some (fn [[_ spec]]
+ (let [model-key (:model-key spec)
+ model-type (:model-type spec)
+ ;; Exclude built-in entities from count (they are system-created, not user data)
+ local-count (case model-key
+ :model/TransformTag (t2/count model-key :built_in_type nil)
+ :model/PythonLibrary (t2/count model-key :entity_id [:not= transforms-python/builtin-entity-id])
+ (t2/count model-key))
+ synced-count (t2/count :model/RemoteSyncObject :model_type model-type)]
+ (and (pos? local-count)
+ (> local-count synced-count))))
+ specs-for-feature))
+
+(defn check-feature-conflicts
+ "Checks if import contains models that conflict with existing local entities that are NOT already remote synced.
+ Derives conflict categories from specs with keyword `:enabled?` values (optional features).
+ Only triggers conflict when local has unsynced entities AND import has matching entities.
+ If local entities are already synced, dirty tracking handles conflicts instead."
+ [models-present]
+ (let [feature-groups (optional-feature-specs)]
+ (into []
+ (for [[setting-kw specs-for-feature] feature-groups
+ :let [feature-model-types (into #{} (map (fn [[_ s]] (:model-type s))) specs-for-feature)]
+ :when (some feature-model-types models-present)
+ :when (has-unsynced-entities-for-feature? specs-for-feature)
+ :let [category (setting->category setting-kw)]]
+ {:type (keyword (str (u/lower-case-en category) "-conflict"))
+ :category category
+ :message (format "Import contains %s but local instance has unsynced %s"
+ category category)}))))
+
;;; ------------------------------------------------ Eligibility Checking ----------------------------------------------
(defn transforms-namespace-collection?
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/core.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/core.clj
new file mode 100644
index 00000000000..9349e86ea95
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/core.clj
@@ -0,0 +1,10 @@
+(ns metabase-enterprise.transforms-python.core
+ "Core namespace for Python transforms functionality.
+ Re-exports commonly used vars from internal namespaces."
+ (:require
+ [metabase-enterprise.transforms-python.models.python-library :as python-library]
+ [potemkin :as p]))
+
+(p/import-vars
+ [python-library
+ builtin-entity-id])
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
index 19f46ede548..19d881c0a22 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
@@ -50,6 +50,11 @@
"Set of allowed library paths. Currently only 'common' is supported."
#{"common.py"})
+(def builtin-entity-id
+ "The entity_id of the built-in common.py PythonLibrary created by migration.
+ Used to protect it from deletion during remote-sync import."
+ "cWWH9qJPvHNB3rP2vLZrK")
+
(defn- validate-path!
"Validates that the given path is allowed. Throws an exception if not."
[path]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 22aac37c8bd..fba8d057c2f 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -1137,3 +1137,166 @@ serdes/meta:
"python-libraries path should be included in filters")
(is (some #(str/includes? % "snippets") filter-strs)
"snippets path should be included in filters"))))))))))))
+
+(deftest import!-blocks-if-it-encounters-library-conflicts
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)
+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"
+ :type "library"
+ :entity_id collection/library-entity-id
+ :location "/"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-collection-yaml collection/library-entity-id "Another Library")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (= #{"Library"} (:conflicts result)))))))
+
+(deftest import!-blocks-if-it-encounters-snippet-conflicts
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "Test Snippet"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-snippet-yaml "blahblahblah" "A Snippet" "select 123")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (= #{"Snippets"} (:conflicts result)))))))
+
+(deftest import!-blocks-if-it-encounters-transform-conflicts
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Transform _ {:name "Test Transform"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-transform-yaml "blahblahblah" "A Transform")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (= #{"Transforms"} (:conflicts result)))))))
+
+(deftest import!-reports-multiple-conflicts
+ (testing "when multiple conflict types exist, all are reported in the :conflicts set"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)
+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"
+ :type "library"
+ :entity_id collection/library-entity-id
+ :location "/"}
+ :model/NativeQuerySnippet _ {:name "Test Snippet"}
+ :model/Transform _ {:name "Test Transform"}]
+ (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")
+ "collections/snip-_/snip.yaml"
+ (test-helpers/generate-snippet-yaml "snip-entity-id" "Remote Snippet" "select 1")
+ "collections/trans-_/trans.yaml"
+ (test-helpers/generate-transform-yaml "trans-entity-id" "Remote Transform")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (= #{"Library" "Transforms" "Snippets"} (:conflicts result))))))))
+
+(deftest import!-force-bypasses-conflicts
+ (testing "force?: true allows import to proceed despite conflicts"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)
+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"
+ :type "library"
+ :entity_id collection/library-entity-id
+ :location "/"}]
+ (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id :force? true)]
+ (is (= :success (:status result))))))))
+
+(deftest import!-conflicts-only-block-initial-import
+ (testing "when last-imported-version is set, conflicts do not block subsequent imports"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)
+ (mt/with-temp [:model/Collection _ {:name "Local Library"
+ :type "library"
+ :entity_id collection/library-entity-id
+ :location "/"}]
+ (with-redefs [remote-sync.task/last-version (constantly "previous-version")]
+ (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ "Conflicts should not block when last-imported-version is set")))))))
+
+(deftest import!-no-conflict-when-local-only
+ (testing "having local entities but no matching remote entities is not a conflict"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "Local Snippet Only"}]
+ (let [result (impl/import! (source.p/snapshot (test-helpers/create-mock-source)) task-id)]
+ (is (= :success (:status result))))))))
+
+(deftest import!-no-conflict-when-remote-only
+ (testing "having remote entities but no matching local entities is not a conflict"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source)) task-id)]
+ (is (= :success (:status result))
+ "Should not trigger conflict when only remote has entities"))))
+
+(deftest import!-library-conflict-requires-correct-type
+ (testing "a Collection with library entity_id but type!=library does not trigger conflict"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)
+ (mt/with-temp [:model/Collection _ {:name "Not A Library"
+ :type nil
+ :entity_id collection/library-entity-id
+ :location "/"}]
+ (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))))))))
+
+;; --- Spec-driven conflict detection tests ---
+
+(deftest import!-returns-conflict-details-test
+ (testing "import! returns detailed conflict information in :conflict-details"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)
+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"
+ :type "library"
+ :entity_id collection/library-entity-id
+ :location "/"}]
+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (test-helpers/generate-collection-yaml collection/library-entity-id "Another Library")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (= #{"Library"} (:conflicts result))
+ "Should return backward-compatible :conflicts set")
+ (is (seq (:conflict-details result))
+ "Should include detailed conflict information")
+ (is (some #(= :library-conflict (:type %)) (:conflict-details result))
+ "Should have library-conflict type in details"))))))
+
+(deftest import!-transforms-conflict-test
+ (testing "import! detects transforms conflict when local has transforms and import has transforms"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Transform _ {:name "Local Transform"}]
+ (let [test-files {"main" {"collections/test-coll-_/test-coll.yaml"
+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
+ "transforms/test-transform-_/test-transform.yaml"
+ (test-helpers/generate-transform-yaml "test-transform-xxxxx" "Remote Transform")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (contains? (:conflicts result) "Transforms"))
+ (is (some #(= :transforms-conflict (:type %)) (:conflict-details result))))))))
+
+(deftest import!-snippets-conflict-test
+ (testing "import! detects snippets conflict when local has snippets and import has snippets"
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "Local Snippet"}]
+ (let [test-files {"main" {"collections/test-coll-_/test-coll.yaml"
+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
+ "snippets/test-snippet-_/test-snippet.yaml"
+ (test-helpers/generate-snippet-yaml "test-snippet-xxxxxxx" "Remote Snippet" "SELECT 1")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :conflict (:status result)))
+ (is (contains? (:conflicts result) "Snippets"))
+ (is (some #(= :snippets-conflict (:type %)) (:conflict-details result))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
index ff790f27884..daa5c7622ed 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
@@ -276,26 +276,6 @@ is_sample: false
name entity-id (str/replace (u/lower-case-en name) #"\s+" "_")
entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
-(defn- generate-snippet-yaml
- "Generates YAML content for a NativeQuerySnippet."
- [entity-id name content & {:keys [collection-id]}]
- (format "name: %s
-description: null
-entity_id: %s
-content: '%s'
-archived: false
-template_tags: null
-created_at: '2024-08-28T09:46:18.671622Z'
-creator_id: [email protected]
-collection_id: %s
-serdes/meta:
-- id: %s
- label: %s
- model: NativeQuerySnippet
-"
- name entity-id content (or collection-id "null")
- entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
-
(deftest export-includes-snippets-when-library-synced-test
(testing "Export includes snippets when Library is synced"
(collections.tu/with-library-synced
@@ -345,7 +325,7 @@ serdes/meta:
test-files {"main" {(str "collections/" coll-entity-id "_snippets_collection/" coll-entity-id "_snippets_collection.yaml")
(generate-snippets-namespace-collection-yaml coll-entity-id "Snippets Collection")
(str "snippets/" coll-entity-id "_snippets_collection/" snippet-entity-id "_test_snippet.yaml")
- (generate-snippet-yaml snippet-entity-id "Test Snippet" "SELECT 42" :collection-id coll-entity-id)}}
+ (test-helpers/generate-snippet-yaml snippet-entity-id "Test Snippet" "SELECT 42" :collection-id coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result))
@@ -430,7 +410,7 @@ serdes/meta:
snippet-removal-path (first (filter #(str/includes? % snippet-eid) removal-paths))
_ (is (some? snippet-removal-path) "Should have a removal path for the archived snippet")
initial-files {"main" {(str snippet-removal-path ".yaml")
- (generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}
+ (test-helpers/generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
(is (= :success (:status result))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 31e8f9b80f6..059e1f440e8 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -4,6 +4,7 @@
[clojure.string :as str]
[clojure.test :as t]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -284,9 +285,42 @@ width: fixed
(when (seq old-models)
(t2/insert! :model/RemoteSyncTask old-models))))))
+(def ^:private builtin-python-library
+ "The built-in PythonLibrary created by migration. Recreated by fixture after cleanup."
+ {:path "common.py" :source "" :entity_id transforms-python/builtin-entity-id})
+
+(defn- ensure-builtin-python-library!
+ "Ensures the built-in common.py PythonLibrary exists, creating it if missing."
+ []
+ (when-not (t2/exists? :model/PythonLibrary :path (:path builtin-python-library))
+ (t2/insert! :model/PythonLibrary builtin-python-library)))
+
+(defn clean-optional-feature-models
+ "Test fixture that cleans Transform, TransformTag, and PythonLibrary tables to prevent
+ conflict detection during first-import tests. Preserves built-in TransformTags and
+ recreates the built-in common.py PythonLibrary after cleanup."
+ [f]
+ (let [old-transforms (t2/select :model/Transform)
+ old-tags (t2/select :model/TransformTag :built_in_type nil)
+ old-libs (t2/select :model/PythonLibrary)]
+ (try
+ (t2/delete! :model/TransformTag :built_in_type nil)
+ (t2/delete! :model/Transform)
+ (t2/delete! :model/PythonLibrary)
+ (f)
+ (finally
+ (t2/delete! :model/TransformTag :built_in_type nil)
+ (t2/delete! :model/Transform)
+ (t2/delete! :model/PythonLibrary)
+ (when (seq old-transforms) (t2/insert! :model/Transform old-transforms))
+ (when (seq old-tags) (t2/insert! :model/TransformTag old-tags))
+ (when (seq old-libs) (t2/insert! :model/PythonLibrary old-libs))
+ (ensure-builtin-python-library!)))))
+
(def clean-remote-sync-state
- "Composed test fixture that ensures both RemoteSyncObject and RemoteSyncTask tables are clean."
- (t/compose-fixtures clean-object clean-task-table))
+ "Composed test fixture that ensures RemoteSyncObject, RemoteSyncTask, and optional feature
+ model tables (Transform, TransformTag, PythonLibrary) are clean."
+ (t/compose-fixtures clean-object (t/compose-fixtures clean-task-table clean-optional-feature-models)))
(defn generate-table-yaml
"Generate YAML content for a table with the given `table-name` and `db-name`.
@@ -582,3 +616,23 @@ serdes/meta:
table-name
eid
(str/replace (u/lower-case-en measure-name) #"\s+" "_"))))
+
+(defn generate-snippet-yaml
+ "Generates YAML content for a NativeQuerySnippet."
+ [entity-id name content & {:keys [collection-id]}]
+ (format "name: %s
+description: null
+entity_id: %s
+content: '%s'
+archived: false
+template_tags: null
+created_at: '2024-08-28T09:46:18.671622Z'
+creator_id: [email protected]
+collection_id: %s
+serdes/meta:
+- id: %s
+ label: %s
+ model: NativeQuerySnippet
+"
+ name entity-id content (or collection-id "null")
+ entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index 3bf94ff11f4..be52c6d80dc 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -10,6 +10,7 @@
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.spec :as spec]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
+ [metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.collections.models.collection :as collection]
[metabase.events.core :as events]
[metabase.search.core :as search]
@@ -454,10 +455,11 @@ is_sample: false
(generate-transforms-namespace-collection-yaml coll-entity-id "Transforms Collection")
(str "collections/" coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id coll-entity-id)}}
- mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result))
- (str "Import should succeed. Result: " result))
+ mock-source (test-helpers/create-mock-source :initial-files test-files)]
+ (testing "fails with `conflict` status because local transforms will be deleted"
+ (is (= :conflict (:status (impl/import! (source.p/snapshot mock-source) task-id)))))
+ (testing "passing `force? true` allows overriding the conflict."
+ (is (= :success (:status (impl/import! (source.p/snapshot mock-source) task-id :force? true)))))
(is (t2/exists? :model/Collection :id coll-id)
"Transforms collection should still exist")
(is (not (t2/exists? :model/Transform :id local-transform-id))
@@ -487,7 +489,7 @@ is_sample: false
(let [test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_transforms/" remote-coll-entity-id "_remote_transforms.yaml")
(generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
+ result (impl/import! (source.p/snapshot mock-source) task-id :force? true)]
(is (= :success (:status result))
(str "Import should succeed. Result: " result))
(is (not (t2/exists? :model/Transform :id local-transform-id))
@@ -516,148 +518,144 @@ serdes/meta:
entity-id
entity-id))
-(defn- with-clean-python-library
- "Ensures PythonLibrary table is clean before and after running the test function.
- Saves and restores any existing entries."
- [f]
- (let [existing (t2/select :model/PythonLibrary)]
- (try
- (t2/delete! :model/PythonLibrary)
- (f)
- (finally
- (t2/delete! :model/PythonLibrary)
- (when (seq existing)
- (t2/insert! :model/PythonLibrary existing))))))
-
(deftest python-library-event-creates-sync-object-when-setting-enabled-test
(testing "Creating a PythonLibrary creates a RemoteSyncObject entry when remote-sync-transforms is enabled"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "PythonLibrary"
- :model_id (:id library))
- "PythonLibrary should be tracked when remote-sync-transforms is enabled"))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "PythonLibrary"
+ :model_id (:id library))
+ "PythonLibrary should be tracked when remote-sync-transforms is enabled"))))))
(deftest python-library-event-ignored-when-setting-disabled-test
(testing "PythonLibrary events are ignored when remote-sync-transforms is disabled"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms false
- remote-sync-enabled true]
- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
- (is (not (t2/exists? :model/RemoteSyncObject
- :model_type "PythonLibrary"
- :model_id (:id library)))
- "PythonLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms false
+ remote-sync-enabled true]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
+ (is (not (t2/exists? :model/RemoteSyncObject
+ :model_type "PythonLibrary"
+ :model_id (:id library)))
+ "PythonLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))
(deftest python-library-event-updates-sync-object-test
(testing "Updating a PythonLibrary updates the RemoteSyncObject entry when setting is enabled"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
- (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
- {:status "synced"})
- (t2/update! :model/PythonLibrary (:id library) {:source "# updated"})
- (let [entry (t2/select-one :model/RemoteSyncObject
- :model_type "PythonLibrary"
- :model_id (:id library))]
- (is (= "update" (:status entry))
- "PythonLibrary should have 'update' status after modification")))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
+ (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
+ {:status "synced"})
+ (t2/update! :model/PythonLibrary (:id library) {:source "# updated"})
+ (let [entry (t2/select-one :model/RemoteSyncObject
+ :model_type "PythonLibrary"
+ :model_id (:id library))]
+ (is (= "update" (:status entry))
+ "PythonLibrary should have 'update' status after modification")))))))
(deftest export-includes-python-library-when-setting-enabled-test
(testing "Export includes PythonLibrary when remote-sync-transforms is enabled"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-type :read-write
- remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-model-cleanup [:model/RemoteSyncTask]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
- library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# shared code"})]
- (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
- {:status "create"})
- (let [mock-source (test-helpers/create-mock-source)
- result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
- (is (= :success (:status result))
- (str "Export should succeed. Result: " result))
- (let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (some #(str/includes? % "python-libraries/") (keys files-after-export))
- "Export should include the PythonLibrary file")))))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
+ library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# shared code"})]
+ (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
+ {:status "create"})
+ (let [mock-source (test-helpers/create-mock-source)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
+ (is (= :success (:status result))
+ (str "Export should succeed. Result: " result))
+ (let [files-after-export (get @(:files-atom mock-source) "main")]
+ (is (some #(str/includes? % "python-libraries/") (keys files-after-export))
+ "Export should include the PythonLibrary file")))))))))
(deftest import-python-library-from-yaml-test
(testing "Import brings in PythonLibrary from YAML files"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-model-cleanup [:model/RemoteSyncTask]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- lib-entity-id (u/generate-nano-id)
- test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
- (generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
- mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result))
- (str "Import should succeed. Result: " result))
- (is (t2/exists? :model/PythonLibrary :path "common.py")
- "PythonLibrary should be imported")
- (when-let [library (t2/select-one :model/PythonLibrary :path "common.py")]
- (is (str/includes? (:source library) "def shared_func()")
- "PythonLibrary source should be imported correctly"))))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ lib-entity-id (u/generate-nano-id)
+ test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
+ (generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ snapshot (source.p/snapshot mock-source)
+ result (impl/import! snapshot task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (t2/exists? :model/PythonLibrary :path "common.py")
+ "PythonLibrary should be imported")
+ (when-let [library (t2/select-one :model/PythonLibrary :path "common.py")]
+ (is (str/includes? (:source library) "def shared_func()")
+ "PythonLibrary source should be imported correctly"))))))))
(deftest import-removes-python-library-not-on-remote-test
(testing "Import removes local PythonLibrary that doesn't exist on the remote"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-model-cleanup [:model/RemoteSyncTask]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]
- (is (t2/exists? :model/PythonLibrary :id (:id local-library))
- "Local PythonLibrary should exist before import")
- (let [test-files {"main" {}}
- mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result))
- (str "Import should succeed. Result: " result))
- (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))
- "Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]
+ (is (t2/exists? :model/PythonLibrary :id (:id local-library))
+ "Local PythonLibrary should exist before import")
+ (let [test-files {"main" {}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))
+ "Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))
+
+(deftest import-preserves-builtin-python-library-test
+ (testing "Import does NOT delete the built-in PythonLibrary even when it's not on remote"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ ;; Create the builtin library explicitly (fixture cleans PythonLibrary before test)
+ (t2/insert-returning-instance! :model/PythonLibrary
+ {:path "common.py"
+ :source "# builtin"
+ :entity_id transforms-python/builtin-entity-id})
+ (is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)
+ "Built-in PythonLibrary should exist before import")
+ (let [test-files {"main" {}} ;; Empty remote - no python libraries
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)
+ "Built-in PythonLibrary should NOT be deleted after import"))))))))
(deftest import-replaces-python-library-with-remote-version-test
(testing "Import updates local PythonLibrary when remote has same entity_id"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-model-cleanup [:model/RemoteSyncTask]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local version"})
- local-entity-id (:entity_id local-library)]
- (is (t2/exists? :model/PythonLibrary :path "common.py")
- "Local PythonLibrary should exist before import")
- (let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")
- (generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
- mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
- (is (= :success (:status result))
- (str "Import should succeed. Result: " result))
- (let [library (t2/select-one :model/PythonLibrary :path "common.py")]
- (is (some? library) "PythonLibrary should still exist")
- (is (str/includes? (:source library) "# remote version")
- "PythonLibrary source should be updated with remote version")))))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local version"})
+ local-entity-id (:entity_id local-library)]
+ (is (t2/exists? :model/PythonLibrary :path "common.py")
+ "Local PythonLibrary should exist before import")
+ (let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")
+ (generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (let [library (t2/select-one :model/PythonLibrary :path "common.py")]
+ (is (some? library) "PythonLibrary should still exist")
+ (is (str/includes? (:source library) "# remote version")
+ "PythonLibrary source should be updated with remote version")))))))))
;;; ------------------------------------------- Transform Sync Behavior Tests -------------------------------------------
@@ -726,7 +724,9 @@ serdes/meta:
(str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
+ result-without-force (impl/import! (source.p/snapshot mock-source) task-id)
+ result (impl/import! (source.p/snapshot mock-source) task-id {:force? true})]
+ (is (= :conflict (:status result-without-force)))
(is (= :success (:status result))
(str "Import should succeed. Result: " result))
(is (settings/remote-sync-transforms)
@@ -805,7 +805,7 @@ serdes/meta:
(str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
+ result (impl/import! (source.p/snapshot mock-source) task-id :force? true)]
(is (= :success (:status result))
(str "Import should succeed. Result: " result))
(is (settings/remote-sync-transforms)
@@ -819,40 +819,38 @@ serdes/meta:
(deftest build-all-removal-paths-includes-all-transforms-on-setting-disable-test
(testing "build-all-removal-paths returns paths for all transforms content when sentinel RSO has 'delete' status"
- (with-clean-python-library
- (fn []
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
- :namespace collection/transforms-ns
- :location "/"}
- :model/Transform {transform-id :id transform-eid :entity_id} {:name "Test Transform"
- :collection_id coll-id}
- :model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"
- :built_in_type nil}]
- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})
- lib-eid (:entity_id library)]
- (is (t2/exists? :model/Transform :id transform-id))
- (is (t2/exists? :model/TransformTag :id tag-id))
- (is (t2/exists? :model/PythonLibrary :id (:id library)))
- (let [paths-before (spec/build-all-removal-paths)]
- (is (not (some #(str/includes? % transform-eid) paths-before))
- "Transform should not be in removal paths before setting is disabled"))
- (settings/sync-transform-tracking! true)
- (settings/sync-transform-tracking! false)
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id
- :status "delete")
- "Sentinel RSO should exist with 'delete' status")
- (let [paths-after (spec/build-all-removal-paths)]
- (is (some #(str/includes? % transform-eid) paths-after)
- "Transform should be in removal paths after setting is disabled")
- (is (some #(str/includes? % tag-eid) paths-after)
- "TransformTag should be in removal paths after setting is disabled")
- (is (some #(str/includes? % lib-eid) paths-after)
- "PythonLibrary should be in removal paths after setting is disabled"))))))))))
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
+ :namespace collection/transforms-ns
+ :location "/"}
+ :model/Transform {transform-id :id transform-eid :entity_id} {:name "Test Transform"
+ :collection_id coll-id}
+ :model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"
+ :built_in_type nil}]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})
+ lib-eid (:entity_id library)]
+ (is (t2/exists? :model/Transform :id transform-id))
+ (is (t2/exists? :model/TransformTag :id tag-id))
+ (is (t2/exists? :model/PythonLibrary :id (:id library)))
+ (let [paths-before (spec/build-all-removal-paths)]
+ (is (not (some #(str/includes? % transform-eid) paths-before))
+ "Transform should not be in removal paths before setting is disabled"))
+ (settings/sync-transform-tracking! true)
+ (settings/sync-transform-tracking! false)
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id settings/transforms-root-id
+ :status "delete")
+ "Sentinel RSO should exist with 'delete' status")
+ (let [paths-after (spec/build-all-removal-paths)]
+ (is (some #(str/includes? % transform-eid) paths-after)
+ "Transform should be in removal paths after setting is disabled")
+ (is (some #(str/includes? % tag-eid) paths-after)
+ "TransformTag should be in removal paths after setting is disabled")
+ (is (some #(str/includes? % lib-eid) paths-after)
+ "PythonLibrary should be in removal paths after setting is disabled"))))))))
(deftest build-all-removal-paths-excludes-builtin-transform-tags-test
(testing "build-all-removal-paths respects :conditions and excludes built-in tags"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 3b996b0471e..56966e2af90 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -221,7 +221,7 @@
io/input-stream)
ba (#'api.serialization/ba-copy res)]
(testing "Archive contains correct number of files with proper log entries"
- (is (= 12
+ (is (= 13
(with-open [tar (open-tar ba)]
(count
(for [^TarArchiveEntry e (u.compress/entries tar)
@@ -229,7 +229,7 @@
(do
(condp re-find (.getName e)
#"/export.log$" (testing "Log contains extract and store entries"
- (is (= (+ #_extract 11 #_store 11)
+ (is (= (+ #_extract 12 #_store 12)
(count (line-seq (io/reader tar))))))
nil)
(.getName e))))))))
@@ -243,7 +243,7 @@
"settings" false
"field_values" false
"duration_ms" (every-pred number? pos?)
- "count" 11
+ "count" 12
"error_count" 0
"source" "api"
"secrets" false
@@ -273,7 +273,7 @@
{:request-options {:headers {"content-type" "multipart/form-data"}}}
{:file ba}))]
(testing "Log contains imported entity types"
- (is (= #{"Collection" "Dashboard" "Card" "TransformJob" "TransformTag"}
+ (is (= #{"Collection" "Dashboard" "Card" "PythonLibrary" "TransformJob" "TransformTag"}
(log-types (line-seq (io/reader (io/input-stream res)))))))
(testing "Entities are restored in the database"
@@ -285,8 +285,8 @@
"direction" "import"
"duration_ms" pos?
"source" "api"
- "models" "Card,Collection,Dashboard,TransformJob,TransformTag"
- "count" 11
+ "models" "Card,Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"
+ "count" 12
"error_count" 0
"success" true
"error_message" nil}
@@ -349,7 +349,7 @@
:continue_on_error true)
log (slurp (io/input-stream res))]
(testing "Log shows loaded entities and error"
- (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag"}
+ (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "PythonLibrary"}
(log-types (str/split-lines log))))
(is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log)))
@@ -359,9 +359,9 @@
"direction" "import"
"source" "api"
"duration_ms" int?
- "count" 10
+ "count" 11
"error_count" 1
- "models" "Collection,Dashboard,TransformJob,TransformTag"}
+ "models" "Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"}
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))))
(deftest import-invalid-archive-test
@@ -429,7 +429,7 @@
(doseq [^TarArchiveEntry e (u.compress/entries tar)]
(condp re-find (.getName e)
#"/export.log$" (testing "Log shows extract entries, error, and store entries"
- (is (= (+ #_extract 11 #_error 1 #_store 10)
+ (is (= (+ #_extract 12 #_error 1 #_store 11)
(count (line-seq (io/reader tar))))))
nil))))
@@ -442,7 +442,7 @@
"settings" false
"field_values" false
"duration_ms" pos?
- "count" 10
+ "count" 11
"error_count" 1
"source" "api"
"secrets" false
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index 48b704477f0..3cbc52ef5ae 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -2971,6 +2971,21 @@ databaseChangeLog:
constraintName: fk_metabase_table_transform_id
onDelete: SET NULL
+ - changeSet:
+ id: v59.2026-02-05T12:00:00
+ author: edpaget
+ comment: Add conflicts column to remote_sync_task table
+ changes:
+ - addColumn:
+ tableName: remote_sync_task
+ columns:
+ - column:
+ name: conflicts
+ type: ${text.type}
+ remarks: JSON serialized list of conflicts detected during import
+ constraints:
+ nullable: true
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index d77829a3a80..cbc41a9a8a7 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -179,6 +179,16 @@
[]
(t2/select-one :model/Collection :type library-collection-type))
+(def library-entity-id
+ "The entity_id for the Library collection."
+ "librarylibrarylibrary")
+
+(def ^:private library-data-entity-id
+ "librarylibrarydatadat")
+
+(def ^:private library-metrics-entity-id
+ "librarylibrarymetrics")
+
(defn create-library-collection!
"Create the Library collection. Returns Created collection. Throws if it already exists."
[]
@@ -186,14 +196,17 @@
(throw (ex-info "Library already exists" {})))
(let [library (t2/insert-returning-instance! :model/Collection {:name "Library"
:type library-collection-type
- :location "/"})
+ :location "/"
+ :entity_id library-entity-id})
base-location (str "/" (:id library) "/")
data (t2/insert-returning-instance! :model/Collection {:name "Data"
:type library-data-collection-type
- :location base-location})
+ :location base-location
+ :entity_id library-data-entity-id})
metrics (t2/insert-returning-instance! :model/Collection {:name "Metrics"
:type library-metrics-collection-type
- :location base-location})]
+ :location base-location
+ :entity_id library-metrics-entity-id})]
(doseq [col [library data metrics]]
(t2/delete! :model/Permissions :collection_id (:id col))
(perms/grant-collection-read-permissions! (perms/all-users-group) col)