The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Report conflicts on transforms/snippets/lib (#68506)
John Swanson00c573febe68
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex b76caf6e338..77a5fa49e65 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1589,7 +1589,6 @@enterprise/transforms-pythonenterprise/workspaces}}-types{:team "Querying":api #{metabase.types.core@@ -2252,7 +2251,8 @@tasktask-historyutil- enterprise/serialization}}+ enterprise/serialization+ enterprise/transforms-python}}enterprise/sandbox{:team "UX West"@@ -2420,6 +2420,7 @@enterprise/transforms-python{:team "Querying":api #{metabase-enterprise.transforms-python.api+ metabase-enterprise.transforms-python.coremetabase-enterprise.transforms-python.initmetabase-enterprise.transforms-python.python-runner}:uses #{analyticsdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex d7b0d7ce827..b4348c41dd5 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -23,20 +23,15 @@[toucan2.core :as t2])(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))-(def ^:private transform-models- "Models that indicate transforms content in a snapshot."- #{"Transform" "TransformTag" "PythonLibrary"})-(defn- snapshot-has-transforms?"Checks if the snapshot contains any Transform, TransformTag, or PythonLibrary entities.Used to auto-enable remote-sync-transforms setting during import.Uses the ingestable to list all entities and checks their :model metadata."[ingestable]- (let [serdes-paths (serialization/ingest-list ingestable)]- (some (fn [path]- (some #(transform-models (:model %)) path))- serdes-paths)))+ (let [serdes-paths (serialization/ingest-list ingestable)+ models-present (spec/models-in-import serdes-paths)]+ (some spec/transform-models models-present)))(defn- sync-objects!"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and@@ -51,6 +46,19 @@(when (seq all-inserts)(t2/insert! :model/RemoteSyncObject all-inserts))))+(defn- build-entity-id-where-clause+ "Builds a HoneySQL WHERE clause for entity_id filtering.+ Combines the imported entity-ids exclusion with any spec-level entity_id condition."+ [entity-ids spec-entity-id-condition]+ (let [imported-condition (when (seq entity-ids)+ [:not-in :entity_id entity-ids])+ spec-condition (when spec-entity-id-condition+ (let [[op value] spec-entity-id-condition]+ [op :entity_id value]))+ conditions (filterv some? [imported-condition spec-condition])]+ (when (pos? (count conditions))+ (into [:and] conditions))))+(defn- remove-unsynced!"Deletes any remote sync content that was NOT part of the import.@@ -68,22 +76,32 @@(doseq [[model-key model-spec] (spec/specs-for-deletion):let [serdes-model (:model-type model-spec)entity-ids (get by-entity-id serdes-model [])- entity-id-clause (if (seq entity-ids)- [:not-in entity-ids]- :entity_id)+ spec-entity-id-condition (get-in model-spec [:conditions :entity_id])+ entity-id-where (build-entity-id-where-clause entity-ids spec-entity-id-condition)scope-key (get-in model-spec [:removal :scope-key])- extra-conditions (into [] cat (:conditions model-spec))]]+ ;; Get non-entity_id conditions from spec+ other-conditions (into [] cat (dissoc (:conditions model-spec) :entity_id))]](if scope-key;; Collection-scoped: delete only within synced collections(when (seq synced-collection-ids)+ (if entity-id-where+ (apply t2/delete! model-key+ {:where [:and+ [:in scope-key synced-collection-ids]+ entity-id-where]}+ other-conditions)+ (apply t2/delete! model-key+ scope-key [:in synced-collection-ids]+ other-conditions)))+ ;; Global: delete by entity_id+ (if entity-id-where(apply t2/delete! model-key- scope-key [:in synced-collection-ids]- :entity_id entity-id-clause- extra-conditions))- ;; Global: delete by entity_id only- (apply t2/delete! model-key- :entity_id entity-id-clause- extra-conditions))))+ {:where entity-id-where}+ other-conditions)+ ;; No entity_id conditions - delete all (respecting other spec conditions if any)+ (if (seq other-conditions)+ (apply t2/delete! model-key other-conditions)+ (t2/delete! model-key))))))(defn source-error-message"Constructs user-friendly error messages from remote sync source exceptions.@@ -132,6 +150,37 @@:details {:error-type (type e)}})))+(defn- get-conflicts+ "Detects conflicts that would prevent or complicate import.+ Returns a map with :conflicts (detailed list) and :summary (set of category names).++ Conflict types detected:+ - :entity-id-conflict - Items with existing entity IDs that are NOT already synced+ - :library-conflict - First import only, local Library exists, import has Library+ - :transforms-not-enabled - Import has Transform/TransformTag/PythonLibrary, setting disabled+ - :snippets-without-library - Import has NativeQuerySnippet, Library not remote-synced"+ [ingestable first-import?]+ (let [ingest-list (serialization/ingest-list ingestable)+ imported-data (spec/extract-imported-entities ingest-list)+ models-present (spec/models-in-import ingest-list)+ ;; TODO (epaget 2026-02-02) -- entity-id conflict checking (detect unsynced local entities with matching entity_ids)+ feature-conflicts (spec/check-feature-conflicts models-present)+ library-conflict (when-let [local-library (t2/select-one :model/Collection :type collection/library-collection-type)]+ (when (and first-import?+ (contains? (get-in imported-data [:by-entity-id "Collection"] #{})+ collection/library-entity-id)+ (not (t2/exists? :model/RemoteSyncObject+ :model_type "Collection"+ :model_id (:id local-library))))+ {:type :library-conflict+ :category "Library"+ :message "Import contains Library but local instance has an unsynced Library collection"}))+ all-conflicts (concat+ feature-conflicts+ (when library-conflict [library-conflict]))]+ {:conflicts (vec all-conflicts)+ :summary (into #{} (map :category) all-conflicts)}))+(defn import!"Imports and reloads Metabase entities from a remote snapshot.@@ -150,18 +199,32 @@(if snapshot(try(let [snapshot-version (source.p/version snapshot)- last-imported-version (remote-sync.task/last-version)]- (if (and (not force?) (= last-imported-version snapshot-version))+ last-imported-version (remote-sync.task/last-version)+ first-import? (nil? last-imported-version)+ path-filters [#"collections/.*" #"databases/.*" #"actions/.*"+ #"transforms/.*" #"python-libraries/.*" #"snippets/.*"]+ base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})+ has-transforms? (snapshot-has-transforms? base-ingestable)+ {:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)+ ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)]++ (cond+ (and first-import? (not force?) (seq conflicts))+ (u/prog1 {:status :conflict+ :version (source.p/version snapshot)+ :conflicts summary ; Keep backward compatibility: return set of category names+ :conflict-details conflicts ; New: detailed conflict info+ :message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)}+ (log/infof (:message <>)))++ (and (not force?) (= last-imported-version snapshot-version))(u/prog1 {:status :success:version (source.p/version snapshot):message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}(log/infof (:message <>)))- (let [path-filters [#"collections/.*" #"databases/.*" #"actions/.*"- #"transforms/.*" #"python-libraries/.*" #"snippets/.*"]- base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})- has-transforms? (snapshot-has-transforms? base-ingestable)- ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)- load-result (serdes/with-cache++ :else+ (let [load-result (serdes/with-cache(serialization/load-metabase! ingestable-snapshot))seen-paths (:seen load-result)imported-data (spec/extract-imported-entities seen-paths)]@@ -309,10 +372,11 @@(defn handle-task-result!"Handles the outcome of running import! or export! by updating the RemoteSyncTask record.- Takes a result map with a :status key (either :success or :error) and optional :message key, a RemoteSyncTask ID,- and an optional branch name. On success, updates the remote-sync-branch setting (if branch provided), marks the- task complete, and invalidates the remote changes cache. On error, marks the task as failed with the error message.- For any other status, marks the task as failed with 'Unexpected Error'."+ Takes a result map with a :status key (either :success, :conflict, or :error) and optional :message key, a+ RemoteSyncTask ID, and an optional branch name. On success, updates the remote-sync-branch setting (if branch+ provided), marks the task complete, and invalidates the remote changes cache. On conflict, sets the version and+ stores the conflicts. On error, marks the task as failed with the error message. For any other status, marks the+ task as failed with 'Unexpected Error'."[result task-id & [branch]](case (:status result):success (do@@ -321,6 +385,9 @@(settings/remote-sync-branch! branch))(remote-sync.task/complete-sync-task! task-id))(invalidate-remote-changes-cache!))+ :conflict (do+ (remote-sync.task/set-version! task-id (:version result))+ (remote-sync.task/conflict-sync-task! task-id (:conflicts result))):error (remote-sync.task/fail-sync-task! task-id (:message result))(remote-sync.task/fail-sync-task! task-id "Unexpected Error")))@@ -351,8 +418,8 @@(defn async-import!"Imports remote-synced collections from a remote source repository asynchronously.- Takes a branch name to import from, a force? boolean (if true, imports even if there are unsaved changes), and an- import-args map of additional arguments to pass to the import function. Checks for dirty changes and throws an+ Takes a branch name to import from, a force? boolean (if true, imports even if there are unsaved changes or conflicts),+ and an import-args map of additional arguments to pass to the import function. Checks for dirty changes and throws anexception if force? is false and changes exist.Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if therediff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.cljindex 2d9760dff0e..bf2b6fcb8f4 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj@@ -20,6 +20,9 @@(derive :model/RemoteSyncTask :metabase/model)+(t2/deftransforms :model/RemoteSyncTask+ {:conflicts mi/transform-json})+(declare current-task)(t2/define-before-insert :model/RemoteSyncTask@@ -208,6 +211,30 @@(t/< (:last_progress_report_at task)(t/minus (t/offset-date-time) (t/millis (settings/remote-sync-task-time-limit-ms))))))+(defn conflict?+ "Checks if a task ended with conflicts.++ Takes a RemoteSyncTask instance.++ Returns true if the task has conflicts stored, false otherwise."+ [task]+ (and (some? (:ended_at task))+ (false? (:cancelled task))+ (nil? (:error_message task))+ (some? (:conflicts task))))++(defn conflict-sync-task!+ "Marks a sync task as having conflicts.++ Takes the ID of the sync task and a collection of conflicts (vector of strings).+ Conflicts are automatically serialized to JSON via the model transform.++ Returns the number of rows updated (should be 1 if successful)."+ [task-id conflicts]+ (t2/update! :model/RemoteSyncTask task-id+ {:ended_at (mi/now)+ :conflicts conflicts}))+;;; ------------------------------------------- Hydration -------------------------------------------(methodical/defmethod t2/batched-hydrate [:model/RemoteSyncTask :initiated_by_user]@@ -222,6 +249,7 @@(for [task tasks](assoc task :status (cond(failed? task) :errored+ (conflict? task) :conflict(successful? task) :successful(cancelled? task) :cancelled(timed-out? task) :timed-outdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.cljindex 7039012f2a1..8cac8bb5229 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj@@ -5,7 +5,7 @@(def TaskStatus"Status of a remote sync task."- [:enum :running :successful :errored :cancelled :timed-out])+ [:enum :running :successful :errored :cancelled :timed-out :conflict])(def TaskType"Type of remote sync task."@@ -24,8 +24,38 @@[:version {:optional true} [:maybe :string]][:cancelled {:optional true} [:maybe :boolean]][:error_message {:optional true} [:maybe :string]]+ [:conflicts {:optional true} [:maybe [:sequential :string]]][:status TaskStatus]])+;;; ------------------------------------------- Conflict Schemas -------------------------------------------++(def ConflictType+ "Type of import conflict detected during pre-flight check."+ [:enum+ :entity-id-conflict ; Local entity exists with same entity_id but not in RemoteSyncObject+ :library-conflict ; First import, local Library exists, import has Library+ :transforms-conflict ; Local has transforms AND import has transforms+ :snippets-conflict ; Local has snippets AND import has snippets+ :dirty]) ; RemoteSyncObject has items with status != "synced"++(def ConflictDetail+ "Schema for detailed conflict information."+ [:map+ [:type ConflictType]+ [:category {:optional true} :string]+ [:count {:optional true} pos-int?]+ [:entity-ids {:optional true} [:set :string]]+ [:message {:optional true} :string]])++(def ConflictResponse+ "Schema for conflict error response from async-import!."+ [:map+ [:status-code [:= 400]]+ [:conflicts [:= true]]+ [:conflict-type {:optional true} ConflictType]+ [:conflict-details {:optional true} [:sequential ConflictDetail]]+ [:conflict-summary {:optional true} [:set :string]]])+;;; ------------------------------------------- Dirty Item Schemas -------------------------------------------(def DirtyItemdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 2b4e140114c..5ad9be679e0 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -11,10 +11,13 @@To add a new syncable model, add a spec entry to `remote-sync-specs` andoptionally implement custom multimethods if the default behavior doesn't fit."(:require+ [clojure.set :as set][clojure.string :as str][clojure.walk :as walk][metabase-enterprise.remote-sync.settings :as rs-settings]+ [metabase-enterprise.transforms-python.core :as transforms-python][metabase.collections.core :as collections]+ [metabase.collections.models.collection :as collection][metabase.models.serialization :as serdes][metabase.settings.core :as setting][metabase.util :as u]@@ -304,7 +307,8 @@:archived-key nil ; no archived field:tracking {:select-fields [:path]:field-mappings {:model_name :path}}- :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion+ :conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; exclude built-in common.py from sync+ :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion:all-on-setting-disable :remote-sync-transforms}:export-scope :all ; query for all instances:enabled? :remote-sync-transforms}})@@ -405,6 +409,111 @@[](into #{} (map :model-type (vals remote-sync-specs))))+;;; ---------------------------------------- Import Conflict Detection -----------------------------------------------++(defn optional-feature-specs+ "Returns specs for optional features, grouped by their enabling setting keyword.+ Optional features are those with a keyword `:enabled?` value (not `true`).+ Returns map of {setting-keyword {model-key spec}}."+ []+ (-> remote-sync-specs+ (->> (filter (fn [[_ spec]] (keyword? (:enabled? spec))))+ (group-by (fn [[_ spec]] (:enabled? spec))))+ (update-vals #(into {} %))))++(defn- setting->category+ "Converts a setting keyword to a human-readable conflict category name."+ [setting-kw]+ (case setting-kw+ :remote-sync-transforms "Transforms"+ :library-synced "Snippets"+ (str/capitalize (name setting-kw))))++(defn models-for-setting+ "Returns set of model-type strings for specs with the given `:enabled?` setting."+ [setting-kw]+ (->> remote-sync-specs+ (filter (fn [[_ spec]] (= (:enabled? spec) setting-kw)))+ (map (fn [[_ spec]] (:model-type spec)))+ (into #{})))++(def transform-models+ "Models that indicate transforms content in a snapshot.+ Derived from specs with `:enabled? :remote-sync-transforms`."+ (disj (models-for-setting :remote-sync-transforms) :model/TransformTag))++(defn models-in-import+ "Returns set of model-type strings present in the import.+ Takes an ingest-list (sequence of serdes paths from serialization/ingest-list)."+ [ingest-list]+ (->> ingest-list+ (map (fn [path] (:model (last path))))+ (into #{})))++(defn check-entity-id-conflicts+ "Checks if imported entity_ids exist locally but are NOT in RemoteSyncObject.+ Returns map of {model-type #{conflicting-entity-ids}}.++ Excludes the Library collection entity_id since that's handled by the library-conflict check."+ [imported-entity-ids-by-model]+ (into {}+ (for [[model-type entity-ids] imported-entity-ids-by-model+ :when (seq entity-ids)+ :let [spec (spec-for-model-type model-type)+ model-key (:model-key spec)]+ :when (and spec model-key (#{:entity-id :hybrid} (:identity spec)))+ :let [local-entity-ids (t2/select-fn-set :entity_id model-key :entity_id [:in entity-ids])+ tracked-entity-ids (when (seq local-entity-ids)+ (let [pks (t2/select-pks-vec model-key+ :entity_id [:in local-entity-ids])]+ (t2/select-fn-set+ (fn [rso]+ (:entity_id (t2/select-one model-key :id (:model_id rso))))+ :model/RemoteSyncObject+ :model_type model-type+ :model_id [:in pks])))+ conflicting-entity-ids (set/difference local-entity-ids (or tracked-entity-ids #{}))+ conflicting-entity-ids (if (= model-type "Collection")+ (disj conflicting-entity-ids collection/library-entity-id)+ conflicting-entity-ids)]+ :when (seq conflicting-entity-ids)]+ [model-type conflicting-entity-ids])))++(defn- has-unsynced-entities-for-feature?+ "Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.+ Excludes built-in TransformTags from the count since they are system-created and not user data."+ [specs-for-feature]+ (some (fn [[_ spec]]+ (let [model-key (:model-key spec)+ model-type (:model-type spec)+ ;; Exclude built-in entities from count (they are system-created, not user data)+ local-count (case model-key+ :model/TransformTag (t2/count model-key :built_in_type nil)+ :model/PythonLibrary (t2/count model-key :entity_id [:not= transforms-python/builtin-entity-id])+ (t2/count model-key))+ synced-count (t2/count :model/RemoteSyncObject :model_type model-type)]+ (and (pos? local-count)+ (> local-count synced-count))))+ specs-for-feature))++(defn check-feature-conflicts+ "Checks if import contains models that conflict with existing local entities that are NOT already remote synced.+ Derives conflict categories from specs with keyword `:enabled?` values (optional features).+ Only triggers conflict when local has unsynced entities AND import has matching entities.+ If local entities are already synced, dirty tracking handles conflicts instead."+ [models-present]+ (let [feature-groups (optional-feature-specs)]+ (into []+ (for [[setting-kw specs-for-feature] feature-groups+ :let [feature-model-types (into #{} (map (fn [[_ s]] (:model-type s))) specs-for-feature)]+ :when (some feature-model-types models-present)+ :when (has-unsynced-entities-for-feature? specs-for-feature)+ :let [category (setting->category setting-kw)]]+ {:type (keyword (str (u/lower-case-en category) "-conflict"))+ :category category+ :message (format "Import contains %s but local instance has unsynced %s"+ category category)}))))+;;; ------------------------------------------------ Eligibility Checking ----------------------------------------------(defn transforms-namespace-collection?diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/core.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/core.cljnew file mode 100644index 00000000000..9349e86ea95--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/core.clj@@ -0,0 +1,10 @@+(ns metabase-enterprise.transforms-python.core+ "Core namespace for Python transforms functionality.+ Re-exports commonly used vars from internal namespaces."+ (:require+ [metabase-enterprise.transforms-python.models.python-library :as python-library]+ [potemkin :as p]))++(p/import-vars+ [python-library+ builtin-entity-id])diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.cljindex 19f46ede548..19d881c0a22 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj@@ -50,6 +50,11 @@"Set of allowed library paths. Currently only 'common' is supported."#{"common.py"})+(def builtin-entity-id+ "The entity_id of the built-in common.py PythonLibrary created by migration.+ Used to protect it from deletion during remote-sync import."+ "cWWH9qJPvHNB3rP2vLZrK")+(defn- validate-path!"Validates that the given path is allowed. Throws an exception if not."[path]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 22aac37c8bd..fba8d057c2f 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -1137,3 +1137,166 @@ serdes/meta:"python-libraries path should be included in filters")(is (some #(str/includes? % "snippets") filter-strs)"snippets path should be included in filters"))))))))))))++(deftest import!-blocks-if-it-encounters-library-conflicts+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"+ :type "library"+ :entity_id collection/library-entity-id+ :location "/"}]+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"+ (test-helpers/generate-collection-yaml collection/library-entity-id "Another Library")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (= #{"Library"} (:conflicts result)))))))++(deftest import!-blocks-if-it-encounters-snippet-conflicts+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "Test Snippet"}]+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"+ (test-helpers/generate-snippet-yaml "blahblahblah" "A Snippet" "select 123")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (= #{"Snippets"} (:conflicts result)))))))++(deftest import!-blocks-if-it-encounters-transform-conflicts+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Transform _ {:name "Test Transform"}]+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"+ (test-helpers/generate-transform-yaml "blahblahblah" "A Transform")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (= #{"Transforms"} (:conflicts result)))))))++(deftest import!-reports-multiple-conflicts+ (testing "when multiple conflict types exist, all are reported in the :conflicts set"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"+ :type "library"+ :entity_id collection/library-entity-id+ :location "/"}+ :model/NativeQuerySnippet _ {:name "Test Snippet"}+ :model/Transform _ {:name "Test Transform"}]+ (let [test-files {"main" {"collections/lib-_/lib.yaml"+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")+ "collections/snip-_/snip.yaml"+ (test-helpers/generate-snippet-yaml "snip-entity-id" "Remote Snippet" "select 1")+ "collections/trans-_/trans.yaml"+ (test-helpers/generate-transform-yaml "trans-entity-id" "Remote Transform")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (= #{"Library" "Transforms" "Snippets"} (:conflicts result))))))))++(deftest import!-force-bypasses-conflicts+ (testing "force?: true allows import to proceed despite conflicts"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"+ :type "library"+ :entity_id collection/library-entity-id+ :location "/"}]+ (let [test-files {"main" {"collections/lib-_/lib.yaml"+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id :force? true)]+ (is (= :success (:status result))))))))++(deftest import!-conflicts-only-block-initial-import+ (testing "when last-imported-version is set, conflicts do not block subsequent imports"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)+ (mt/with-temp [:model/Collection _ {:name "Local Library"+ :type "library"+ :entity_id collection/library-entity-id+ :location "/"}]+ (with-redefs [remote-sync.task/last-version (constantly "previous-version")]+ (let [test-files {"main" {"collections/lib-_/lib.yaml"+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ "Conflicts should not block when last-imported-version is set")))))))++(deftest import!-no-conflict-when-local-only+ (testing "having local entities but no matching remote entities is not a conflict"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "Local Snippet Only"}]+ (let [result (impl/import! (source.p/snapshot (test-helpers/create-mock-source)) task-id)]+ (is (= :success (:status result))))))))++(deftest import!-no-conflict-when-remote-only+ (testing "having remote entities but no matching local entities is not a conflict"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source)) task-id)]+ (is (= :success (:status result))+ "Should not trigger conflict when only remote has entities"))))++(deftest import!-library-conflict-requires-correct-type+ (testing "a Collection with library entity_id but type!=library does not trigger conflict"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)+ (mt/with-temp [:model/Collection _ {:name "Not A Library"+ :type nil+ :entity_id collection/library-entity-id+ :location "/"}]+ (let [test-files {"main" {"collections/lib-_/lib.yaml"+ (test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))))))))++;; --- Spec-driven conflict detection tests ---++(deftest import!-returns-conflict-details-test+ (testing "import! returns detailed conflict information in :conflict-details"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (t2/delete! :model/Collection :entity_id collection/library-entity-id)+ (mt/with-temp [:model/Collection _ {:name "Test Library Collection"+ :type "library"+ :entity_id collection/library-entity-id+ :location "/"}]+ (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"+ (test-helpers/generate-collection-yaml collection/library-entity-id "Another Library")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (= #{"Library"} (:conflicts result))+ "Should return backward-compatible :conflicts set")+ (is (seq (:conflict-details result))+ "Should include detailed conflict information")+ (is (some #(= :library-conflict (:type %)) (:conflict-details result))+ "Should have library-conflict type in details"))))))++(deftest import!-transforms-conflict-test+ (testing "import! detects transforms conflict when local has transforms and import has transforms"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Transform _ {:name "Local Transform"}]+ (let [test-files {"main" {"collections/test-coll-_/test-coll.yaml"+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")+ "transforms/test-transform-_/test-transform.yaml"+ (test-helpers/generate-transform-yaml "test-transform-xxxxx" "Remote Transform")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (contains? (:conflicts result) "Transforms"))+ (is (some #(= :transforms-conflict (:type %)) (:conflict-details result))))))))++(deftest import!-snippets-conflict-test+ (testing "import! detects snippets conflict when local has snippets and import has snippets"+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "Local Snippet"}]+ (let [test-files {"main" {"collections/test-coll-_/test-coll.yaml"+ (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")+ "snippets/test-snippet-_/test-snippet.yaml"+ (test-helpers/generate-snippet-yaml "test-snippet-xxxxxxx" "Remote Snippet" "SELECT 1")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :conflict (:status result)))+ (is (contains? (:conflicts result) "Snippets"))+ (is (some #(= :snippets-conflict (:type %)) (:conflict-details result))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.cljindex ff790f27884..daa5c7622ed 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj@@ -276,26 +276,6 @@ is_sample: falsename entity-id (str/replace (u/lower-case-en name) #"\s+" "_")entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))-(defn- generate-snippet-yaml- "Generates YAML content for a NativeQuerySnippet."- [entity-id name content & {:keys [collection-id]}]- (format "name: %s-description: null-entity_id: %s-content: '%s'-archived: false-template_tags: null-created_at: '2024-08-28T09:46:18.671622Z'-creator_id: [email protected]-collection_id: %s-serdes/meta:-- id: %s- label: %s- model: NativeQuerySnippet-"- name entity-id content (or collection-id "null")- entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))-(deftest export-includes-snippets-when-library-synced-test(testing "Export includes snippets when Library is synced"(collections.tu/with-library-synced@@ -345,7 +325,7 @@ serdes/meta:test-files {"main" {(str "collections/" coll-entity-id "_snippets_collection/" coll-entity-id "_snippets_collection.yaml")(generate-snippets-namespace-collection-yaml coll-entity-id "Snippets Collection")(str "snippets/" coll-entity-id "_snippets_collection/" snippet-entity-id "_test_snippet.yaml")- (generate-snippet-yaml snippet-entity-id "Test Snippet" "SELECT 42" :collection-id coll-entity-id)}}+ (test-helpers/generate-snippet-yaml snippet-entity-id "Test Snippet" "SELECT 42" :collection-id coll-entity-id)}}mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result))@@ -430,7 +410,7 @@ serdes/meta:snippet-removal-path (first (filter #(str/includes? % snippet-eid) removal-paths))_ (is (some? snippet-removal-path) "Should have a removal path for the archived snippet")initial-files {"main" {(str snippet-removal-path ".yaml")- (generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}+ (test-helpers/generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}mock-source (test-helpers/create-mock-source :initial-files initial-files)result (impl/export! (source.p/snapshot mock-source) task-id "Test export")](is (= :success (:status result))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 31e8f9b80f6..059e1f440e8 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -4,6 +4,7 @@[clojure.string :as str][clojure.test :as t][metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.transforms-python.core :as transforms-python][metabase.util :as u][toucan2.core :as t2]))@@ -284,9 +285,42 @@ width: fixed(when (seq old-models)(t2/insert! :model/RemoteSyncTask old-models))))))+(def ^:private builtin-python-library+ "The built-in PythonLibrary created by migration. Recreated by fixture after cleanup."+ {:path "common.py" :source "" :entity_id transforms-python/builtin-entity-id})++(defn- ensure-builtin-python-library!+ "Ensures the built-in common.py PythonLibrary exists, creating it if missing."+ []+ (when-not (t2/exists? :model/PythonLibrary :path (:path builtin-python-library))+ (t2/insert! :model/PythonLibrary builtin-python-library)))++(defn clean-optional-feature-models+ "Test fixture that cleans Transform, TransformTag, and PythonLibrary tables to prevent+ conflict detection during first-import tests. Preserves built-in TransformTags and+ recreates the built-in common.py PythonLibrary after cleanup."+ [f]+ (let [old-transforms (t2/select :model/Transform)+ old-tags (t2/select :model/TransformTag :built_in_type nil)+ old-libs (t2/select :model/PythonLibrary)]+ (try+ (t2/delete! :model/TransformTag :built_in_type nil)+ (t2/delete! :model/Transform)+ (t2/delete! :model/PythonLibrary)+ (f)+ (finally+ (t2/delete! :model/TransformTag :built_in_type nil)+ (t2/delete! :model/Transform)+ (t2/delete! :model/PythonLibrary)+ (when (seq old-transforms) (t2/insert! :model/Transform old-transforms))+ (when (seq old-tags) (t2/insert! :model/TransformTag old-tags))+ (when (seq old-libs) (t2/insert! :model/PythonLibrary old-libs))+ (ensure-builtin-python-library!)))))+(def clean-remote-sync-state- "Composed test fixture that ensures both RemoteSyncObject and RemoteSyncTask tables are clean."- (t/compose-fixtures clean-object clean-task-table))+ "Composed test fixture that ensures RemoteSyncObject, RemoteSyncTask, and optional feature+ model tables (Transform, TransformTag, PythonLibrary) are clean."+ (t/compose-fixtures clean-object (t/compose-fixtures clean-task-table clean-optional-feature-models)))(defn generate-table-yaml"Generate YAML content for a table with the given `table-name` and `db-name`.@@ -582,3 +616,23 @@ serdes/meta:table-nameeid(str/replace (u/lower-case-en measure-name) #"\s+" "_"))))++(defn generate-snippet-yaml+ "Generates YAML content for a NativeQuerySnippet."+ [entity-id name content & {:keys [collection-id]}]+ (format "name: %s+description: null+entity_id: %s+content: '%s'+archived: false+template_tags: null+created_at: '2024-08-28T09:46:18.671622Z'+creator_id: [email protected]+collection_id: %s+serdes/meta:+- id: %s+ label: %s+ model: NativeQuerySnippet+"+ name entity-id content (or collection-id "null")+ entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex 3bf94ff11f4..be52c6d80dc 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -10,6 +10,7 @@[metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.spec :as spec][metabase-enterprise.remote-sync.test-helpers :as test-helpers]+ [metabase-enterprise.transforms-python.core :as transforms-python][metabase.collections.models.collection :as collection][metabase.events.core :as events][metabase.search.core :as search]@@ -454,10 +455,11 @@ is_sample: false(generate-transforms-namespace-collection-yaml coll-entity-id "Transforms Collection")(str "collections/" coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id coll-entity-id)}}- mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result))- (str "Import should succeed. Result: " result))+ mock-source (test-helpers/create-mock-source :initial-files test-files)]+ (testing "fails with `conflict` status because local transforms will be deleted"+ (is (= :conflict (:status (impl/import! (source.p/snapshot mock-source) task-id)))))+ (testing "passing `force? true` allows overriding the conflict."+ (is (= :success (:status (impl/import! (source.p/snapshot mock-source) task-id :force? true)))))(is (t2/exists? :model/Collection :id coll-id)"Transforms collection should still exist")(is (not (t2/exists? :model/Transform :id local-transform-id))@@ -487,7 +489,7 @@ is_sample: false(let [test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_transforms/" remote-coll-entity-id "_remote_transforms.yaml")(generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")}}mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]+ result (impl/import! (source.p/snapshot mock-source) task-id :force? true)](is (= :success (:status result))(str "Import should succeed. Result: " result))(is (not (t2/exists? :model/Transform :id local-transform-id))@@ -516,148 +518,144 @@ serdes/meta:entity-identity-id))-(defn- with-clean-python-library- "Ensures PythonLibrary table is clean before and after running the test function.- Saves and restores any existing entries."- [f]- (let [existing (t2/select :model/PythonLibrary)]- (try- (t2/delete! :model/PythonLibrary)- (f)- (finally- (t2/delete! :model/PythonLibrary)- (when (seq existing)- (t2/insert! :model/PythonLibrary existing))))))-(deftest python-library-event-creates-sync-object-when-setting-enabled-test(testing "Creating a PythonLibrary creates a RemoteSyncObject entry when remote-sync-transforms is enabled"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]- (is (t2/exists? :model/RemoteSyncObject- :model_type "PythonLibrary"- :model_id (:id library))- "PythonLibrary should be tracked when remote-sync-transforms is enabled"))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]+ (is (t2/exists? :model/RemoteSyncObject+ :model_type "PythonLibrary"+ :model_id (:id library))+ "PythonLibrary should be tracked when remote-sync-transforms is enabled"))))))(deftest python-library-event-ignored-when-setting-disabled-test(testing "PythonLibrary events are ignored when remote-sync-transforms is disabled"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms false- remote-sync-enabled true]- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]- (is (not (t2/exists? :model/RemoteSyncObject- :model_type "PythonLibrary"- :model_id (:id library)))- "PythonLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]+ (is (not (t2/exists? :model/RemoteSyncObject+ :model_type "PythonLibrary"+ :model_id (:id library)))+ "PythonLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))(deftest python-library-event-updates-sync-object-test(testing "Updating a PythonLibrary updates the RemoteSyncObject entry when setting is enabled"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]- (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}- {:status "synced"})- (t2/update! :model/PythonLibrary (:id library) {:source "# updated"})- (let [entry (t2/select-one :model/RemoteSyncObject- :model_type "PythonLibrary"- :model_id (:id library))]- (is (= "update" (:status entry))- "PythonLibrary should have 'update' status after modification")))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]+ (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}+ {:status "synced"})+ (t2/update! :model/PythonLibrary (:id library) {:source "# updated"})+ (let [entry (t2/select-one :model/RemoteSyncObject+ :model_type "PythonLibrary"+ :model_id (:id library))]+ (is (= "update" (:status entry))+ "PythonLibrary should have 'update' status after modification")))))))(deftest export-includes-python-library-when-setting-enabled-test(testing "Export includes PythonLibrary when remote-sync-transforms is enabled"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-type :read-write- remote-sync-transforms true- remote-sync-enabled true]- (mt/with-model-cleanup [:model/RemoteSyncTask]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})- library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# shared code"})]- (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}- {:status "create"})- (let [mock-source (test-helpers/create-mock-source)- result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]- (is (= :success (:status result))- (str "Export should succeed. Result: " result))- (let [files-after-export (get @(:files-atom mock-source) "main")]- (is (some #(str/includes? % "python-libraries/") (keys files-after-export))- "Export should include the PythonLibrary file")))))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})+ library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# shared code"})]+ (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}+ {:status "create"})+ (let [mock-source (test-helpers/create-mock-source)+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]+ (is (= :success (:status result))+ (str "Export should succeed. Result: " result))+ (let [files-after-export (get @(:files-atom mock-source) "main")]+ (is (some #(str/includes? % "python-libraries/") (keys files-after-export))+ "Export should include the PythonLibrary file")))))))))(deftest import-python-library-from-yaml-test(testing "Import brings in PythonLibrary from YAML files"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (mt/with-model-cleanup [:model/RemoteSyncTask]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- lib-entity-id (u/generate-nano-id)- test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")- (generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}- mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result))- (str "Import should succeed. Result: " result))- (is (t2/exists? :model/PythonLibrary :path "common.py")- "PythonLibrary should be imported")- (when-let [library (t2/select-one :model/PythonLibrary :path "common.py")]- (is (str/includes? (:source library) "def shared_func()")- "PythonLibrary source should be imported correctly"))))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ lib-entity-id (u/generate-nano-id)+ test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")+ (generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ snapshot (source.p/snapshot mock-source)+ result (impl/import! snapshot task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (t2/exists? :model/PythonLibrary :path "common.py")+ "PythonLibrary should be imported")+ (when-let [library (t2/select-one :model/PythonLibrary :path "common.py")]+ (is (str/includes? (:source library) "def shared_func()")+ "PythonLibrary source should be imported correctly"))))))))(deftest import-removes-python-library-not-on-remote-test(testing "Import removes local PythonLibrary that doesn't exist on the remote"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (mt/with-model-cleanup [:model/RemoteSyncTask]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]- (is (t2/exists? :model/PythonLibrary :id (:id local-library))- "Local PythonLibrary should exist before import")- (let [test-files {"main" {}}- mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result))- (str "Import should succeed. Result: " result))- (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))- "Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]+ (is (t2/exists? :model/PythonLibrary :id (:id local-library))+ "Local PythonLibrary should exist before import")+ (let [test-files {"main" {}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))+ "Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))++(deftest import-preserves-builtin-python-library-test+ (testing "Import does NOT delete the built-in PythonLibrary even when it's not on remote"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ ;; Create the builtin library explicitly (fixture cleans PythonLibrary before test)+ (t2/insert-returning-instance! :model/PythonLibrary+ {:path "common.py"+ :source "# builtin"+ :entity_id transforms-python/builtin-entity-id})+ (is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)+ "Built-in PythonLibrary should exist before import")+ (let [test-files {"main" {}} ;; Empty remote - no python libraries+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)+ "Built-in PythonLibrary should NOT be deleted after import"))))))))(deftest import-replaces-python-library-with-remote-version-test(testing "Import updates local PythonLibrary when remote has same entity_id"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (mt/with-model-cleanup [:model/RemoteSyncTask]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local version"})- local-entity-id (:entity_id local-library)]- (is (t2/exists? :model/PythonLibrary :path "common.py")- "Local PythonLibrary should exist before import")- (let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")- (generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}- mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]- (is (= :success (:status result))- (str "Import should succeed. Result: " result))- (let [library (t2/select-one :model/PythonLibrary :path "common.py")]- (is (some? library) "PythonLibrary should still exist")- (is (str/includes? (:source library) "# remote version")- "PythonLibrary source should be updated with remote version")))))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local version"})+ local-entity-id (:entity_id local-library)]+ (is (t2/exists? :model/PythonLibrary :path "common.py")+ "Local PythonLibrary should exist before import")+ (let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")+ (generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (let [library (t2/select-one :model/PythonLibrary :path "common.py")]+ (is (some? library) "PythonLibrary should still exist")+ (is (str/includes? (:source library) "# remote version")+ "PythonLibrary source should be updated with remote version")))))))));;; ------------------------------------------- Transform Sync Behavior Tests -------------------------------------------@@ -726,7 +724,9 @@ serdes/meta:(str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]+ result-without-force (impl/import! (source.p/snapshot mock-source) task-id)+ result (impl/import! (source.p/snapshot mock-source) task-id {:force? true})]+ (is (= :conflict (:status result-without-force)))(is (= :success (:status result))(str "Import should succeed. Result: " result))(is (settings/remote-sync-transforms)@@ -805,7 +805,7 @@ serdes/meta:(str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]+ result (impl/import! (source.p/snapshot mock-source) task-id :force? true)](is (= :success (:status result))(str "Import should succeed. Result: " result))(is (settings/remote-sync-transforms)@@ -819,40 +819,38 @@ serdes/meta:(deftest build-all-removal-paths-includes-all-transforms-on-setting-disable-test(testing "build-all-removal-paths returns paths for all transforms content when sentinel RSO has 'delete' status"- (with-clean-python-library- (fn []- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"- :namespace collection/transforms-ns- :location "/"}- :model/Transform {transform-id :id transform-eid :entity_id} {:name "Test Transform"- :collection_id coll-id}- :model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"- :built_in_type nil}]- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})- lib-eid (:entity_id library)]- (is (t2/exists? :model/Transform :id transform-id))- (is (t2/exists? :model/TransformTag :id tag-id))- (is (t2/exists? :model/PythonLibrary :id (:id library)))- (let [paths-before (spec/build-all-removal-paths)]- (is (not (some #(str/includes? % transform-eid) paths-before))- "Transform should not be in removal paths before setting is disabled"))- (settings/sync-transform-tracking! true)- (settings/sync-transform-tracking! false)- (is (t2/exists? :model/RemoteSyncObject- :model_type "Collection"- :model_id settings/transforms-root-id- :status "delete")- "Sentinel RSO should exist with 'delete' status")- (let [paths-after (spec/build-all-removal-paths)]- (is (some #(str/includes? % transform-eid) paths-after)- "Transform should be in removal paths after setting is disabled")- (is (some #(str/includes? % tag-eid) paths-after)- "TransformTag should be in removal paths after setting is disabled")- (is (some #(str/includes? % lib-eid) paths-after)- "PythonLibrary should be in removal paths after setting is disabled"))))))))))+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"+ :namespace collection/transforms-ns+ :location "/"}+ :model/Transform {transform-id :id transform-eid :entity_id} {:name "Test Transform"+ :collection_id coll-id}+ :model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"+ :built_in_type nil}]+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})+ lib-eid (:entity_id library)]+ (is (t2/exists? :model/Transform :id transform-id))+ (is (t2/exists? :model/TransformTag :id tag-id))+ (is (t2/exists? :model/PythonLibrary :id (:id library)))+ (let [paths-before (spec/build-all-removal-paths)]+ (is (not (some #(str/includes? % transform-eid) paths-before))+ "Transform should not be in removal paths before setting is disabled"))+ (settings/sync-transform-tracking! true)+ (settings/sync-transform-tracking! false)+ (is (t2/exists? :model/RemoteSyncObject+ :model_type "Collection"+ :model_id settings/transforms-root-id+ :status "delete")+ "Sentinel RSO should exist with 'delete' status")+ (let [paths-after (spec/build-all-removal-paths)]+ (is (some #(str/includes? % transform-eid) paths-after)+ "Transform should be in removal paths after setting is disabled")+ (is (some #(str/includes? % tag-eid) paths-after)+ "TransformTag should be in removal paths after setting is disabled")+ (is (some #(str/includes? % lib-eid) paths-after)+ "PythonLibrary should be in removal paths after setting is disabled"))))))))(deftest build-all-removal-paths-excludes-builtin-transform-tags-test(testing "build-all-removal-paths respects :conditions and excludes built-in tags"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 3b996b0471e..56966e2af90 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -221,7 +221,7 @@io/input-stream)ba (#'api.serialization/ba-copy res)](testing "Archive contains correct number of files with proper log entries"- (is (= 12+ (is (= 13(with-open [tar (open-tar ba)](count(for [^TarArchiveEntry e (u.compress/entries tar)@@ -229,7 +229,7 @@(do(condp re-find (.getName e)#"/export.log$" (testing "Log contains extract and store entries"- (is (= (+ #_extract 11 #_store 11)+ (is (= (+ #_extract 12 #_store 12)(count (line-seq (io/reader tar))))))nil)(.getName e))))))))@@ -243,7 +243,7 @@"settings" false"field_values" false"duration_ms" (every-pred number? pos?)- "count" 11+ "count" 12"error_count" 0"source" "api""secrets" false@@ -273,7 +273,7 @@{:request-options {:headers {"content-type" "multipart/form-data"}}}{:file ba}))](testing "Log contains imported entity types"- (is (= #{"Collection" "Dashboard" "Card" "TransformJob" "TransformTag"}+ (is (= #{"Collection" "Dashboard" "Card" "PythonLibrary" "TransformJob" "TransformTag"}(log-types (line-seq (io/reader (io/input-stream res)))))))(testing "Entities are restored in the database"@@ -285,8 +285,8 @@"direction" "import""duration_ms" pos?"source" "api"- "models" "Card,Collection,Dashboard,TransformJob,TransformTag"- "count" 11+ "models" "Card,Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"+ "count" 12"error_count" 0"success" true"error_message" nil}@@ -349,7 +349,7 @@:continue_on_error true)log (slurp (io/input-stream res))](testing "Log shows loaded entities and error"- (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag"}+ (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "PythonLibrary"}(log-types (str/split-lines log))))(is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log)))@@ -359,9 +359,9 @@"direction" "import""source" "api""duration_ms" int?- "count" 10+ "count" 11"error_count" 1- "models" "Collection,Dashboard,TransformJob,TransformTag"}+ "models" "Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"}(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))))(deftest import-invalid-archive-test@@ -429,7 +429,7 @@(doseq [^TarArchiveEntry e (u.compress/entries tar)](condp re-find (.getName e)#"/export.log$" (testing "Log shows extract entries, error, and store entries"- (is (= (+ #_extract 11 #_error 1 #_store 10)+ (is (= (+ #_extract 12 #_error 1 #_store 11)(count (line-seq (io/reader tar))))))nil))))@@ -442,7 +442,7 @@"settings" false"field_values" false"duration_ms" pos?- "count" 10+ "count" 11"error_count" 1"source" "api""secrets" falsediff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 48b704477f0..3cbc52ef5ae 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -2971,6 +2971,21 @@ databaseChangeLog:constraintName: fk_metabase_table_transform_idonDelete: SET NULL+ - changeSet:+ id: v59.2026-02-05T12:00:00+ author: edpaget+ comment: Add conflicts column to remote_sync_task table+ changes:+ - addColumn:+ tableName: remote_sync_task+ columns:+ - column:+ name: conflicts+ type: ${text.type}+ remarks: JSON serialized list of conflicts detected during import+ constraints:+ nullable: true+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex d77829a3a80..cbc41a9a8a7 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -179,6 +179,16 @@[](t2/select-one :model/Collection :type library-collection-type))+(def library-entity-id+ "The entity_id for the Library collection."+ "librarylibrarylibrary")++(def ^:private library-data-entity-id+ "librarylibrarydatadat")++(def ^:private library-metrics-entity-id+ "librarylibrarymetrics")+(defn create-library-collection!"Create the Library collection. Returns Created collection. Throws if it already exists."[]@@ -186,14 +196,17 @@(throw (ex-info "Library already exists" {})))(let [library (t2/insert-returning-instance! :model/Collection {:name "Library":type library-collection-type- :location "/"})+ :location "/"+ :entity_id library-entity-id})base-location (str "/" (:id library) "/")data (t2/insert-returning-instance! :model/Collection {:name "Data":type library-data-collection-type- :location base-location})+ :location base-location+ :entity_id library-data-entity-id})metrics (t2/insert-returning-instance! :model/Collection {:name "Metrics":type library-metrics-collection-type- :location base-location})]+ :location base-location+ :entity_id library-metrics-entity-id})](doseq [col [library data metrics]](t2/delete! :model/Permissions :collection_id (:id col))(perms/grant-collection-read-permissions! (perms/all-users-group) col)