Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

feat: enable transform setting on first import (#68807)

Edward Paget9c003314beef
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 4fdbb040381..04f9ba5cef0 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -23,41 +23,55 @@
[toucan2.core :as t2])
(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
+(def ^:private transform-models
+ "Models that indicate transforms content in a snapshot."
+ #{"Transform" "TransformTag" "PythonLibrary"})
+
+(defn- snapshot-has-transforms?
+ "Checks if the snapshot contains any Transform, TransformTag, or PythonLibrary entities.
+ Used to auto-enable remote-sync-transforms setting during import.
+
+ Uses the ingestable to list all entities and checks their :model metadata."
+ [ingestable]
+ (let [serdes-paths (serialization/ingest-list ingestable)]
+ (some (fn [path]
+ (some #(transform-models (:model %)) path))
+ serdes-paths)))
+
(defn- sync-objects!
"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and
inserts new ones for each imported entity, marking them as 'synced' with the given timestamp.
- Takes a timestamp instant, a map of imported entities grouped by model name (for standard models with entity_id),
- and separate path collections for Table and Field models which use name-based identity.
+ Takes a timestamp instant and imported-data map from spec/extract-imported-entities.
Uses the spec system to determine how to query and build sync objects for each model type."
- [timestamp imported-entities-by-model table-paths field-paths]
+ [timestamp imported-data]
(t2/delete! :model/RemoteSyncObject)
- (let [all-inserts (spec/sync-all-entities! timestamp imported-entities-by-model table-paths field-paths)]
+ (let [all-inserts (spec/sync-all-entities! timestamp imported-data)]
(when (seq all-inserts)
(t2/insert! :model/RemoteSyncObject all-inserts))))
(defn- remove-unsynced!
"Deletes any remote sync content that was NOT part of the import.
- Takes a sequence of remote-synced collection IDs and a map of imported entities grouped by model name. For each
- entity-id based model, deletes entities whose entity_id is not in the imported set.
+ Takes a sequence of remote-synced collection IDs and imported-data map from spec/extract-imported-entities.
+ For each entity-id based model, deletes entities whose entity_id is not in the imported set.
Models with :scope-key in their spec are scoped to synced collections (using :id for Collection, :collection_id
for others). Models without :scope-key (like TransformTag) are deleted globally by entity_id.
+ Path-based models (Table, Field) are not removed here - they are controlled by published table settings.
+
Uses specs-for-deletion to process models in dependency order (models with FK references to
Collection are deleted before Collection itself)."
- [synced-collection-ids imported-entities-by-model]
+ [synced-collection-ids {:keys [by-entity-id]}]
(doseq [[model-key model-spec] (spec/specs-for-deletion)
:let [serdes-model (:model-type model-spec)
- entity-ids (get imported-entities-by-model serdes-model [])
+ entity-ids (get by-entity-id serdes-model [])
entity-id-clause (if (seq entity-ids)
[:not-in entity-ids]
:entity_id)
- ;; :scope-key determines collection scoping - nil means global
scope-key (get-in model-spec [:removal :scope-key])
- ;; Extra conditions to protect certain rows from removal (e.g. built-in TransformTags)
extra-conditions (into [] cat (get-in model-spec [:removal :conditions]))]]
(if scope-key
;; Collection-scoped: delete only within synced collections
@@ -142,47 +156,23 @@
:version (source.p/version snapshot)
:message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}
(log/infof (:message <>)))
- (let [path-filters (cond-> [#"collections/.*" #"databases/.*" #"actions/.*"]
- (settings/remote-sync-transforms)
- (conj #"transforms/.*" #"python-libraries/.*")
- (settings/library-is-remote-synced?)
- (conj #"snippets/.*"))
- ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters path-filters})
- (source.ingestable/wrap-progress-ingestable task-id 0.7))
+ (let [path-filters [#"collections/.*" #"databases/.*" #"actions/.*"
+ #"transforms/.*" #"python-libraries/.*" #"snippets/.*"]
+ base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
+ has-transforms? (snapshot-has-transforms? base-ingestable)
+ ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)
load-result (serdes/with-cache
(serialization/load-metabase! ingestable-snapshot))
seen-paths (:seen load-result)
- data-model-models #{"Table" "Field"}
- ;; For standard models (Collection, Card, Segment, etc.), extract entity_id as before
- imported-entities-by-model (->> seen-paths
- (remove #(data-model-models (:model (last %))))
- (map last)
- (group-by :model)
- (map (fn [[model entities]]
- [model (set (map :id entities))]))
- (into {}))
- ;; For Tables, capture full path for unique identification
- ;; Path format: [{:model "Database" :id db-name} {:model "Schema" :id schema}? {:model "Table" :id table-name}]
- table-paths (->> seen-paths
- (filter #(= "Table" (:model (last %))))
- (map (fn [path]
- {:db_name (-> path first :id)
- :schema (when (= 3 (count path)) (-> path second :id))
- :table_name (-> path last :id)})))
- ;; For Fields, path format includes Table
- ;; [{:model "Database"} {:model "Schema"}? {:model "Table"} {:model "Field" :id field-name}]
- field-paths (->> seen-paths
- (filter #(= "Field" (:model (last %))))
- (map (fn [path]
- (let [table-idx (dec (count path))]
- {:db_name (-> path first :id)
- :schema (when (> (count path) 3) (-> path second :id))
- :table_name (-> path (nth (dec table-idx)) :id)
- :field_name (-> path last :id)}))))]
+ imported-data (spec/extract-imported-entities seen-paths)]
(remote-sync.task/update-progress! task-id 0.8)
+ (when (and has-transforms?
+ (not (settings/remote-sync-transforms)))
+ (log/info "Detected transforms in remote source, enabling remote-sync-transforms setting")
+ (settings/remote-sync-transforms! true))
(t2/with-transaction [_conn]
- (remove-unsynced! (spec/all-syncable-collection-ids) imported-entities-by-model)
- (sync-objects! sync-timestamp imported-entities-by-model table-paths field-paths))
+ (remove-unsynced! (spec/all-syncable-collection-ids) imported-data)
+ (sync-objects! sync-timestamp imported-data))
(remote-sync.task/update-progress! task-id 0.95)
(remote-sync.task/set-version!
task-id
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index d0f7aa51247..88cebef4481 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -123,7 +123,11 @@
(defn- sync-transform-tracking-on-change
"Called when remote-sync-transforms setting changes."
[_old-value new-value]
- (sync-transform-tracking! (boolean new-value)))
+ ;; new-value may be a string "true"/"false" or a boolean, so we need to parse it
+ (let [enabled? (if (string? new-value)
+ (parse-boolean new-value)
+ (boolean new-value))]
+ (sync-transform-tracking! enabled?)))
(defsetting remote-sync-transforms
(deferred-tru "Whether to sync transforms via remote-sync. When enabled, all transforms, transform tags, and transform jobs are synced as a single unit (all-or-nothing).")
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 8cb0095046d..5becdc7c078 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -559,6 +559,67 @@
{:entity-id (:entity_id instance)
:path (select-keys instance path-keys)})
+;;; ----------------------------------------- Serdes Path Identity Extraction ------------------------------------------
+
+(def ^:private serdes-path-identity-hierarchy
+ "Hierarchy for extract-identity-from-serdes-path dispatch. Both :entity-id and :hybrid models
+ extract identity the same way (entity_id from last path element)."
+ (-> (make-hierarchy)
+ (derive :entity-id ::entity-id-extractor)
+ (derive :hybrid ::entity-id-extractor)))
+
+(defmulti extract-identity-from-serdes-path
+ "Extracts identity data from a serdes path based on the spec's identity strategy. For entity-id
+ and hybrid models, returns the entity_id string from the last path element. For path-based models
+ like Table and Field, returns a map with database, schema, and table/field names that can be used
+ to look up the entity."
+ {:arglists '([spec serdes-path])}
+ (fn [spec _path] (:identity spec))
+ :hierarchy #'serdes-path-identity-hierarchy)
+
+(defmethod extract-identity-from-serdes-path ::entity-id-extractor
+ [_ serdes-path]
+ (:id (last serdes-path)))
+
+(defmethod extract-identity-from-serdes-path :path
+ [_ serdes-path]
+ (let [path-map (into {} (map (fn [elem] [(keyword (u/lower-case-en (:model elem))) (:id elem)]) serdes-path))]
+ (cond-> {}
+ (contains? path-map :database) (assoc :db_name (:database path-map))
+ (contains? path-map :schema) (assoc :schema (:schema path-map))
+ (contains? path-map :table) (assoc :table_name (:table path-map))
+ (contains? path-map :field) (assoc :field_name (:field path-map)))))
+
+(defmethod extract-identity-from-serdes-path :default
+ [_ _]
+ nil)
+
+(defn extract-imported-entities
+ "Processes serdes paths from an import and extracts entity identities grouped by how they should be looked up.
+ Returns a map with :by-entity-id containing entity_ids grouped by model type, and :by-path containing
+ path lookup maps for models like Table and Field that use path-based identity."
+ [seen-paths]
+ (reduce
+ (fn [acc path]
+ (let [model-type (-> path last :model)]
+ (if-let [spec (spec-for-model-type model-type)]
+ (let [identity-type (:identity spec)
+ identity-data (extract-identity-from-serdes-path spec path)]
+ (if identity-data
+ (case identity-type
+ (:entity-id :hybrid)
+ (update-in acc [:by-entity-id model-type] (fnil conj #{}) identity-data)
+
+ :path
+ (update-in acc [:by-path (:model-key spec)] (fnil conj []) identity-data)
+
+ acc)
+ acc))
+ acc)))
+ {:by-entity-id {}
+ :by-path {}}
+ seen-paths))
+
;;; --------------------------------------------- Export Path Construction ---------------------------------------------
(defn- transform-entity-for-serdes
@@ -799,25 +860,18 @@
nil)
(defn sync-all-entities!
- "Syncs all entities based on specs. Takes:
- - timestamp: The sync timestamp
- - imported-entities-by-model: Map of model-type string -> set of entity_ids
- - table-paths: Collection of table path maps
- - field-paths: Collection of field path maps
-
- Returns a sequence of all sync objects to insert."
- [timestamp imported-entities-by-model table-paths field-paths]
+ "Builds RemoteSyncObject entries for all imported entities based on their specs. Iterates over enabled
+ specs and queries the database to hydrate the fields needed for each sync object. Returns a sequence
+ of maps ready for insertion into the RemoteSyncObject table."
+ [timestamp {:keys [by-entity-id by-path]}]
(into []
(for [[model-key spec] (enabled-specs)
:let [identity-type (:identity spec)
model-type (:model-type spec)
data (case identity-type
- :entity-id (get imported-entities-by-model model-type)
- :path (case model-key
- :model/Table table-paths
- :model/Field field-paths
- nil)
- :hybrid (get imported-entities-by-model model-type))]
+ :entity-id (get by-entity-id model-type)
+ :path (get by-path model-key)
+ :hybrid (get by-entity-id model-type))]
:when (seq data)
entity (query-entities-for-sync spec data timestamp)]
entity)))
@@ -835,7 +889,6 @@
[{:keys [export-scope]}]
(case (or export-scope :derived)
:root-collections
- ;; Collection model: query for root-level remote-synced + transforms-namespace + snippets-namespace collections
;; Excludes archived collections - their files are handled by the removal logic
(concat
(t2/select-fn-set (juxt (constantly "Collection") :id)
@@ -859,7 +912,6 @@
[:= :location "/"]
[:not :archived]]})))
:derived
- ;; Other collection-based models: no root query, derived from collection expansion
nil))
(defmethod query-export-roots :setting
@@ -867,27 +919,22 @@
(when (spec-enabled? spec)
(case export-scope
:root-only
- ;; Transform: root transforms (collection_id = nil)
(t2/select-fn-set (juxt (constantly model-type) :id) model-key :collection_id nil)
:all
- ;; TransformTag: all instances
(t2/select-fn-set (juxt (constantly model-type) :id) model-key)
- ;; Default: no root query
nil)))
-(defmethod query-export-roots :published-table [_] nil) ; Derived via serdes/descendants
-(defmethod query-export-roots :parent-table [_] nil) ; Derived via serdes/descendants
+(defmethod query-export-roots :published-table [_] nil)
+(defmethod query-export-roots :parent-table [_] nil)
(defmethod query-export-roots :library-synced
[{:keys [export-scope model-key model-type archived-key] :as spec}]
(when (spec-enabled? spec)
(case export-scope
:all
- ;; NativeQuerySnippet: all non-archived instances
(if archived-key
(t2/select-fn-set (juxt (constantly model-type) :id) model-key archived-key false)
(t2/select-fn-set (juxt (constantly model-type) :id) model-key))
- ;; Default: no root query
nil)))
(defmethod query-export-roots :default [_] nil)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
index 0175ae0e006..bb26f7e5aec 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
@@ -513,7 +513,7 @@
:model/Card card {:name "Test Card"
:dataset_query (mt/mbql-query venues)
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Card" #{(:entity_id card)}} nil nil)
+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Card" #{(:entity_id card)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))]
(is (= "synced" (:status initial-entry)))
@@ -551,7 +551,7 @@
:model/Collection normal-collection {:name "Normal"}
:model/Dashboard dashboard {:name "Test Dashboard"
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Dashboard" #{(:entity_id dashboard)}} nil nil)
+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Dashboard" #{(:entity_id dashboard)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id (:id dashboard))]
(is (= "synced" (:status initial-entry)))
@@ -585,7 +585,7 @@
(mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
:model/Collection normal-collection {:name "Normal"}
:model/Document document {:collection_id (u/the-id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Document" #{(:entity_id document)}} nil nil)
+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Document" #{(:entity_id document)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Document" :model_id (:id document))]
(is (= "synced" (:status initial-entry)))
@@ -616,7 +616,7 @@
(deftest existing-collection-type-changed-from-remote-synced-test
(testing "existing collection type changed from remote-synced is marked as removed"
(mt/with-temp [:model/Collection collection {:is_remote_synced true :name "Remote-Sync"}]
- (#'impl/sync-objects! (t/instant) {"Collection" #{(:entity_id collection)}} nil nil)
+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Collection" #{(:entity_id collection)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id collection))]
(is (= "synced" (:status initial-entry)))
(events/publish-event! :event/collection-update
@@ -765,7 +765,7 @@
:model/Collection normal-collection {:name "Normal Collection"}
:model/Timeline timeline {:name "Test Timeline"
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {"Timeline" #{(:entity_id timeline)}} nil nil)
+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Timeline" #{(:entity_id timeline)}}})
(is (= 1 (count (t2/select :model/RemoteSyncObject))))
(events/publish-event! :event/timeline-update
{:object (assoc timeline :collection_id (:id normal-collection))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 2ae1d5e5113..22aac37c8bd 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -5,6 +5,7 @@
[java-time.api :as t]
[metabase-enterprise.remote-sync.impl :as impl]
[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
+ [metabase-enterprise.remote-sync.settings :as remote-sync.settings]
[metabase-enterprise.remote-sync.source :as source]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
@@ -144,7 +145,6 @@
(testing "export! errors when there are no remote-synced collections"
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
- ;; Create a regular collection (not remote-synced) to verify it's not included
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}]
(mt/with-temporary-setting-values [remote-sync-transforms false]
(let [mock-source (test-helpers/create-mock-source)
@@ -194,27 +194,22 @@
(remote-sync.task/complete-sync-task! (:id export-task))
(is (= :success (:status export-result)))
- ;; Verify files were written to the mock source atom
(let [files-after-export (get @(:files-atom mock-main) "test-branch")]
(is (map? files-after-export))
(is (not-empty files-after-export))
- ;; Should have at least collection and card files
(is (some #(str/includes? % "collection") (keys files-after-export)))
(is (some #(str/includes? % "card") (keys files-after-export))))
(t2/delete! :model/RemoteSyncTask :id (:id export-task))
- ;; Then import - verify it succeeds and processes the exported files
(let [import-task (t2/with-connection [_conn (app-db/app-db) (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})])
import-result (impl/import! (source.p/snapshot mock-main) (:id import-task))]
(remote-sync.task/complete-sync-task! (:id import-task))
(is (= :success (:status import-result)))
(is (= "Successfully reloaded from git repository" (:message import-result)))
- ;; Verify the entities still exist after import
(is (t2/exists? :model/Collection :id coll-id))
(is (t2/exists? :model/Card :id card-id))
- ;; Verify the collection and card still have the correct attributes
(let [collection (t2/select-one :model/Collection :id coll-id)
card (t2/select-one :model/Card :id card-id)]
(is (= "Test Collection" (:name collection)))
@@ -239,7 +234,6 @@
result (impl/import! (source.p/snapshot mock-main) (:id import-task))]
(is (= :success (:status result)))
- ;; Verify the entities still exist (real cleanup would require more complex setup)
(is (t2/exists? :model/Card :id card1-id))
(is (not (t2/exists? :model/Collection :id coll2-id)))
(is (not (t2/exists? :model/Card :id card2-id))))))))
@@ -263,12 +257,10 @@
(swap! progress-calls conj {:task-id task-id :progress progress}))]
(let [result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- ;; Verify progress was called with expected values
(is (= 5 (count @progress-calls)))
(is (= task-id (:task-id (first @progress-calls))))
(is (= task-id (:task-id (second @progress-calls))))
(is (= task-id (:task-id (nth @progress-calls 2))))
- ;; Check progress values are in expected sequence
(is (= 0.7 (:progress (nth @progress-calls 2))))
(is (= 0.8 (:progress (nth @progress-calls 3))))
(is (= 0.95 (:progress (nth @progress-calls 4)))))))))))
@@ -288,11 +280,8 @@
(swap! progress-calls conj {:task-id task-id :progress progress}))]
(let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
- ;; Verify progress was called with expected values
- ;; 1 initial call at 0.3, then N calls during store! for each entity
(is (pos? (count @progress-calls)))
(is (= task-id (:task-id (first @progress-calls))))
- ;; Check first progress value is expected (before store!)
(is (= 0.3 (:progress (first @progress-calls)))))))))))))
(deftest import!-resets-remote-sync-object-table-test
@@ -452,10 +441,8 @@
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- ;; Active collection should be written by export
(is (some #(str/includes? % "active-coll-xxxxxxxxx") (keys files-after-export))
"Active collection files should exist after export")
- ;; Both removed collections should be deleted
(is (not (some #(str/includes? % "removed-coll-1xxxxxxx") (keys files-after-export)))
"First removed collection files should be deleted")
(is (not (some #(str/includes? % "removed-coll-2xxxxxxx") (keys files-after-export)))
@@ -582,7 +569,6 @@
:is_remote_synced true
:entity_id "active-coll-xxxxxxxxx"
:location "/"}]
- ;; Mark the table as 'removed' in RemoteSyncObject
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
@@ -613,7 +599,6 @@
:definition {:source-table table-id
:filter [:> [:field 1 nil] 0]}
:entity_id "test-segment-xxxxxxxx"}]
- ;; Mark the segment as 'removed' in RemoteSyncObject
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
@@ -646,18 +631,15 @@
:definition {:source-table table-id
:filter [:> [:field 1 nil] 0]}
:entity_id "test-segment-xxxxxxxx"}]
- ;; Create RemoteSyncObject entries with 'removed' status
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
{:model_type "Field" :model_id field-id :model_name "test-field" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}
{:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
- ;; Verify entries exist before export
(is (= 4 (t2/count :model/RemoteSyncObject)))
(let [mock-source (test-helpers/create-mock-source)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
- ;; After export, all entries should have status 'synced'
(is (= 4 (t2/count :model/RemoteSyncObject))
"All RemoteSyncObject entries should remain after export")
(let [table-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id table-id)
@@ -685,12 +667,10 @@
:is_remote_synced true
:entity_id "active-coll-xxxxxxxxx"
:location "/"}]
- ;; Mark the table as 'removed' in RemoteSyncObject
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
- (let [;; Path should include schema: databases/{db}/schemas/{schema}/tables/{table}
- initial-files {"main" {"databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"
+ (let [initial-files {"main" {"databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
@@ -729,7 +709,6 @@
:filter [:> [:field 2 nil] 0]}
:entity_id "archived-segment-xxxx"
:archived true}]
- ;; Track the collection so export runs
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])
(let [mock-source (test-helpers/create-mock-source)
@@ -737,7 +716,6 @@
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")
file-keys (keys files-after-export)]
- ;; File paths use entity_id, e.g. "active-segment-xxxxxx_active_segment.yaml"
(is (some #(str/includes? % "active-segment-xxxxxx") file-keys)
"Active segment should be exported")
(is (not (some #(str/includes? % "archived-segment-xxxx") file-keys))
@@ -762,11 +740,9 @@
:filter [:> [:field 1 nil] 0]}
:entity_id "archived-seg-xxxxxxxx"
:archived true}]
- ;; Mark the segment as 'delete' in RemoteSyncObject (simulating archival event)
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Segment" :model_id segment-id :model_name "Archived Segment" :model_table_id table-id :model_table_name "test-table" :status "delete" :status_changed_at (t/offset-date-time)}])
- ;; File path uses slugified name to match what serdes/storage-path generates
(let [initial-files {"main" {"databases/test-db/tables/test-table/segments/archived-seg-xxxxxxxx_archived_segment.yaml"
(test-helpers/generate-segment-yaml "Archived Segment" "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
@@ -799,7 +775,6 @@
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- ;; Verify RemoteSyncObject entries exist
(is (t2/exists? :model/RemoteSyncObject
:model_type "Collection"
:model_id coll-id
@@ -829,7 +804,6 @@
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- ;; Verify RemoteSyncObject entry exists for the field
(is (t2/exists? :model/RemoteSyncObject
:model_type "Field"
:model_id field-id
@@ -866,7 +840,6 @@
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- ;; Verify RemoteSyncObject entry exists for the segment
(is (t2/exists? :model/RemoteSyncObject
:model_type "Segment"
:model_id segment-id
@@ -890,7 +863,6 @@
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- ;; Verify RemoteSyncObject entry exists for the table
(is (t2/exists? :model/RemoteSyncObject
:model_type "Table"
:model_id table-id
@@ -954,7 +926,6 @@
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- ;; Verify RemoteSyncObject entry exists for the measure
(is (t2/exists? :model/RemoteSyncObject
:model_type "Measure"
:model_id measure-id
@@ -977,7 +948,6 @@
{:name "Test Measure"
:table_id table-id
:entity_id "test-measure-xxxxxxxx"}]
- ;; Mark the measure as 'removed' in RemoteSyncObject
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Measure" :model_id measure-id :model_name "Test Measure" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
@@ -1017,7 +987,6 @@
:table_id table-id
:entity_id "archived-measure-xxxx"
:archived true}]
- ;; Track the collection so export runs
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])
(let [mock-source (test-helpers/create-mock-source)
@@ -1025,7 +994,6 @@
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")
file-keys (keys files-after-export)]
- ;; File paths use entity_id, e.g. "active-measure-xxxxxx_active_measure.yaml"
(is (some #(str/includes? % "active-measure-xxxxxx") file-keys)
(str "Active measure should be exported. Keys: " (pr-str file-keys)))
(is (not (some #(str/includes? % "archived-measure-xxxx") file-keys))
@@ -1048,11 +1016,9 @@
:table_id table-id
:entity_id "archived-meas-xxxxxxx"
:archived true}]
- ;; Mark the measure as 'delete' in RemoteSyncObject (simulating archival event)
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Measure" :model_id measure-id :model_name "Archived Measure" :model_table_id table-id :model_table_name "test-table" :status "delete" :status_changed_at (t/offset-date-time)}])
- ;; File path uses slugified name to match what serdes/storage-path generates
(let [initial-files {"main" {"databases/test-db/tables/test-table/measures/archived-meas-xxxxxxx_archived_measure.yaml"
(test-helpers/generate-measure-yaml "Archived Measure" "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
@@ -1064,3 +1030,110 @@
(testing "RemoteSyncObject entry is cleaned up after export"
(is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Measure" :model_id measure-id)))
"RemoteSyncObject entry for archived measure should have synced status")))))))))
+
+;; Auto-enable transforms tests
+
+(deftest import!-auto-enables-transforms-setting-when-transforms-detected-test
+ (testing "import! auto-enables remote-sync-transforms setting only after successful import with transforms"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/RemoteSyncObject]
+ (mt/with-temporary-setting-values [remote-sync-transforms false
+ remote-sync-enabled true]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ transform-entity-id "auto-enable-trans-xxx"
+ test-files {"main" {(str "transforms/" transform-entity-id "_test_transform.yaml")
+ (test-helpers/generate-transform-yaml transform-entity-id "Test Transform")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)]
+ (is (false? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should be initially disabled")
+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ "Import should succeed")
+ (is (true? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should be auto-enabled after successful import with transforms"))))))))
+
+(deftest import!-auto-enables-transforms-setting-when-python-libraries-detected-test
+ (testing "import! auto-enables remote-sync-transforms setting only after successful import with python-libraries"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/PythonLibrary :model/RemoteSyncObject]
+ (mt/with-temporary-setting-values [remote-sync-transforms false
+ remote-sync-enabled true]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ lib-entity-id "auto-enable-lib-xxxxx"
+ test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
+ (format "path: uncommon.py
+source: |
+ # shared code
+ def shared_func():
+ return 42
+entity_id: %s
+created_at: '2024-08-28T09:46:18.671622Z'
+serdes/meta:
+- id: %s
+ model: PythonLibrary
+" lib-entity-id lib-entity-id)}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)]
+ (is (false? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should be initially disabled")
+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ "Import should succeed")
+ (is (true? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should be auto-enabled after successful import with python-libraries"))))))))
+
+(deftest import!-keeps-transforms-setting-disabled-when-no-transforms-present-test
+ (testing "import! keeps remote-sync-transforms setting disabled when no transforms are present"
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temporary-setting-values [remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "No Transforms Coll" :is_remote_synced true :entity_id "no-transforms-coll-xx" :location "/"}]
+ (let [test-files {"main" {"collections/no-transforms-coll-xx-_/no-transforms-coll-xx.yaml"
+ (test-helpers/generate-collection-yaml "no-transforms-coll-xx" "No Transforms Coll")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)]
+ (is (false? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should be initially disabled")
+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ (is (false? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should remain disabled when no transforms in remote")))))))))
+
+(deftest import!-does-not-disable-transforms-setting-when-already-enabled-test
+ (testing "import! does not modify remote-sync-transforms setting when it's already enabled"
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temporary-setting-values [remote-sync-transforms true]
+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "Already Enabled Coll" :is_remote_synced true :entity_id "already-enabled-collx" :location "/"}]
+ (let [test-files {"main" {"collections/already-enabled-collx-_/already-enabled-collx.yaml"
+ (test-helpers/generate-collection-yaml "already-enabled-collx" "Already Enabled Coll")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)]
+ (is (true? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should be initially enabled")
+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ (is (true? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should remain enabled even when no transforms in remote")))))))))
+
+(deftest import!-includes-all-optional-paths-regardless-of-settings-test
+ (testing "import! always includes all optional paths (transforms, python-libraries, snippets)"
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temporary-setting-values [remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "All Paths Coll" :is_remote_synced true :entity_id "all-paths-coll-xxxxxx" :location "/"}]
+ (let [test-files {"main" {"collections/all-paths-coll-xxxxxx-_/all-paths-coll-xxxxxx.yaml"
+ (test-helpers/generate-collection-yaml "all-paths-coll-xxxxxx" "All Paths Coll")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ paths-passed (atom nil)
+ original-fn source.p/->ingestable]
+ (with-redefs [source.p/->ingestable (fn [snapshot opts]
+ (reset! paths-passed (:path-filters opts))
+ (original-fn snapshot opts))]
+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ (when @paths-passed
+ (let [filter-strs (map str @paths-passed)]
+ (is (some #(str/includes? % "transforms") filter-strs)
+ "transforms path should be included in filters")
+ (is (some #(str/includes? % "python-libraries") filter-strs)
+ "python-libraries path should be included in filters")
+ (is (some #(str/includes? % "snippets") filter-strs)
+ "snippets path should be included in filters"))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index ef3604fe310..31e8f9b80f6 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -8,15 +8,8 @@
[toucan2.core :as t2]))
(defn generate-collection-yaml
- "Generates YAML content for a collection.
-
- Args:
- entity-id: The unique identifier for the collection.
- name: The name of the collection.
- parent-id: Optional parent collection ID.
-
- Returns:
- A string containing the YAML representation of the collection."
+ "Generate YAML content for a collection with the given `entity-id` and `name`.
+ Optionally accepts `:parent-id` for nested collections."
[entity-id name & {:keys [parent-id]}]
(format "name: %s
description: null
@@ -41,20 +34,10 @@ is_sample: false
(or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
(defn generate-v57-collection-yaml
- "Generates YAML content for a collection in v57 format.
-
- In v57, remote-synced collections used `type: remote-synced` instead of
- `is_remote_synced: true`. This helper is used to test backward compatibility
- when importing exports from v57 Metabase instances.
-
- Args:
- entity-id: The unique identifier for the collection.
- name: The name of the collection.
- parent-id: Optional parent collection ID.
- type: Optional type value (e.g., \"remote-synced\" for v57 remote-synced collections).
-
- Returns:
- A string containing the v57-format YAML representation of the collection."
+ "Generate YAML content for a collection in v57 format. In v57, remote-synced collections
+ used `type: remote-synced` instead of `is_remote_synced: true`. Used to test backward
+ compatibility when importing exports from v57 Metabase instances. Optionally accepts
+ `:parent-id` and `:type` (e.g., `\"remote-synced\"`)."
[entity-id name & {:keys [parent-id type]}]
(format "name: %s
description: null
@@ -79,15 +62,8 @@ is_sample: false
(or type "null") (or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
(defn generate-card-yaml
- "Generates YAML content for a card.
-
- Args:
- entity-id: The unique identifier for the card.
- name: The name of the card.
- collection-id: The ID of the collection containing this card.
-
- Returns:
- A string containing the YAML representation of the card."
+ "Generate YAML content for a card with the given `entity-id`, `name`, and `collection-id`.
+ Optionally accepts a `type` (defaults to `\"question\"`)."
[entity-id name collection-id & [type]]
(format "name: %s
description: null
@@ -127,16 +103,8 @@ document_id: null
name entity-id collection-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_") (or type "question")))
(defn generate-dashboard-yaml
- "Generates YAML content for a dashboard.
-
- Args:
- entity-id: The unique identifier for the dashboard.
- name: The name of the dashboard.
- collection-id: The ID of the collection containing this dashboard.
- dashcards: Optional collection of dashboard cards to include.
-
- Returns:
- A string containing the YAML representation of the dashboard."
+ "Generate YAML content for a dashboard with the given `entity-id`, `name`, and `collection-id`.
+ Optionally accepts `:dashcards` to include dashboard cards."
[entity-id name collection-id & {:keys [dashcards]}]
(let [dashcards-yaml (if dashcards
(str/join "\n" (map (fn [dc]
@@ -261,7 +229,7 @@ width: fixed
(->MockSourceSnapshot source-id base-url branch fail-mode files-atom)))
(defn create-mock-source
- "Create a mock Source for testing"
+ "Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, and `:initial-files`."
[& {:keys [branch fail-mode initial-files]
:or {branch "main"
fail-mode nil
@@ -287,7 +255,8 @@ width: fixed
(->MockSource "test-source" "https://test.example.com" branch fail-mode files-atom branches-atom)))
(defn clean-object
- "Reset the object table before running tests to prevent existing extries from affecting dirty state checks"
+ "Test fixture that resets the RemoteSyncObject table before running tests to prevent existing
+ entries from affecting dirty state checks."
[f]
(let [old-models (t2/select :model/RemoteSyncObject)]
(try
@@ -299,12 +268,12 @@ width: fixed
(t2/insert! :model/RemoteSyncObject old-models))))))
(defmacro with-clean-object
- "Macro to wrap a body to execute in a clean change log table"
+ "Execute `body` with a clean RemoteSyncObject table."
[& body]
`(clean-object (fn [] ~@body)))
(defn clean-task-table
- "Reset the task table to an empty state before running tests."
+ "Test fixture that resets the RemoteSyncTask table to an empty state before running tests."
[f]
(let [old-models (t2/select :model/RemoteSyncTask)]
(try
@@ -316,18 +285,12 @@ width: fixed
(t2/insert! :model/RemoteSyncTask old-models))))))
(def clean-remote-sync-state
- "Fixture to make sure sync state is clean"
+ "Composed test fixture that ensures both RemoteSyncObject and RemoteSyncTask tables are clean."
(t/compose-fixtures clean-object clean-task-table))
(defn generate-table-yaml
- "Generates YAML content for a table.
-
- Args:
- table-name: The name of the table (used as entity_id).
- db-name: The name of the database containing this table.
-
- Returns:
- A string containing the YAML representation of the table."
+ "Generate YAML content for a table with the given `table-name` and `db-name`.
+ Optionally accepts `:schema`, `:is-published` (defaults to true), and `:description`."
[table-name db-name & {:keys [schema is-published description]
:or {is-published true
description nil}}]
@@ -375,15 +338,8 @@ serdes/meta:
table-name))
(defn generate-field-yaml
- "Generates YAML content for a field.
-
- Args:
- field-name: The name of the field (used as entity_id).
- table-name: The name of the table containing this field.
- db-name: The name of the database containing this field.
-
- Returns:
- A string containing the YAML representation of the field."
+ "Generate YAML content for a field with the given `field-name`, `table-name`, and `db-name`.
+ Optionally accepts `:schema`, `:base-type`, `:description`, and `:database-type`."
[field-name table-name db-name & {:keys [schema base-type description database-type]
:or {base-type "type/Text"
database-type "VARCHAR"
@@ -446,16 +402,8 @@ database_partitioned: null
field-name))
(defn generate-segment-yaml
- "Generates YAML content for a segment.
-
- Args:
- segment-name: The name of the segment (used as entity_id).
- table-name: The name of the table containing this segment.
- db-name: The name of the database containing this segment.
- filter-field-name: The name of the field to use in the filter clause.
-
- Returns:
- A string containing the YAML representation of the segment."
+ "Generate YAML content for a segment with the given `segment-name`, `table-name`, and `db-name`.
+ Optionally accepts `:schema`, `:description`, `:entity-id`, and `:filter-field-name`."
[segment-name table-name db-name & {:keys [schema description entity-id filter-field-name]
:or {description "Test segment"
filter-field-name "Some Field"}}]
@@ -513,15 +461,8 @@ serdes/meta:
(str/replace (u/lower-case-en segment-name) #"\s+" "_"))))
(defn generate-action-yaml
- "Generates YAML content for an action.
-
- Args:
- entity-id: The unique identifier for the action.
- name: The name of the action.
- model-id: The entity ID of the model (Card) this action belongs to.
-
- Returns:
- A string containing the YAML representation of the action."
+ "Generate YAML content for an action with the given `entity-id`, `name`, and `model-id`.
+ Optionally accepts `:type` (defaults to `\"implicit\"`) and `:kind` (defaults to `\"row/create\"`)."
[entity-id name model-id & {:keys [type kind]
:or {type "implicit"
kind "row/create"}}]
@@ -555,10 +496,41 @@ serdes/meta:
entity-id
(str/replace (u/lower-case-en name) #"\s+" "_")))
+(defn generate-transform-yaml
+ "Generate YAML content for a transform with the given `entity-id` and `name`.
+ Optionally accepts `:collection-id` for transforms inside a collection."
+ [entity-id name & {:keys [collection-id]}]
+ (format "name: %s
+description: null
+entity_id: %s
+collection_id: %s
+created_at: '2024-08-28T09:46:18.671622Z'
+creator_id: [email protected]
+source_database_id: test-data (h2)
+source:
+ type: query
+ query:
+ database: test-data (h2)
+ type: query
+ query:
+ source-table:
+ - test-data (h2)
+ - PUBLIC
+ - VENUES
+target:
+ type: table
+ name: test_output
+ schema: PUBLIC
+serdes/meta:
+- id: %s
+ label: %s
+ model: Transform
+"
+ name entity-id (or collection-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
+
(defn generate-measure-yaml
- "Generates YAML content for a measure with the given `measure-name`, `table-name`, and `db-name`.
- Optional keyword args include `:schema`, `:description`, `:entity-id`, and `:agg-field-name`
- for the aggregation field. Returns a string containing the YAML representation of the measure."
+ "Generate YAML content for a measure with the given `measure-name`, `table-name`, and `db-name`.
+ Optionally accepts `:schema`, `:description`, `:entity-id`, and `:agg-field-name`."
[measure-name table-name db-name & {:keys [schema description entity-id agg-field-name]
:or {description "Test measure"
agg-field-name "Some Field"}}]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index d83fbf70d5f..1cf885f511e 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -230,7 +230,7 @@
remote-sync-enabled true]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
- (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxx" :location "/"}
+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxxxx" :location "/"}
:model/Transform transform {:name "Export Transform" :collection_id coll-id}
:model/RemoteSyncObject _rso1 {:model_type "Collection" :model_id coll-id :model_name "Transforms Collection" :status "create" :status_changed_at (t/offset-date-time)}
:model/RemoteSyncObject _rso2 {:model_type "Transform" :model_id (:id transform) :model_name "Export Transform" :model_collection_id coll-id :status "create" :status_changed_at (t/offset-date-time)}]
@@ -259,7 +259,7 @@
remote-sync-enabled true]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
- (mt/with-temp [:model/Collection {rs-coll-id :id rs-coll-eid :entity_id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxx" :location "/"}
+ (mt/with-temp [:model/Collection {rs-coll-id :id rs-coll-eid :entity_id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxxxxx" :location "/"}
:model/RemoteSyncObject _rso {:model_type "Collection" :model_id rs-coll-id :model_name "Remote Synced" :status "create" :status_changed_at (t/offset-date-time)}]
(let [saved-coll (t2/select-one :model/Collection :id rs-coll-id)]
(is (true? (:is_remote_synced saved-coll))
@@ -277,33 +277,6 @@
(is (not (some #(str/includes? % "transforms/") (keys files-after-export)))
"Export should NOT include transform files when setting is disabled"))))))))))
-(defn- generate-transform-yaml
- "Generates YAML content for a transform."
- [entity-id name database-id table-id]
- (format "name: %s
-description: null
-entity_id: %s
-collection_id: null
-created_at: '2024-08-28T09:46:18.671622Z'
-creator_id: [email protected]
-source:
- type: query
- query:
- database: %d
- type: query
- query:
- source-table: %d
-target:
- type: table
- name: test_output
- schema: PUBLIC
-serdes/meta:
-- id: %s
- label: %s
- model: Transform
-"
- name entity-id database-id table-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
-
(defn- generate-transforms-namespace-collection-yaml
"Generates YAML content for a transforms-namespace collection."
[entity-id name]
@@ -335,14 +308,14 @@ is_sample: false
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-model-cleanup [:model/Transform :model/Collection]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- coll-entity-id "transforms-coll-xxx"
- transform-entity-id "test-transform-xxxxx"
- test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")
- (generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")
- (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_test_transform.yaml")
- (generate-transform-yaml transform-entity-id "Test Transform" (mt/id) (mt/id :venues))}}
- mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! (source.p/snapshot mock-source) task-id)]
+ coll-entity-id "transforms-coll-xxxxx"
+ transform-entity-id "test-transform-xxxxxx"
+ test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")
+ (generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")
+ (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_test_transform.yaml")
+ (test-helpers/generate-transform-yaml transform-entity-id "Test Transform" :collection-id coll-entity-id)}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
(is (t2/exists? :model/Collection :entity_id coll-entity-id :namespace "transforms")
"Transforms-namespace collection should be imported")
@@ -390,7 +363,7 @@ is_sample: false
(let [initial-files {"main" {(str "collections/" coll-eid "_transforms_collection/" coll-eid "_transforms_collection.yaml")
(generate-transforms-namespace-collection-yaml coll-eid "Transforms Collection")
(str "collections/" coll-eid "_transforms_collection/transforms/" transform-eid "_child_transform.yaml")
- (generate-transform-yaml transform-eid "Child Transform" (mt/id) (mt/id :venues))}}
+ (test-helpers/generate-transform-yaml transform-eid "Child Transform" :collection-id coll-eid)}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)]
(is (some #(str/includes? % coll-eid) (keys (get @(:files-atom mock-source) "main"))))
(is (some #(str/includes? % transform-eid) (keys (get @(:files-atom mock-source) "main"))))
@@ -404,7 +377,6 @@ is_sample: false
(let [files-after-export (get @(:files-atom mock-source) "main")]
(is (not (some #(str/includes? % coll-eid) (keys files-after-export))))
(is (not (some #(str/includes? % transform-eid) (keys files-after-export))))
- ;; Root transform should still be exported
(is (some #(str/includes? % root-transform-eid) (keys files-after-export))
"Root transform should be exported")))))))))))
@@ -461,24 +433,24 @@ is_sample: false
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
- (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- coll-entity-id (u/generate-nano-id)
- local-transform-entity-id (u/generate-nano-id)
- remote-transform-entity-id (u/generate-nano-id)]
- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
- :namespace collection/transforms-ns
- :entity_id coll-entity-id
- :location "/"}
- :model/Transform {local-transform-id :id} {:name "Local Transform"
- :entity_id local-transform-entity-id
- :collection_id coll-id}]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ coll-entity-id (u/generate-nano-id)
+ local-transform-entity-id (u/generate-nano-id)
+ remote-transform-entity-id (u/generate-nano-id)]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
+ :namespace collection/transforms-ns
+ :entity_id coll-entity-id
+ :location "/"}
+ :model/Transform {local-transform-id :id} {:name "Local Transform"
+ :entity_id local-transform-entity-id
+ :collection_id coll-id}]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]
(is (t2/exists? :model/Collection :id coll-id))
(is (t2/exists? :model/Transform :id local-transform-id))
(let [test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")
(generate-transforms-namespace-collection-yaml coll-entity-id "Transforms Collection")
(str "collections/" coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
- (generate-transform-yaml remote-transform-entity-id "Remote Transform" (mt/id) (mt/id :venues))}}
+ (test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result))
@@ -673,7 +645,6 @@ serdes/meta:
local-entity-id (:entity_id local-library)]
(is (t2/exists? :model/PythonLibrary :path "common.py")
"Local PythonLibrary should exist before import")
- ;; Use the same entity_id as the local library so it gets updated
(let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")
(generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
@@ -684,3 +655,159 @@ serdes/meta:
(is (some? library) "PythonLibrary should still exist")
(is (str/includes? (:source library) "# remote version")
"PythonLibrary source should be updated with remote version")))))))))))
+
+;;; ------------------------------------------- Transform Sync Behavior Tests -------------------------------------------
+
+(deftest import-preserves-local-transforms-when-setting-disabled-and-remote-has-none-test
+ (testing "When remote-sync-transforms is disabled and remote has no transforms, local transforms are preserved"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms false
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/TransformTag]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-transform-entity-id (u/generate-nano-id)
+ local-tag-entity-id (u/generate-nano-id)]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
+ :namespace collection/transforms-ns
+ :location "/"}
+ :model/Transform {local-transform-id :id} {:name "Local Transform"
+ :entity_id local-transform-entity-id
+ :collection_id coll-id}
+ :model/TransformTag {local-tag-id :id} {:name "Local Tag"
+ :entity_id local-tag-entity-id}]
+ (is (t2/exists? :model/Transform :id local-transform-id)
+ "Local transform should exist before import")
+ (is (t2/exists? :model/TransformTag :id local-tag-id)
+ "Local tag should exist before import")
+ (is (t2/exists? :model/Collection :id coll-id)
+ "Local transforms collection should exist before import")
+ (let [remote-coll-entity-id (u/generate-nano-id)
+ test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_collection/" remote-coll-entity-id "_remote_collection.yaml")
+ (test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (t2/exists? :model/Transform :id local-transform-id)
+ "Local transform should still exist after import when setting is disabled")
+ (is (t2/exists? :model/TransformTag :id local-tag-id)
+ "Local tag should still exist after import when setting is disabled")
+ (is (t2/exists? :model/Collection :id coll-id)
+ "Local transforms collection should still exist after import when setting is disabled")))))))))
+
+(deftest import-replaces-local-transforms-when-remote-has-transforms-test
+ (testing "When remote has transforms, local transforms are replaced with remote content (setting auto-enabled)"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms false
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/TransformTag]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-transform-entity-id (u/generate-nano-id)
+ local-tag-entity-id (u/generate-nano-id)
+ remote-transform-entity-id (u/generate-nano-id)
+ remote-coll-entity-id (u/generate-nano-id)]
+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Local Transforms Collection"
+ :namespace collection/transforms-ns
+ :location "/"}
+ :model/Transform {local-transform-id :id} {:name "Local Transform"
+ :entity_id local-transform-entity-id
+ :collection_id coll-id}
+ :model/TransformTag {local-tag-id :id} {:name "Local Tag"
+ :entity_id local-tag-entity-id}]
+ (is (t2/exists? :model/Transform :id local-transform-id)
+ "Local transform should exist before import")
+ (is (t2/exists? :model/TransformTag :id local-tag-id)
+ "Local tag should exist before import")
+ (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_transforms/" remote-coll-entity-id "_transforms.yaml")
+ (generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")
+ (str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
+ (test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (settings/remote-sync-transforms)
+ "remote-sync-transforms setting should be auto-enabled when remote has transforms")
+ (is (not (t2/exists? :model/Transform :id local-transform-id))
+ "Local transform should be deleted after import when remote has different transforms")
+ (is (not (t2/exists? :model/TransformTag :id local-tag-id))
+ "Local tag should be deleted after import when remote has different tags")
+ (is (t2/exists? :model/Transform :entity_id remote-transform-entity-id)
+ "Remote transform should be imported")
+ (is (t2/exists? :model/Collection :entity_id remote-coll-entity-id :namespace "transforms")
+ "Remote transforms collection should be imported")
+ (is (not (t2/exists? :model/Collection :entity_id coll-eid))
+ "Local transforms collection should be removed when not on remote")))))))))
+
+(deftest import-removes-all-local-transforms-when-setting-enabled-and-remote-has-none-test
+ (testing "When remote-sync-transforms is enabled and remote has no transforms, all local transforms are removed"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/TransformTag]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-transform-entity-id (u/generate-nano-id)
+ local-tag-entity-id (u/generate-nano-id)]
+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Local Transforms Collection"
+ :namespace collection/transforms-ns
+ :entity_id (u/generate-nano-id)
+ :location "/"}
+ :model/Transform {local-transform-id :id} {:name "Local Transform"
+ :entity_id local-transform-entity-id
+ :collection_id coll-id}
+ :model/TransformTag {local-tag-id :id} {:name "Local Tag"
+ :entity_id local-tag-entity-id}]
+ (is (t2/exists? :model/Transform :id local-transform-id)
+ "Local transform should exist before import")
+ (is (t2/exists? :model/TransformTag :id local-tag-id)
+ "Local tag should exist before import")
+ (is (t2/exists? :model/Collection :id coll-id)
+ "Local transforms collection should exist before import")
+ (let [remote-coll-entity-id (u/generate-nano-id)
+ test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_collection/" remote-coll-entity-id "_remote_collection.yaml")
+ (test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (not (t2/exists? :model/Transform :id local-transform-id))
+ "Local transform should be deleted when setting enabled and remote has no transforms")
+ (is (not (t2/exists? :model/TransformTag :id local-tag-id))
+ "Local tag should be deleted when setting enabled and remote has no transforms")
+ (is (not (t2/exists? :model/Collection :entity_id coll-eid))
+ "Local transforms collection should be deleted when setting enabled and remote has none")))))))))
+
+(deftest import-auto-enables-setting-when-remote-has-transforms-test
+ (testing "When setting is disabled and remote has transforms, setting is auto-enabled and transforms are synced"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms false
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-transform-entity-id (u/generate-nano-id)
+ remote-transform-entity-id (u/generate-nano-id)
+ remote-coll-entity-id (u/generate-nano-id)]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Local Transforms Collection"
+ :namespace collection/transforms-ns
+ :location "/"}
+ :model/Transform {local-transform-id :id} {:name "Local Transform"
+ :entity_id local-transform-entity-id
+ :collection_id coll-id}]
+ (is (not (settings/remote-sync-transforms))
+ "remote-sync-transforms should be disabled initially")
+ (is (t2/exists? :model/Transform :id local-transform-id)
+ "Local transform should exist before import")
+ (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_transforms/" remote-coll-entity-id "_transforms.yaml")
+ (generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")
+ (str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
+ (test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (settings/remote-sync-transforms)
+ "remote-sync-transforms setting should be auto-enabled when remote has transforms")
+ (is (t2/exists? :model/Transform :entity_id remote-transform-entity-id)
+ "Remote transform should be imported")
+ (is (not (t2/exists? :model/Transform :id local-transform-id))
+ "Local transform should be removed because it's not on the remote")))))))))