Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Do not delete built-in transform tags on sync (#68727)

John Swanson36a5234efcaa
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index d665063f08e..1bd6a6b3299 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -56,16 +56,20 @@
[:not-in entity-ids]
:entity_id)
;; :scope-key determines collection scoping - nil means global
- scope-key (get-in model-spec [:removal :scope-key])]]
+ scope-key (get-in model-spec [:removal :scope-key])
+ ;; Extra conditions to protect certain rows from removal (e.g. built-in TransformTags)
+ extra-conditions (into [] cat (get-in model-spec [:removal :conditions]))]]
(if scope-key
;; Collection-scoped: delete only within synced collections
(when (seq synced-collection-ids)
- (t2/delete! model-key
- scope-key [:in synced-collection-ids]
- :entity_id entity-id-clause))
+ (apply t2/delete! model-key
+ scope-key [:in synced-collection-ids]
+ :entity_id entity-id-clause
+ extra-conditions))
;; Global: delete by entity_id only
- (t2/delete! model-key
- :entity_id entity-id-clause))))
+ (apply t2/delete! model-key
+ :entity_id entity-id-clause
+ extra-conditions))))
(defn source-error-message
"Constructs user-friendly error messages from remote sync source exceptions.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 42c131b090d..3000c5229b6 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -290,7 +290,8 @@
:archived-key nil ; no archived field
:tracking {:select-fields [:name]
:field-mappings {:model_name :name}}
- :removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion
+ :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
+ :conditions {:built_in_type nil}} ; protect built-in tags from removal
:export-path {:type :transform-tag-path}
:export-scope :all ; query for all instances
:enabled? :remote-sync-transforms}})
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index 65524ba3f69..97cbd1787c9 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -985,18 +985,23 @@
remote-sync-branch "main"
remote-sync-type :read-write
remote-sync-transforms false]
- (testing "can enable transforms sync"
- (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"
- {:remote-sync-transforms true})]
- (wait-for-task-completion task_id)
- (is (=? {:success true} resp))
- (is (true? (settings/remote-sync-transforms)))))
- (testing "can disable transforms sync"
- (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"
- {:remote-sync-transforms false})]
- (wait-for-task-completion task_id)
- (is (=? {:success true} resp))
- (is (false? (settings/remote-sync-transforms))))))))))
+ (let [built-in-count (t2/count :model/TransformTag :built_in_type [:not= nil])]
+ (testing "can enable transforms sync"
+ (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"
+ {:remote-sync-transforms true})]
+ (wait-for-task-completion task_id)
+ (is (=? {:success true} resp))
+ (is (true? (settings/remote-sync-transforms)))
+ (is (= built-in-count (t2/count :model/TransformTag :built_in_type [:not= nil]))
+ "Built-in transform tags should not be deleted by sync")))
+ (testing "can disable transforms sync"
+ (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"
+ {:remote-sync-transforms false})]
+ (wait-for-task-completion task_id)
+ (is (=? {:success true} resp))
+ (is (false? (settings/remote-sync-transforms)))
+ (is (= built-in-count (t2/count :model/TransformTag :built_in_type [:not= nil]))
+ "Built-in transform tags should not be deleted by sync")))))))))
(deftest settings-preserves-transforms-when-not-specified-test
(testing "PUT /api/ee/remote-sync/settings preserves transforms setting when not specified"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 19cee29cd1d..ef89aa6e201 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -220,7 +220,7 @@
io/input-stream)
ba (#'api.serialization/ba-copy res)]
(testing "Archive contains correct number of files with proper log entries"
- (is (= 8
+ (is (= 12
(with-open [tar (open-tar ba)]
(count
(for [^TarArchiveEntry e (u.compress/entries tar)
@@ -228,7 +228,7 @@
(do
(condp re-find (.getName e)
#"/export.log$" (testing "Log contains extract and store entries"
- (is (= (+ #_extract 7 #_store 7)
+ (is (= (+ #_extract 11 #_store 11)
(count (line-seq (io/reader tar))))))
nil)
(.getName e))))))))
@@ -242,7 +242,7 @@
"settings" false
"field_values" false
"duration_ms" (every-pred number? pos?)
- "count" 7
+ "count" 11
"error_count" 0
"source" "api"
"secrets" false
@@ -272,7 +272,7 @@
{:request-options {:headers {"content-type" "multipart/form-data"}}}
{:file ba}))]
(testing "Log contains imported entity types"
- (is (= #{"Collection" "Dashboard" "Card" "TransformJob"}
+ (is (= #{"Collection" "Dashboard" "Card" "TransformJob" "TransformTag"}
(log-types (line-seq (io/reader (io/input-stream res)))))))
(testing "Entities are restored in the database"
@@ -284,8 +284,8 @@
"direction" "import"
"duration_ms" pos?
"source" "api"
- "models" "Card,Collection,Dashboard,TransformJob"
- "count" 7
+ "models" "Card,Collection,Dashboard,TransformJob,TransformTag"
+ "count" 11
"error_count" 0
"success" true
"error_message" nil}
@@ -348,7 +348,7 @@
:continue_on_error true)
log (slurp (io/input-stream res))]
(testing "Log shows loaded entities and error"
- (is (= #{"Dashboard" "Card" "Collection" "TransformJob"}
+ (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag"}
(log-types (str/split-lines log))))
(is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log)))
@@ -358,9 +358,9 @@
"direction" "import"
"source" "api"
"duration_ms" int?
- "count" 6
+ "count" 10
"error_count" 1
- "models" "Collection,Dashboard,TransformJob"}
+ "models" "Collection,Dashboard,TransformJob,TransformTag"}
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))))
(deftest import-invalid-archive-test
@@ -428,7 +428,7 @@
(doseq [^TarArchiveEntry e (u.compress/entries tar)]
(condp re-find (.getName e)
#"/export.log$" (testing "Log shows extract entries, error, and store entries"
- (is (= (+ #_extract 7 #_error 1 #_store 6)
+ (is (= (+ #_extract 11 #_error 1 #_store 10)
(count (line-seq (io/reader tar))))))
nil))))
@@ -441,7 +441,7 @@
"settings" false
"field_values" false
"duration_ms" pos?
- "count" 6
+ "count" 10
"error_count" 1
"source" "api"
"secrets" false