The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Do not delete built-in transform tags on sync (#68727)
John Swanson36a5234efcaa
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex d665063f08e..1bd6a6b3299 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -56,16 +56,20 @@[:not-in entity-ids]:entity_id);; :scope-key determines collection scoping - nil means global- scope-key (get-in model-spec [:removal :scope-key])]]+ scope-key (get-in model-spec [:removal :scope-key])+ ;; Extra conditions to protect certain rows from removal (e.g. built-in TransformTags)+ extra-conditions (into [] cat (get-in model-spec [:removal :conditions]))]](if scope-key;; Collection-scoped: delete only within synced collections(when (seq synced-collection-ids)- (t2/delete! model-key- scope-key [:in synced-collection-ids]- :entity_id entity-id-clause))+ (apply t2/delete! model-key+ scope-key [:in synced-collection-ids]+ :entity_id entity-id-clause+ extra-conditions));; Global: delete by entity_id only- (t2/delete! model-key- :entity_id entity-id-clause))))+ (apply t2/delete! model-key+ :entity_id entity-id-clause+ extra-conditions))))(defn source-error-message"Constructs user-friendly error messages from remote sync source exceptions.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 42c131b090d..3000c5229b6 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -290,7 +290,8 @@:archived-key nil ; no archived field:tracking {:select-fields [:name]:field-mappings {:model_name :name}}- :removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion+ :removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion+ :conditions {:built_in_type nil}} ; protect built-in tags from removal:export-path {:type :transform-tag-path}:export-scope :all ; query for all instances:enabled? :remote-sync-transforms}})diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex 65524ba3f69..97cbd1787c9 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -985,18 +985,23 @@remote-sync-branch "main"remote-sync-type :read-writeremote-sync-transforms false]- (testing "can enable transforms sync"- (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"- {:remote-sync-transforms true})]- (wait-for-task-completion task_id)- (is (=? {:success true} resp))- (is (true? (settings/remote-sync-transforms)))))- (testing "can disable transforms sync"- (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"- {:remote-sync-transforms false})]- (wait-for-task-completion task_id)- (is (=? {:success true} resp))- (is (false? (settings/remote-sync-transforms))))))))))+ (let [built-in-count (t2/count :model/TransformTag :built_in_type [:not= nil])]+ (testing "can enable transforms sync"+ (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"+ {:remote-sync-transforms true})]+ (wait-for-task-completion task_id)+ (is (=? {:success true} resp))+ (is (true? (settings/remote-sync-transforms)))+ (is (= built-in-count (t2/count :model/TransformTag :built_in_type [:not= nil]))+ "Built-in transform tags should not be deleted by sync")))+ (testing "can disable transforms sync"+ (let [{:as resp :keys [task_id]} (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"+ {:remote-sync-transforms false})]+ (wait-for-task-completion task_id)+ (is (=? {:success true} resp))+ (is (false? (settings/remote-sync-transforms)))+ (is (= built-in-count (t2/count :model/TransformTag :built_in_type [:not= nil]))+ "Built-in transform tags should not be deleted by sync")))))))))(deftest settings-preserves-transforms-when-not-specified-test(testing "PUT /api/ee/remote-sync/settings preserves transforms setting when not specified"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 19cee29cd1d..ef89aa6e201 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -220,7 +220,7 @@io/input-stream)ba (#'api.serialization/ba-copy res)](testing "Archive contains correct number of files with proper log entries"- (is (= 8+ (is (= 12(with-open [tar (open-tar ba)](count(for [^TarArchiveEntry e (u.compress/entries tar)@@ -228,7 +228,7 @@(do(condp re-find (.getName e)#"/export.log$" (testing "Log contains extract and store entries"- (is (= (+ #_extract 7 #_store 7)+ (is (= (+ #_extract 11 #_store 11)(count (line-seq (io/reader tar))))))nil)(.getName e))))))))@@ -242,7 +242,7 @@"settings" false"field_values" false"duration_ms" (every-pred number? pos?)- "count" 7+ "count" 11"error_count" 0"source" "api""secrets" false@@ -272,7 +272,7 @@{:request-options {:headers {"content-type" "multipart/form-data"}}}{:file ba}))](testing "Log contains imported entity types"- (is (= #{"Collection" "Dashboard" "Card" "TransformJob"}+ (is (= #{"Collection" "Dashboard" "Card" "TransformJob" "TransformTag"}(log-types (line-seq (io/reader (io/input-stream res)))))))(testing "Entities are restored in the database"@@ -284,8 +284,8 @@"direction" "import""duration_ms" pos?"source" "api"- "models" "Card,Collection,Dashboard,TransformJob"- "count" 7+ "models" "Card,Collection,Dashboard,TransformJob,TransformTag"+ "count" 11"error_count" 0"success" true"error_message" nil}@@ -348,7 +348,7 @@:continue_on_error true)log (slurp (io/input-stream res))](testing "Log shows loaded entities and error"- (is (= #{"Dashboard" "Card" "Collection" "TransformJob"}+ (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag"}(log-types (str/split-lines log))))(is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log)))@@ -358,9 +358,9 @@"direction" "import""source" "api""duration_ms" int?- "count" 6+ "count" 10"error_count" 1- "models" "Collection,Dashboard,TransformJob"}+ "models" "Collection,Dashboard,TransformJob,TransformTag"}(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))))(deftest import-invalid-archive-test@@ -428,7 +428,7 @@(doseq [^TarArchiveEntry e (u.compress/entries tar)](condp re-find (.getName e)#"/export.log$" (testing "Log shows extract entries, error, and store entries"- (is (= (+ #_extract 7 #_error 1 #_store 6)+ (is (= (+ #_extract 11 #_error 1 #_store 10)(count (line-seq (io/reader tar))))))nil))))@@ -441,7 +441,7 @@"settings" false"field_values" false"duration_ms" pos?- "count" 6+ "count" 10"error_count" 1"source" "api""secrets" false