Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: only skip deleting commonpy not pushing it (#70439)

Edward Paget76e96af83490
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 4f1fcb41dcb..e4cb7a610b8 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -91,11 +91,12 @@
(doseq [[model-key model-spec] (spec/specs-for-deletion)
:let [serdes-model (:model-type model-spec)
entity-ids (get by-entity-id serdes-model [])
- spec-entity-id-condition (get-in model-spec [:conditions :entity_id])
+ removal-conds (spec/removal-conditions model-spec)
+ spec-entity-id-condition (get removal-conds :entity_id)
entity-id-where (build-entity-id-where-clause entity-ids spec-entity-id-condition)
scope-key (get-in model-spec [:removal :scope-key])
;; Get non-entity_id conditions from spec
- other-conditions (into [] cat (dissoc (:conditions model-spec) :entity_id))]]
+ other-conditions (into [] cat (dissoc removal-conds :entity_id))]]
(if scope-key
;; Collection-scoped: delete only within synced collections
(when (seq synced-collection-ids)
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 4201d68b85f..2078d9c4729 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -62,6 +62,8 @@
Only entities matching these conditions are eligible for sync
(export, import cleanup, removal). Example: {:built_in_type nil}
to exclude built-in items from sync.
+ - :export-conditions - Optional. Overrides :conditions for export queries. Falls back to :conditions.
+ - :removal-conditions - Optional. Overrides :conditions for removal/cleanup. Falls back to :conditions.
- :removal - Removal/cleanup configuration:
:statuses - Set of statuses to check for removal (e.g., #{\"removed\" \"delete\"})
:scope-key - Optional key for scoping deletions (e.g., :collection_id, :id).
@@ -313,7 +315,7 @@
:archived-key nil ; no archived field
:tracking {:select-fields [:path]
:field-mappings {:model_name :path}}
- :conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; exclude built-in common.py from sync
+ :removal-conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; protect built-in common.py from deletion
:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
:all-on-setting-disable :remote-sync-transforms}
:export-scope :all ; query for all instances
@@ -330,6 +332,18 @@
(keyword? enabled?) (boolean (setting/get-value-of-type :boolean enabled?))
:else false))
+(defn export-conditions
+ "Returns conditions to apply when querying entities for export.
+ Uses :export-conditions if present, else falls back to :conditions."
+ [spec]
+ (or (:export-conditions spec) (:conditions spec)))
+
+(defn removal-conditions
+ "Returns conditions to apply when removing unsynced entities.
+ Uses :removal-conditions if present, else falls back to :conditions."
+ [spec]
+ (or (:removal-conditions spec) (:conditions spec)))
+
(defn enabled-specs
"Returns a map of model-key -> spec for all currently enabled specs."
[]
@@ -506,17 +520,16 @@
(defn- has-unsynced-entities-for-feature?
"Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.
- Excludes built-in TransformTags and the built-in PythonLibrary from the count since they are
- system-created and not user data. Namespace collections are not checked here because they are
+ Excludes entities filtered by export-conditions (e.g., built-in TransformTags) from the count since
+ they are system-created and not user data. Namespace collections are not checked here because they are
organizational containers, not user data that would be lost on import."
[specs-for-feature]
(some (fn [[_ spec]]
(let [model-key (:model-key spec)
model-type (:model-type spec)
- ;; Exclude built-in entities from count (they are system-created, not user data)
- local-count (case model-key
- :model/TransformTag (t2/count model-key :built_in_type nil)
- :model/PythonLibrary (t2/count model-key :entity_id [:not= transforms-python/builtin-entity-id])
+ conditions (export-conditions spec)
+ local-count (if conditions
+ (apply t2/count model-key (into [] cat conditions))
(t2/count model-key))
synced-count (t2/count :model/RemoteSyncObject :model_type model-type)]
(and (pos? local-count)
@@ -957,9 +970,10 @@
(defn- build-bulk-removal-paths
"Builds removal paths for ALL entities of a spec's model type.
Used when the controlling setting is disabled (sentinel RSO has 'delete' status).
- Respects :conditions from the spec to exclude ineligible entities."
+ Respects :removal-conditions (or :conditions) from the spec to exclude ineligible entities."
[spec ctx]
- (let [{:keys [model-type model-key conditions]} spec]
+ (let [{:keys [model-type model-key]} spec
+ conditions (removal-conditions spec)]
(for [entity (if conditions
(apply t2/select model-key (into [] cat conditions))
(t2/select model-key))
@@ -1178,17 +1192,18 @@
nil))
(defmethod query-export-roots :setting
- [{:keys [export-scope model-key model-type conditions] :as spec}]
+ [{:keys [export-scope model-key model-type] :as spec}]
(when (spec-enabled? spec)
- (case export-scope
- :root-only
- (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key
- :collection_id nil
- (into [] cat conditions))
- :all
- (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key
- (into [] cat conditions))
- nil)))
+ (let [conditions (export-conditions spec)]
+ (case export-scope
+ :root-only
+ (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key
+ :collection_id nil
+ (into [] cat conditions))
+ :all
+ (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key
+ (into [] cat conditions))
+ nil))))
(defmethod query-export-roots :published-table [_] nil)
(defmethod query-export-roots :parent-table [_] nil)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index 3d25295770e..d2e7b1d550f 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -3,6 +3,7 @@
[clojure.set :as set]
[clojure.test :refer :all]
[metabase-enterprise.remote-sync.spec :as spec]
+ [metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.test :as mt]))
(set! *warn-on-reflection* true)
@@ -457,3 +458,62 @@
(let [instances [{:id 1} {:id 2} {:id 3}]
result (spec/batch-model-editable? :model/NativeQuerySnippet instances)]
(is (= {1 true, 2 true, 3 true} result))))))))
+
+;;; ------------------------------------------ Condition Helper Tests ----------------------------------------------
+
+(deftest export-conditions-test
+ (testing "export-conditions returns :export-conditions when present"
+ (is (= {:foo :bar}
+ (spec/export-conditions {:export-conditions {:foo :bar}
+ :conditions {:baz :qux}}))))
+
+ (testing "export-conditions falls back to :conditions when :export-conditions absent"
+ (is (= {:baz :qux}
+ (spec/export-conditions {:conditions {:baz :qux}}))))
+
+ (testing "export-conditions returns nil when neither key present"
+ (is (nil? (spec/export-conditions {})))))
+
+(deftest removal-conditions-test
+ (testing "removal-conditions returns :removal-conditions when present"
+ (is (= {:foo :bar}
+ (spec/removal-conditions {:removal-conditions {:foo :bar}
+ :conditions {:baz :qux}}))))
+
+ (testing "removal-conditions falls back to :conditions when :removal-conditions absent"
+ (is (= {:baz :qux}
+ (spec/removal-conditions {:conditions {:baz :qux}}))))
+
+ (testing "removal-conditions returns nil when neither key present"
+ (is (nil? (spec/removal-conditions {})))))
+
+(deftest python-library-spec-conditions-test
+ (testing "PythonLibrary spec has :removal-conditions but no :conditions or :export-conditions"
+ (let [spec (spec/spec-for-model-key :model/PythonLibrary)]
+ (is (nil? (:conditions spec))
+ "PythonLibrary should have no :conditions")
+ (is (nil? (:export-conditions spec))
+ "PythonLibrary should have no :export-conditions")
+ (is (= {:entity_id [:not= transforms-python/builtin-entity-id]}
+ (:removal-conditions spec))
+ "PythonLibrary should have :removal-conditions protecting builtin entity")))
+
+ (testing "export-conditions returns nil for PythonLibrary (no export filtering)"
+ (let [spec (spec/spec-for-model-key :model/PythonLibrary)]
+ (is (nil? (spec/export-conditions spec)))))
+
+ (testing "removal-conditions returns the builtin protection for PythonLibrary"
+ (let [spec (spec/spec-for-model-key :model/PythonLibrary)]
+ (is (= {:entity_id [:not= transforms-python/builtin-entity-id]}
+ (spec/removal-conditions spec))))))
+
+(deftest transform-tag-spec-uses-conditions-test
+ (testing "TransformTag spec still uses :conditions (not split)"
+ (let [spec (spec/spec-for-model-key :model/TransformTag)]
+ (is (= {:built_in_type nil} (:conditions spec)))
+ (is (nil? (:export-conditions spec)))
+ (is (nil? (:removal-conditions spec)))
+ (is (= {:built_in_type nil} (spec/export-conditions spec))
+ "export-conditions falls back to :conditions for TransformTag")
+ (is (= {:built_in_type nil} (spec/removal-conditions spec))
+ "removal-conditions falls back to :conditions for TransformTag"))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index 0141acbd197..ef0a65f75a6 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -927,3 +927,39 @@ serdes/meta:
"Custom tag (built_in_type nil) should be in export roots")
(is (not (contains? exported-ids builtin-tag-id))
"Built-in tag should NOT be in export roots")))))))
+
+(deftest export-includes-builtin-python-library-test
+ (testing "Export includes built-in PythonLibrary (common.py) since it has no export-conditions"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-temp [:model/PythonLibrary builtin-lib {:path "common.py"
+ :source "# builtin"
+ :entity_id transforms-python/builtin-entity-id}
+ :model/PythonLibrary custom-lib {:path "custom.py"
+ :source "# custom"}]
+ (let [python-lib-spec (spec/spec-for-model-key :model/PythonLibrary)
+ export-roots (spec/query-export-roots python-lib-spec)
+ exported-ids (set (map second export-roots))]
+ (is (contains? exported-ids (:id builtin-lib))
+ "Built-in PythonLibrary should be in export roots")
+ (is (contains? exported-ids (:id custom-lib))
+ "Custom PythonLibrary should be in export roots")))))))
+
+(deftest build-all-removal-paths-excludes-builtin-python-library-test
+ (testing "build-all-removal-paths excludes built-in PythonLibrary via :removal-conditions"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-temp [:model/PythonLibrary _ {:path "common.py"
+ :source "# builtin"
+ :entity_id transforms-python/builtin-entity-id}
+ :model/PythonLibrary custom-lib {:path "custom.py"
+ :source "# custom"}]
+ (settings/sync-transform-tracking! true)
+ (settings/sync-transform-tracking! false)
+ (let [paths (spec/build-all-removal-paths)]
+ (is (some #(str/includes? % (:entity_id custom-lib)) paths)
+ "Custom PythonLibrary should be in removal paths")
+ (is (not (some #(str/includes? % transforms-python/builtin-entity-id) paths))
+ "Built-in PythonLibrary should NOT be in removal paths")))))))