The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix: only skip deleting commonpy not pushing it (#70439)
Edward Paget76e96af83490
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 4f1fcb41dcb..e4cb7a610b8 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -91,11 +91,12 @@(doseq [[model-key model-spec] (spec/specs-for-deletion):let [serdes-model (:model-type model-spec)entity-ids (get by-entity-id serdes-model [])- spec-entity-id-condition (get-in model-spec [:conditions :entity_id])+ removal-conds (spec/removal-conditions model-spec)+ spec-entity-id-condition (get removal-conds :entity_id)entity-id-where (build-entity-id-where-clause entity-ids spec-entity-id-condition)scope-key (get-in model-spec [:removal :scope-key]);; Get non-entity_id conditions from spec- other-conditions (into [] cat (dissoc (:conditions model-spec) :entity_id))]]+ other-conditions (into [] cat (dissoc removal-conds :entity_id))]](if scope-key;; Collection-scoped: delete only within synced collections(when (seq synced-collection-ids)diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 4201d68b85f..2078d9c4729 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -62,6 +62,8 @@Only entities matching these conditions are eligible for sync(export, import cleanup, removal). Example: {:built_in_type nil}to exclude built-in items from sync.+ - :export-conditions - Optional. Overrides :conditions for export queries. Falls back to :conditions.+ - :removal-conditions - Optional. Overrides :conditions for removal/cleanup. Falls back to :conditions.- :removal - Removal/cleanup configuration::statuses - Set of statuses to check for removal (e.g., #{\"removed\" \"delete\"}):scope-key - Optional key for scoping deletions (e.g., :collection_id, :id).@@ -313,7 +315,7 @@:archived-key nil ; no archived field:tracking {:select-fields [:path]:field-mappings {:model_name :path}}- :conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; exclude built-in common.py from sync+ :removal-conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; protect built-in common.py from deletion:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion:all-on-setting-disable :remote-sync-transforms}:export-scope :all ; query for all instances@@ -330,6 +332,18 @@(keyword? enabled?) (boolean (setting/get-value-of-type :boolean enabled?)):else false))+(defn export-conditions+ "Returns conditions to apply when querying entities for export.+ Uses :export-conditions if present, else falls back to :conditions."+ [spec]+ (or (:export-conditions spec) (:conditions spec)))++(defn removal-conditions+ "Returns conditions to apply when removing unsynced entities.+ Uses :removal-conditions if present, else falls back to :conditions."+ [spec]+ (or (:removal-conditions spec) (:conditions spec)))+(defn enabled-specs"Returns a map of model-key -> spec for all currently enabled specs."[]@@ -506,17 +520,16 @@(defn- has-unsynced-entities-for-feature?"Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.- Excludes built-in TransformTags and the built-in PythonLibrary from the count since they are- system-created and not user data. Namespace collections are not checked here because they are+ Excludes entities filtered by export-conditions (e.g., built-in TransformTags) from the count since+ they are system-created and not user data. Namespace collections are not checked here because they areorganizational containers, not user data that would be lost on import."[specs-for-feature](some (fn [[_ spec]](let [model-key (:model-key spec)model-type (:model-type spec)- ;; Exclude built-in entities from count (they are system-created, not user data)- local-count (case model-key- :model/TransformTag (t2/count model-key :built_in_type nil)- :model/PythonLibrary (t2/count model-key :entity_id [:not= transforms-python/builtin-entity-id])+ conditions (export-conditions spec)+ local-count (if conditions+ (apply t2/count model-key (into [] cat conditions))(t2/count model-key))synced-count (t2/count :model/RemoteSyncObject :model_type model-type)](and (pos? local-count)@@ -957,9 +970,10 @@(defn- build-bulk-removal-paths"Builds removal paths for ALL entities of a spec's model type.Used when the controlling setting is disabled (sentinel RSO has 'delete' status).- Respects :conditions from the spec to exclude ineligible entities."+ Respects :removal-conditions (or :conditions) from the spec to exclude ineligible entities."[spec ctx]- (let [{:keys [model-type model-key conditions]} spec]+ (let [{:keys [model-type model-key]} spec+ conditions (removal-conditions spec)](for [entity (if conditions(apply t2/select model-key (into [] cat conditions))(t2/select model-key))@@ -1178,17 +1192,18 @@nil))(defmethod query-export-roots :setting- [{:keys [export-scope model-key model-type conditions] :as spec}]+ [{:keys [export-scope model-key model-type] :as spec}](when (spec-enabled? spec)- (case export-scope- :root-only- (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key- :collection_id nil- (into [] cat conditions))- :all- (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key- (into [] cat conditions))- nil)))+ (let [conditions (export-conditions spec)]+ (case export-scope+ :root-only+ (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key+ :collection_id nil+ (into [] cat conditions))+ :all+ (apply t2/select-fn-set (juxt (constantly model-type) :id) model-key+ (into [] cat conditions))+ nil))))(defmethod query-export-roots :published-table [_] nil)(defmethod query-export-roots :parent-table [_] nil)diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.cljindex 3d25295770e..d2e7b1d550f 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj@@ -3,6 +3,7 @@[clojure.set :as set][clojure.test :refer :all][metabase-enterprise.remote-sync.spec :as spec]+ [metabase-enterprise.transforms-python.core :as transforms-python][metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -457,3 +458,62 @@(let [instances [{:id 1} {:id 2} {:id 3}]result (spec/batch-model-editable? :model/NativeQuerySnippet instances)](is (= {1 true, 2 true, 3 true} result))))))))++;;; ------------------------------------------ Condition Helper Tests ----------------------------------------------++(deftest export-conditions-test+ (testing "export-conditions returns :export-conditions when present"+ (is (= {:foo :bar}+ (spec/export-conditions {:export-conditions {:foo :bar}+ :conditions {:baz :qux}}))))++ (testing "export-conditions falls back to :conditions when :export-conditions absent"+ (is (= {:baz :qux}+ (spec/export-conditions {:conditions {:baz :qux}}))))++ (testing "export-conditions returns nil when neither key present"+ (is (nil? (spec/export-conditions {})))))++(deftest removal-conditions-test+ (testing "removal-conditions returns :removal-conditions when present"+ (is (= {:foo :bar}+ (spec/removal-conditions {:removal-conditions {:foo :bar}+ :conditions {:baz :qux}}))))++ (testing "removal-conditions falls back to :conditions when :removal-conditions absent"+ (is (= {:baz :qux}+ (spec/removal-conditions {:conditions {:baz :qux}}))))++ (testing "removal-conditions returns nil when neither key present"+ (is (nil? (spec/removal-conditions {})))))++(deftest python-library-spec-conditions-test+ (testing "PythonLibrary spec has :removal-conditions but no :conditions or :export-conditions"+ (let [spec (spec/spec-for-model-key :model/PythonLibrary)]+ (is (nil? (:conditions spec))+ "PythonLibrary should have no :conditions")+ (is (nil? (:export-conditions spec))+ "PythonLibrary should have no :export-conditions")+ (is (= {:entity_id [:not= transforms-python/builtin-entity-id]}+ (:removal-conditions spec))+ "PythonLibrary should have :removal-conditions protecting builtin entity")))++ (testing "export-conditions returns nil for PythonLibrary (no export filtering)"+ (let [spec (spec/spec-for-model-key :model/PythonLibrary)]+ (is (nil? (spec/export-conditions spec)))))++ (testing "removal-conditions returns the builtin protection for PythonLibrary"+ (let [spec (spec/spec-for-model-key :model/PythonLibrary)]+ (is (= {:entity_id [:not= transforms-python/builtin-entity-id]}+ (spec/removal-conditions spec))))))++(deftest transform-tag-spec-uses-conditions-test+ (testing "TransformTag spec still uses :conditions (not split)"+ (let [spec (spec/spec-for-model-key :model/TransformTag)]+ (is (= {:built_in_type nil} (:conditions spec)))+ (is (nil? (:export-conditions spec)))+ (is (nil? (:removal-conditions spec)))+ (is (= {:built_in_type nil} (spec/export-conditions spec))+ "export-conditions falls back to :conditions for TransformTag")+ (is (= {:built_in_type nil} (spec/removal-conditions spec))+ "removal-conditions falls back to :conditions for TransformTag"))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex 0141acbd197..ef0a65f75a6 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -927,3 +927,39 @@ serdes/meta:"Custom tag (built_in_type nil) should be in export roots")(is (not (contains? exported-ids builtin-tag-id))"Built-in tag should NOT be in export roots")))))))++(deftest export-includes-builtin-python-library-test+ (testing "Export includes built-in PythonLibrary (common.py) since it has no export-conditions"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-temp [:model/PythonLibrary builtin-lib {:path "common.py"+ :source "# builtin"+ :entity_id transforms-python/builtin-entity-id}+ :model/PythonLibrary custom-lib {:path "custom.py"+ :source "# custom"}]+ (let [python-lib-spec (spec/spec-for-model-key :model/PythonLibrary)+ export-roots (spec/query-export-roots python-lib-spec)+ exported-ids (set (map second export-roots))]+ (is (contains? exported-ids (:id builtin-lib))+ "Built-in PythonLibrary should be in export roots")+ (is (contains? exported-ids (:id custom-lib))+ "Custom PythonLibrary should be in export roots")))))))++(deftest build-all-removal-paths-excludes-builtin-python-library-test+ (testing "build-all-removal-paths excludes built-in PythonLibrary via :removal-conditions"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-temp [:model/PythonLibrary _ {:path "common.py"+ :source "# builtin"+ :entity_id transforms-python/builtin-entity-id}+ :model/PythonLibrary custom-lib {:path "custom.py"+ :source "# custom"}]+ (settings/sync-transform-tracking! true)+ (settings/sync-transform-tracking! false)+ (let [paths (spec/build-all-removal-paths)]+ (is (some #(str/includes? % (:entity_id custom-lib)) paths)+ "Custom PythonLibrary should be in removal paths")+ (is (not (some #(str/includes? % transforms-python/builtin-entity-id) paths))+ "Built-in PythonLibrary should NOT be in removal paths")))))))