The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix: dependency check for snippet eligibility (#69024)
Edward Paget13389939e4fb
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/core.cljindex 036c6375434..18919622f1f 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj@@ -78,6 +78,15 @@[model-key instances](spec/batch-model-editable? model-key instances))+(defenterprise batch-model-eligible?+ "Batch check if model instances are eligible for remote sync based on spec rules.+ Returns a map of instance-id -> eligible? boolean."+ :feature :none+ [model-key instances]+ (if-let [spec (spec/spec-for-model-key model-key)]+ (spec/batch-check-eligibility spec instances)+ (into {} (map (fn [inst] [(:id inst) false])) instances)))+(mu/defn bulk-set-remote-sync :- :nil"Sets remote sync to true/false on one or collections in a single transaction. Checks that the remote sync stateafterwards is consistent in terms of dependency rules. Collections are provided as a map of collection-id -> sync state."diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.cljindex 83e872d00e1..e09cafc1569 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase-enterprise.remote-sync.core :as core]+ [metabase.collections.test-utils :refer [with-library-synced with-library-not-synced]][metabase.events.core :as events][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -260,3 +261,45 @@(is (false? (:is_remote_synced (t2/select-one :model/Collection :id parent-id))))(is (false? (:is_remote_synced (t2/select-one :model/Collection :id child1-id))))(is (false? (:is_remote_synced (t2/select-one :model/Collection :id child2-id)))))))++;;; ------------------------------------------- batch-model-eligible? Tests -------------------------------------------++(deftest batch-model-eligible-for-remote-sync-cards-in-remote-synced-collection-test+ (testing "returns true for cards in remote-synced collection"+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced true}+ :model/Card card1 {:collection_id coll-id}+ :model/Card card2 {:collection_id coll-id}]+ (is (= {(:id card1) true, (:id card2) true}+ (core/batch-model-eligible? :model/Card [card1 card2]))))))++(deftest batch-model-eligible-for-remote-sync-cards-not-in-remote-synced-collection-test+ (testing "returns false for cards NOT in remote-synced collection"+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced false}+ :model/Card card {:collection_id coll-id}]+ (is (= {(:id card) false}+ (core/batch-model-eligible? :model/Card [card]))))))++(deftest batch-model-eligible-for-remote-sync-snippets-library-synced-test+ (testing "snippets eligible when Library is synced"+ (with-library-synced+ (mt/with-temp [:model/Collection {snippet-coll-id :id} {:namespace "snippets"}+ :model/NativeQuerySnippet snippet {:name "Test Snippet"+ :content "SELECT 1"+ :collection_id snippet-coll-id}]+ (is (= {(:id snippet) true}+ (core/batch-model-eligible? :model/NativeQuerySnippet [snippet])))))))++(deftest batch-model-eligible-for-remote-sync-snippets-library-not-synced-test+ (testing "snippets NOT eligible when Library is not synced"+ (with-library-not-synced+ (mt/with-temp [:model/Collection {snippet-coll-id :id} {:namespace "snippets"}+ :model/NativeQuerySnippet snippet {:name "Test Snippet"+ :content "SELECT 1"+ :collection_id snippet-coll-id}]+ (is (= {(:id snippet) false}+ (core/batch-model-eligible? :model/NativeQuerySnippet [snippet])))))))++(deftest batch-model-eligible-for-remote-sync-unknown-model-test+ (testing "unknown model returns false for all instances"+ (is (= {1 false, 2 false}+ (core/batch-model-eligible? :model/UnknownModel [{:id 1} {:id 2}])))))diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex a6acef491f4..d149989b840 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -1172,26 +1172,72 @@accum(recur to-traverse traversed accum)))))-(defn remote-synced-dependents- "Finds dependents of a model that are contained in ANY remote-synced collection.-- For cards, checks if there are any other queries that reference this card. For collections, checks dependents of- all cards in the collection or subcollections. For all other models it returns an empty seq. Only checks for- immediate dependents.+(defn- select-for-eligibility-check+ "Select instances with fields needed for remote-sync eligibility checking.+ Returns nil for nested models like DashboardCard that don't need separate eligibility checks."+ [model-key ids]+ (when (seq ids)+ (case model-key+ :model/Collection+ ;; Collections use their own :id for eligibility, plus namespace and is_remote_synced+ (t2/select [model-key :id :namespace :is_remote_synced] :id [:in ids])+ :model/Table+ (t2/select [model-key :id :collection_id :is_published] :id [:in ids])+ ;; DashboardCard and DashboardCardSeries are nested under Dashboard - skip separate eligibility check+ (:model/DashboardCard :model/DashboardCardSeries)+ nil+ ;; Default - most models just need id and collection_id+ (t2/select [model-key :id :collection_id] :id [:in ids]))))++(defn- filter-eligible-dependents+ "Filter a list of dependent info maps to only include those that are eligible for remote sync.+ Each dependent-info is a map like {\"Card\" 123} or {\"Dashboard\" 456}.+ Returns only those where the model instance is eligible for remote sync.+ Nested models (like DashboardCard) are always considered eligible since their parent is checked."+ [dependent-infos]+ (let [;; Group by model type and collect IDs+ grouped (reduce (fn [acc dependent-info]+ (reduce-kv (fn [acc' model-name inst-id]+ (update acc' model-name (fnil conj #{}) inst-id))+ acc+ dependent-info))+ {}+ dependent-infos)+ ;; For each model type, check eligibility+ eligible-ids (reduce-kv (fn [acc model-name ids]+ (let [model-key (keyword "model" model-name)+ instances (select-for-eligibility-check model-key ids)]+ (if (nil? instances)+ ;; Nested models (DashboardCard, etc.) - all are eligible+ (assoc acc model-name ids)+ ;; Check eligibility via batch function+ (let [eligibility-map (remote-sync/batch-model-eligible? model-key instances)+ eligible (into #{} (keep (fn [[iid eligible?]] (when eligible? iid))) eligibility-map)]+ (assoc acc model-name eligible)))))+ {}+ grouped)]+ ;; Filter original list to keep only eligible ones+ (filter (fn [dependent-info]+ (every? (fn [[model-name inst-id]]+ (contains? (get eligible-ids model-name #{}) inst-id))+ dependent-info))+ dependent-infos)))- All remote-synced collections are treated as a unified pool - dependents in any remote-synced collection- (regardless of root) are returned.+(defn remote-synced-dependents+ "Finds dependents of a model that are eligible for remote sync.+ Uses spec-based eligibility rules which account for special cases like+ snippets (eligible when Library is synced, not by collection).Takes model (the model to check dependents for).- Returns a sequence of models immediately dependent on the provided model that are in any remote-synced collection."+ Returns a sequence of {model-name id} maps for dependents that are eligible for remote sync."[{:keys [id archived] :as model}](let [;; Get ALL top-level remote-synced collectionsall-remote-synced-roots (t2/select-pks-set :model/Collection{:where [:and[:= :is_remote_synced true][:= :location "/"]]})- ;; Traverse descendants of all remote-synced roots combined+ ;; Traverse descendants of all remote-synced roots combinedall-remote-synced-descendants (reduce (fn [accum root-id](merge-with concat accum(traverse-descendants ["Collection" root-id] true)))@@ -1199,38 +1245,39 @@all-remote-synced-roots)](case (t2/model model):model/Collection- ;; Get all descendants of the collection being moved and see if any of them are present in the- ;; set of all descendants across all remote-synced roots- (->> (traverse-descendants ["Collection" id] (not archived))- keys- (mapcat #(get all-remote-synced-descendants %)))-- ;; If this is not a collection see if the provided model appears anywhere in the descendants of- ;; any remote-synced collection- (get all-remote-synced-descendants [(name (t2/model model)) id] []))))+ ;; Get all descendants of the collection being moved and see if any of them are present in the+ ;; set of all descendants across all remote-synced roots, then filter by eligibility+ (let [all-dependents (->> (traverse-descendants ["Collection" id] (not archived))+ keys+ (mapcat #(get all-remote-synced-descendants %)))]+ (filter-eligible-dependents all-dependents))++ ;; If this is not a collection see if the provided model appears anywhere in the descendants of+ ;; any remote-synced collection, then filter by eligibility+ (let [direct-dependents (get all-remote-synced-descendants [(name (t2/model model)) id] [])]+ (filter-eligible-dependents direct-dependents)))))(defn non-remote-synced-dependencies- "Finds dependencies of a model that are not contained in ANY remote-synced collection.-- Checks for dependencies that could be contained in a remote-synced collection but are not.- All remote-synced collections are treated as a unified pool - a dependency in any remote-synced- collection (regardless of root) is considered valid.-- Uses serdes/descendants to list dependencies of a model.+ "Finds dependencies of a model that are not eligible for remote sync.+ Uses spec-based eligibility rules which account for special cases like+ snippets (eligible when Library is synced, not by collection).Takes model (the model to check dependencies for).- Returns a set of model IDs for dependencies of the given model that are not in any remote-synced collection."+ Returns a set of model IDs for dependencies of the given model that are not eligible for remote sync."[{:keys [id] :as model}](if (t2/select-one :model/Collection :id (if (= (t2/model model) :model/Collection) (:id model) (:collection_id model)))(let [descendants (u/group-by first second (keys (traverse-descendants [(name (t2/model model)) id] true)))]- (apply set/union (for [m (collectable-models)- :let [key (name m)]]- (set/difference (set (get descendants key))- (t2/select-pks-set m {:inner-join [[:collection :c]- [:and- [:= :c.id :collection_id]- [:= :c.is_remote_synced true]]]})))))+ (apply set/union+ (for [m (collectable-models)+ :let [key (name m)+ descendant-ids (set (get descendants key))]+ :when (seq descendant-ids)]+ (let [instances (select-for-eligibility-check m descendant-ids)+ eligibility-map (remote-sync/batch-model-eligible? m instances)]+ (into #{}+ (keep (fn [[inst-id eligible?]] (when-not eligible? inst-id)))+ eligibility-map)))))#{}))(defn check-non-remote-synced-dependenciesdiff --git a/src/metabase/remote_sync/core.clj b/src/metabase/remote_sync/core.cljindex 28028114e75..ad532cabd8f 100644--- a/src/metabase/remote_sync/core.clj+++ b/src/metabase/remote_sync/core.clj@@ -1,6 +1,7 @@(ns metabase.remote-sync.core(:require- [metabase.premium-features.core :refer [defenterprise]]))+ [metabase.premium-features.core :refer [defenterprise]]+ [toucan2.core :as t2]))(defenterprise collection-editable?"Returns if remote-synced collections are editable. Takes a collection to check for eligibility.@@ -47,3 +48,34 @@metabase-enterprise.remote-sync.core[_model-key instances](into {} (map (fn [inst] [(:id inst) true])) instances))++(defenterprise batch-model-eligible?+ "Batch check if model instances are eligible for remote sync based on spec rules.+ Returns a map of instance-id -> eligible? boolean.++ This checks if instances would be synced when remote sync is active, accounting+ for special eligibility types like :library-synced for snippets.++ OSS uses collection-based eligibility: an instance is eligible if it's in a collection+ with is_remote_synced=true. Collections are eligible if they have is_remote_synced=true.+ EE extends this with spec-based eligibility rules for special models like snippets+ (Library-synced) and transforms (setting-based)."+ metabase-enterprise.remote-sync.core+ [model-key instances]+ (if (= model-key :model/Collection)+ ;; For Collections, check their own is_remote_synced flag+ (into {}+ (map (fn [inst]+ [(:id inst) (boolean (:is_remote_synced inst))]))+ instances)+ ;; For other models, check if they're in a remote-synced collection+ (let [collection-ids (into #{} (keep :collection_id) instances)+ remote-synced-coll-ids (when (seq collection-ids)+ (t2/select-pks-set :model/Collection+ :id [:in collection-ids]+ :is_remote_synced true))]+ (into {}+ (map (fn [inst]+ [(:id inst)+ (boolean (contains? remote-synced-coll-ids (:collection_id inst)))]))+ instances))))