The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Un-publishing remote-sync'ed library tables should remove them from the git repo (#69757)
Nathan Voxland5da65b6c5262
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.cljindex 751f4d2aa90..df5ec237ba7 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj@@ -133,6 +133,33 @@:status_changed_at (t/offset-date-time)}fields))))))+(defn- cascade-filter+ "Derives the filter conditions for querying eligible children from a child spec."+ [child-spec]+ (or (:cascade-filter child-spec)+ (when-let [ak (:archived-key child-spec)]+ {ak false})))++(defn- cascade-to-children!+ "When a parent model becomes eligible/ineligible, cascade to its children.+ For eligible: query child entities using :parent-fk and derived filter, check eligibility, create RSOs.+ For ineligible: query existing RSOs by model_table_id and mark as removed."+ [model-spec model-id status eligible?]+ (doseq [child-spec (spec/children-specs (:model-key model-spec))]+ (let [fk (:parent-fk child-spec)+ filter (cascade-filter child-spec)]+ (if eligible?+ ;; Eligible branch: query actual entities and create RSOs for eligible children+ (doseq [child (apply t2/select (:model-key child-spec) (into [fk model-id] cat filter))]+ (when (spec/check-eligibility child-spec child)+ (create-or-update-sync-object-from-spec! child-spec (:id child) status)))+ ;; Ineligible branch: mark existing child RSOs as removed+ (doseq [child-rso (t2/select :model/RemoteSyncObject+ :model_type (:model-type child-spec)+ :model_table_id model-id+ :status [:not-in ["removed" "delete"]])]+ (create-or-update-sync-object-from-spec! child-spec (:model_id child-rso) "removed"))))))+(defn- handle-model-event-from-spec"Generic event handler that uses a spec for all configuration.Checks eligibility, determines status, and creates/updates the sync object."@@ -147,11 +174,15 @@(do(log/infof "Creating remote sync object entry for %s %s (status: %s)"model-type model-id status)- (create-or-update-sync-object-from-spec! model-spec model-id status))+ (create-or-update-sync-object-from-spec! model-spec model-id status)+ (when (seq (spec/children-specs (:model-key model-spec)))+ (cascade-to-children! model-spec model-id status true)))(and existing-entry (not eligible?))(do(log/infof "%s %s moved out of sync scope, marking as removed" model-type model-id)- (create-or-update-sync-object-from-spec! model-spec model-id "removed")))))+ (create-or-update-sync-object-from-spec! model-spec model-id "removed")+ (when (seq (spec/children-specs (:model-key model-spec)))+ (cascade-to-children! model-spec model-id "removed" false))))))(defn- register-events-for-spec!"Registers event handlers for a single spec. Creates event hierarchy anddiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 5ad9be679e0..b8705b2f526 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -38,6 +38,10 @@- :path-keys - For :path or :hybrid identity: vector of path components [:database :schema :table :field]- :parent-model - For :parent-table eligibility: the parent model key to check eligibility against(e.g., :model/Table for Field, Segment, Measure)+ - :parent-fk - For child models: the FK column pointing to the parent (e.g., :table_id)+ - :cascade-filter - Optional map of additional filter conditions for cascade queries.+ Only needed when the filter differs from {archived-key false}.+ E.g., Field needs {:active true} since it has no :archived-key.- :delete-after - Optional vector of model keys that must be deleted AFTER this model.Used to handle FK constraints during import cleanup. For example, if Cardhas :delete-after [:model/Collection], Card will be deleted before Collection.@@ -179,7 +183,7 @@:identity :path:path-keys [:database :schema :table]:events {:prefix :event/table- :types [:create :update :delete]}+ :types [:create :update :delete :publish :unpublish]}:eligibility {:type :published-table}:archived-key :archived_at ; tables use archived_at, not archived:tracking {:select-fields [:name :collection_id]@@ -196,6 +200,8 @@:identity :path:path-keys [:database :schema :table :field]:parent-model :model/Table+ :parent-fk :table_id+ :cascade-filter {:active true}:events {:prefix :event/field:types [:create :update :delete]}:eligibility {:type :parent-table}@@ -344,6 +350,15 @@[model-key](get remote-sync-specs model-key))+(defn children-specs+ "Returns child specs for a given parent model key, derived from :parent-model references."+ [parent-model-key]+ (into []+ (comp (map val)+ (filter #(and (= (:parent-model %) parent-model-key)+ (:parent-fk %))))+ remote-sync-specs))+(defn specs-by-identity-type"Returns a map of model-key -> spec filtered by identity type."[identity-type]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.cljindex f1a57601c06..3d25295770e 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj@@ -53,8 +53,8 @@"events :prefix should be a keyword")(is (vector? (get-in spec [:events :types]))"events :types should be a vector")- (is (every? #{:create :update :delete} (get-in spec [:events :types]))- "events :types should only contain :create, :update, :delete")))))+ (is (every? #{:create :update :delete :publish :unpublish} (get-in spec [:events :types]))+ "events :types should only contain :create, :update, :delete, :publish, :unpublish")))))(deftest all-specs-have-valid-tracking-test(testing "Every spec has valid tracking configuration"@@ -78,6 +78,27 @@(is (keyword? scope-key)"removal :scope-key should be a keyword when present"))))))+(deftest parent-fk-specs-are-valid-test+ (testing "Every spec with :parent-model and :parent-fk has a keyword :parent-fk"+ (doseq [[model-key spec] spec/remote-sync-specs+ :when (and (:parent-model spec)+ (:parent-fk spec))]+ (testing (str "Spec for " model-key)+ (is (keyword? (:parent-fk spec))+ ":parent-fk should be a keyword")+ (when-let [cf (:cascade-filter spec)]+ (is (map? cf)+ ":cascade-filter should be a map when present")))))++ (testing "children-specs derives the correct children for Table"+ (let [children (spec/children-specs :model/Table)]+ (is (= 1 (count children)))+ (is (= #{:model/Field}+ (into #{} (map :model-key) children)))))++ (testing "children-specs returns empty for models with no children"+ (is (empty? (spec/children-specs :model/Card)))))+;;; ------------------------------------------------ Helper Function Tests ---------------------------------------------(deftest spec-for-model-type-test