The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-4399: Remote Sync doesn't clean up nested content when a parent collection is archived, so orphaned files accumulate in git forever (#82876)
Eric Normandbb9482f0fda1
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.cljindex b50621aeb5d..5322d7ca3dc 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj@@ -3,7 +3,6 @@additional logic, so no other namespace in the module runs a query itself."(:require[metabase-enterprise.remote-sync.schema :as remote-sync.schema]- [metabase.collections.core :as collections][metabase.collections.schema :as collections.schema][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli :as mu]@@ -164,6 +163,12 @@ids :- [:sequential ms/PositiveInt]](t2/select [model :id :name] :id [:in ids]))+(mu/defn archived-by-id+ "A map of ID to the `:archived` flag of the instances of `model` with `ids`."+ [model :- :keyword+ ids :- [:sequential ms/PositiveInt]]+ (t2/select-pk->fn :archived model :id [:in ids]))+(mu/defn instances-in-collections"The instances of `model` in the Collections with `collection-ids`, excluding those archived under the optional`archived-key` column."@@ -321,8 +326,9 @@"Matches `collections` and all of their descendants."[collections](into [:or [:in :id (map :id collections)]]- (for [collection collections]- [:like :location (str (collections/location-path collection) "%")])))+ (for [{:keys [id location]} collections]+ ;; the location its children have, so a nested collection's descendants match too+ [:like :location (str location id "/%")])))(mu/defn collections"The Collections with `collection-ids`."@@ -533,6 +539,13 @@[collection-ids :- [:set ::lib.schema.id/collection]](t2/select [:model/RemoteSyncObject :id :status] {:where (contents-rso-expr collection-ids)}))+(mu/defn content-rsos+ "The `:id`, `:model_type`, `:model_id`, `:model_collection_id`, and `:status` of the RemoteSyncObjects of the+ Collections with `collection-ids` and of their contents."+ [collection-ids :- [:set ::lib.schema.id/collection]]+ (t2/select [:model/RemoteSyncObject :id :model_type :model_id :model_collection_id :status]+ {:where (contents-rso-expr collection-ids)}))+(mu/defn removed-content-rso-ids"The IDs of the RemoteSyncObjects pending removal among those of the Collections with `collection-ids` and theircontents."diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.cljindex c0da33fb6dc..d387c17df6d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj@@ -295,6 +295,37 @@(log/info "Library collection is no longer remote-synced, disabling Library content sync tracking")(disable-library-tracking!)))))+(defn- cascade-archived-state!+ "Brings the RemoteSyncObject rows of `collection`'s subtree, other than its own row, in line with the archived state+ of each entity and of the collection it is in: rows of entities that are archived or in an archived collection+ become 'delete', and 'delete' rows of existing entities that are neither become 'update'."+ [collection]+ ;; Archiving a collection archives its subtree in bulk SQL, which publishes no event per descendant. Transforms+ ;; have no archived column, so only their collection shows that they were archived.+ (let [collection-ids (remote-sync.db/subtree-collection-ids [collection])+ collection-archived? (remote-sync.db/archived-by-id :model/Collection (vec collection-ids))+ rows (->> (remote-sync.db/content-rsos collection-ids)+ (remove #(and (= "Collection" (:model_type %)) (= (:id collection) (:model_id %)))))+ now (t/offset-date-time)]+ (doseq [[model-type type-rows] (group-by :model_type rows)+ :let [{:keys [model-key archived-key]} (spec/spec-for-model-type model-type)]+ :when (= :archived archived-key)+ :let [entity-archived? (remote-sync.db/archived-by-id model-key (mapv :model_id type-rows))+ archived? (fn [{:keys [model_id model_collection_id]}]+ (or (true? (entity-archived? model_id))+ (true? (collection-archived? model_collection_id))))+ deleted (filter #(and (archived? %)+ (not (contains? #{"delete" "removed"} (:status %))))+ type-rows)+ restored (filter #(and (contains? entity-archived? (:model_id %))+ (not (archived? %))+ (= "delete" (:status %)))+ type-rows)]]+ (when (seq deleted)+ (remote-sync.db/set-rsos-status! (map :id deleted) "delete" now))+ (when (seq restored)+ (remote-sync.db/set-rsos-status! (map :id restored) "update" now)))))+(methodical/defmethod events/publish-event! ::collection-change-event[topic event](let [{:keys [object]} event@@ -313,7 +344,10 @@should-sync?(do(log/infof "Creating remote sync object entry for collection %s (status: %s)" (:id object) status)- (create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details))+ (create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details)+ (when (and (= topic :event/collection-update)+ (or (:archived object) (= "delete" (:status existing-entry))))+ (cascade-archived-state! object)))(and existing-entry (not should-sync?))(do(log/infof "Collection %s no longer needs syncing, marking as removed" (:id object))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.cljindex e92252e9e1f..97810b2434a 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj@@ -3,9 +3,16 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [diehard.core :as dh][metabase-enterprise.remote-sync.settings :as settings]+ [metabase-enterprise.remote-sync.source :as source]+ [metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase.collections.models.collection :as collection]+ [metabase.events.core :as events][metabase.test :as mt]+ [metabase.util :as u]+ [metabase.util.yaml :as yaml][toucan2.core :as t2]))(use-fixtures :each@@ -97,3 +104,96 @@(is (false? (:is_remote_synced response)))(is (nil? (t2/select-one-fn :parent_id :model/Collection :id (:id child-collection))))(is (false? (t2/select-one-fn :is_remote_synced :model/Collection :id (:id child-collection)))))))))++(defn- push-via-api!+ "Pushes through `POST /api/ee/remote-sync/export` and waits for the export task to finish."+ []+ (let [{:keys [task_id]} (mt/user-http-request :crowberto :post 200 "ee/remote-sync/export" {:branch "main"})]+ (dh/with-retry {:max-retries 20 :delay-ms 250}+ (u/prog1 (t2/select-one :model/RemoteSyncTask :id task_id)+ (when (nil? (:ended_at <>))+ (throw (ex-info "Not finished" {:task-id task_id})))))))++(defn- exported-entity-ids+ "The entity_ids that have their own file in the current snapshot of `src`."+ [src]+ (let [snapshot (source.p/snapshot src)]+ (into #{}+ (keep (fn [path]+ (try (:entity_id (yaml/parse-string (source.p/read-file snapshot path)))+ (catch Exception _ nil))))+ (source.p/list-files snapshot))))++(deftest archive-collection-removes-nested-content-from-remote-test+ (testing "GHY-4399: archiving a collection inside a synced collection removes its whole subtree from the remote+ on the next push, not only the collection's own file"+ (test-helpers/clean-remote-sync-state+ (fn []+ (test-helpers/commit-with-temp+ (fn []+ (mt/with-premium-features #{:remote-sync}+ (mt/with-model-cleanup [:model/Card :model/Collection]+ (mt/with-temp [:model/Collection {synced-id :id} {:name "Synced" :is_remote_synced true :location "/"}]+ (let [src (test-helpers/versioned-source :current "v-remote" :trees {"v-remote" {}})]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"]+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly src)]+ (let [folder (mt/user-http-request :crowberto :post 200 "collection"+ {:name "Bug Repro" :parent_id synced-id})+ sub-folder (mt/user-http-request :crowberto :post 200 "collection"+ {:name "Sub Folder" :parent_id (:id folder)})+ new-card (fn [card-name coll-id]+ (mt/user-http-request :crowberto :post 200 "card"+ {:name card-name+ :display "table"+ :dataset_query (mt/mbql-query venues)+ :visualization_settings {}+ :collection_id coll-id}))+ card (new-card "Card In Folder" (:id folder))+ sub-card (new-card "Card In Sub Folder" (:id sub-folder))+ subtree (set (map :entity_id [folder sub-folder card sub-card]))]+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "first push pushes")+ (is (= subtree (into #{} (filter subtree) (exported-entity-ids src)))+ "precondition: the folder, its sub-folder, and both cards are on the remote")+ (mt/user-http-request :crowberto :put 200 (str "collection/" (:id folder)) {:archived true})+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "second push pushes")+ (is (= #{} (into #{} (filter subtree) (exported-entity-ids src)))+ "no file from the archived subtree stays on the remote"))))))))))))))++(deftest archive-transforms-folder-removes-its-transforms-from-remote-test+ (testing "GHY-4399: archiving a Transforms folder removes the transforms in it from the remote on the next push.+ Transforms have no archived column, so only their folder being archived says they left."+ (test-helpers/clean-remote-sync-state+ (fn []+ (test-helpers/commit-with-temp+ (fn []+ (mt/with-premium-features #{:remote-sync :transforms-basic}+ (mt/with-model-cleanup [:model/Transform :model/Collection]+ (let [src (test-helpers/versioned-source :current "v-remote" :trees {"v-remote" {}})]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-transforms true]+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly src)]+ (let [folder (mt/user-http-request :crowberto :post 200 "collection"+ {:name "Bug Repro" :namespace "transforms"})+ ;; H2 cannot be a transform target, so POST /api/transform refuses it; insert the transform+ ;; and publish the event that endpoint publishes+ transform-id (t2/insert-returning-pk! :model/Transform+ {:name "Transform In Folder"+ :collection_id (:id folder)+ :source {:type "query"+ :query (mt/native-query {:query "SELECT 1"})}+ :target {:type "table" :schema "PUBLIC" :name "ghy_4399"}})+ transform (t2/select-one :model/Transform :id transform-id)+ _ (events/publish-event! :event/transform-create+ {:object transform :user-id (mt/user->id :crowberto)})+ subtree (set (map :entity_id [folder transform]))]+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "first push pushes")+ (is (= subtree (into #{} (filter subtree) (exported-entity-ids src)))+ "precondition: the folder and its transform are on the remote")+ (mt/user-http-request :crowberto :put 200 (str "collection/" (:id folder)) {:archived true})+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "second push pushes")+ (is (= #{} (into #{} (filter subtree) (exported-entity-ids src)))+ "no file from the archived folder stays on the remote")))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.cljindex 9359f7094c5..489dc856fc0 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj@@ -416,6 +416,98 @@:status "update"}(first entries)))))))+(defn- rso-statuses+ "`{[model_type model_id] status}` for every RemoteSyncObject."+ []+ (into {} (map (juxt (juxt :model_type :model_id) :status)) (t2/select :model/RemoteSyncObject)))++(defn- with-synced-nested-subtree!+ "Calls `f` with `{:root :folder :sub-folder :card :sub-card :sibling-card}` ids: a remote-synced root holding a+ sibling card and a folder, which holds a card and a sub-folder with its own card. Every entity except the root+ has a synced RemoteSyncObject row."+ [f]+ (mt/with-temp [:model/Collection {root :id} {:name "Root" :is_remote_synced true :location "/"}+ :model/Collection {folder :id} {:name "Folder" :is_remote_synced true :location (str "/" root "/")}+ :model/Collection {sub-folder :id} {:name "Sub" :is_remote_synced true :location (str "/" root "/" folder "/")}+ :model/Card {card :id} {:name "Card" :collection_id folder}+ :model/Card {sub-card :id} {:name "Sub Card" :collection_id sub-folder}+ :model/Card {sibling-card :id} {:name "Sibling Card" :collection_id root}]+ (t2/delete! :model/RemoteSyncObject)+ (t2/insert! :model/RemoteSyncObject+ (for [[model-type model-id coll-id] [["Collection" folder folder]+ ["Collection" sub-folder sub-folder]+ ["Card" card folder]+ ["Card" sub-card sub-folder]+ ["Card" sibling-card root]]]+ {:model_type model-type :model_id model-id :model_collection_id coll-id+ :model_name "x" :status "synced" :status_changed_at (t/offset-date-time)}))+ (f {:root root :folder folder :sub-folder sub-folder :card card :sub-card sub-card :sibling-card sibling-card})))++(defn- set-subtree-archived!+ "Sets `archived` on `folder` and its contents, as archiving and unarchiving a collection do in bulk SQL."+ [{:keys [folder sub-folder card sub-card]} archived]+ (t2/update! :model/Collection :id [:in [folder sub-folder]] {:archived archived})+ (t2/update! :model/Card :id [:in [card sub-card]] {:archived archived}))++(deftest collection-update-event-archived-cascades-delete-to-subtree-test+ (testing "GHY-4399: archiving a nested collection marks its descendant collections and all contents of its subtree+ 'delete', and leaves content outside the subtree alone"+ (with-synced-nested-subtree!+ (fn [{:keys [folder sub-folder card sub-card sibling-card] :as ids}]+ (set-subtree-archived! ids true)+ (events/publish-event! :event/collection-update+ {:object (t2/select-one :model/Collection :id folder) :user-id (mt/user->id :rasta)})+ (is (= {["Collection" folder] "delete"+ ["Collection" sub-folder] "delete"+ ["Card" card] "delete"+ ["Card" sub-card] "delete"+ ["Card" sibling-card] "synced"}+ (rso-statuses)))))))++(deftest collection-update-event-unarchived-restores-subtree-test+ (testing "GHY-4399: unarchiving a collection before the next push marks its unarchived contents 'update' again, so+ the push does not delete their files; content still archived on its own stays 'delete'"+ (with-synced-nested-subtree!+ (fn [{:keys [folder sub-folder card sub-card sibling-card] :as ids}]+ (set-subtree-archived! ids true)+ (events/publish-event! :event/collection-update+ {:object (t2/select-one :model/Collection :id folder) :user-id (mt/user->id :rasta)})+ (set-subtree-archived! ids false)+ (t2/update! :model/Card :id sub-card {:archived true})+ (events/publish-event! :event/collection-update+ {:object (t2/select-one :model/Collection :id folder) :user-id (mt/user->id :rasta)})+ (is (= {["Collection" folder] "update"+ ["Collection" sub-folder] "update"+ ["Card" card] "update"+ ["Card" sub-card] "delete"+ ["Card" sibling-card] "synced"}+ (rso-statuses)))))))++(deftest collection-update-event-archive-cascades-to-transforms-test+ (testing "GHY-4399: archiving a Transforms folder marks the transforms in it 'delete', and unarchiving it marks them+ 'update' again; transforms have no archived column, so their folder's archived state decides"+ (mt/with-temporary-setting-values [remote-sync-transforms true]+ (mt/with-temp [:model/Collection {folder :id} {:name "Folder" :namespace "transforms" :location "/"}+ :model/Transform {transform :id} {:name "Transform" :collection_id folder}]+ (t2/delete! :model/RemoteSyncObject)+ (t2/insert! :model/RemoteSyncObject+ (for [[model-type model-id] [["Collection" folder] ["Transform" transform]]]+ {:model_type model-type :model_id model-id :model_collection_id folder+ :model_name "x" :status "synced" :status_changed_at (t/offset-date-time)}))+ (let [set-archived! (fn [archived]+ (t2/update! :model/Collection :id folder {:archived archived})+ (events/publish-event! :event/collection-update+ {:object (t2/select-one :model/Collection :id folder)+ :user-id (mt/user->id :rasta)}))]+ (set-archived! true)+ (is (= {["Collection" folder] "delete"+ ["Transform" transform] "delete"}+ (rso-statuses)))+ (set-archived! false)+ (is (= {["Collection" folder] "update"+ ["Transform" transform] "update"}+ (rso-statuses))))))))+(deftest collection-update-event-no-entry-for-normal-collection-test(testing "collection-update event doesn't create entry for non-remote-synced collections"(mt/with-temp [:model/Collection normal-collection {:name "Normal"}]