Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-4399: Remote Sync doesn't clean up nested content when a parent collection is archived, so orphaned files accumulate in git forever (#82876)

Eric Normandbb9482f0fda1
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index b50621aeb5d..5322d7ca3dc 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -3,7 +3,6 @@
additional logic, so no other namespace in the module runs a query itself."
(:require
[metabase-enterprise.remote-sync.schema :as remote-sync.schema]
- [metabase.collections.core :as collections]
[metabase.collections.schema :as collections.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
@@ -164,6 +163,12 @@
ids :- [:sequential ms/PositiveInt]]
(t2/select [model :id :name] :id [:in ids]))
+(mu/defn archived-by-id
+ "A map of ID to the `:archived` flag of the instances of `model` with `ids`."
+ [model :- :keyword
+ ids :- [:sequential ms/PositiveInt]]
+ (t2/select-pk->fn :archived model :id [:in ids]))
+
(mu/defn instances-in-collections
"The instances of `model` in the Collections with `collection-ids`, excluding those archived under the optional
`archived-key` column."
@@ -321,8 +326,9 @@
"Matches `collections` and all of their descendants."
[collections]
(into [:or [:in :id (map :id collections)]]
- (for [collection collections]
- [:like :location (str (collections/location-path collection) "%")])))
+ (for [{:keys [id location]} collections]
+ ;; the location its children have, so a nested collection's descendants match too
+ [:like :location (str location id "/%")])))
(mu/defn collections
"The Collections with `collection-ids`."
@@ -533,6 +539,13 @@
[collection-ids :- [:set ::lib.schema.id/collection]]
(t2/select [:model/RemoteSyncObject :id :status] {:where (contents-rso-expr collection-ids)}))
+(mu/defn content-rsos
+ "The `:id`, `:model_type`, `:model_id`, `:model_collection_id`, and `:status` of the RemoteSyncObjects of the
+ Collections with `collection-ids` and of their contents."
+ [collection-ids :- [:set ::lib.schema.id/collection]]
+ (t2/select [:model/RemoteSyncObject :id :model_type :model_id :model_collection_id :status]
+ {:where (contents-rso-expr collection-ids)}))
+
(mu/defn removed-content-rso-ids
"The IDs of the RemoteSyncObjects pending removal among those of the Collections with `collection-ids` and their
contents."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
index c0da33fb6dc..d387c17df6d 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
@@ -295,6 +295,37 @@
(log/info "Library collection is no longer remote-synced, disabling Library content sync tracking")
(disable-library-tracking!)))))
+(defn- cascade-archived-state!
+ "Brings the RemoteSyncObject rows of `collection`'s subtree, other than its own row, in line with the archived state
+ of each entity and of the collection it is in: rows of entities that are archived or in an archived collection
+ become 'delete', and 'delete' rows of existing entities that are neither become 'update'."
+ [collection]
+ ;; Archiving a collection archives its subtree in bulk SQL, which publishes no event per descendant. Transforms
+ ;; have no archived column, so only their collection shows that they were archived.
+ (let [collection-ids (remote-sync.db/subtree-collection-ids [collection])
+ collection-archived? (remote-sync.db/archived-by-id :model/Collection (vec collection-ids))
+ rows (->> (remote-sync.db/content-rsos collection-ids)
+ (remove #(and (= "Collection" (:model_type %)) (= (:id collection) (:model_id %)))))
+ now (t/offset-date-time)]
+ (doseq [[model-type type-rows] (group-by :model_type rows)
+ :let [{:keys [model-key archived-key]} (spec/spec-for-model-type model-type)]
+ :when (= :archived archived-key)
+ :let [entity-archived? (remote-sync.db/archived-by-id model-key (mapv :model_id type-rows))
+ archived? (fn [{:keys [model_id model_collection_id]}]
+ (or (true? (entity-archived? model_id))
+ (true? (collection-archived? model_collection_id))))
+ deleted (filter #(and (archived? %)
+ (not (contains? #{"delete" "removed"} (:status %))))
+ type-rows)
+ restored (filter #(and (contains? entity-archived? (:model_id %))
+ (not (archived? %))
+ (= "delete" (:status %)))
+ type-rows)]]
+ (when (seq deleted)
+ (remote-sync.db/set-rsos-status! (map :id deleted) "delete" now))
+ (when (seq restored)
+ (remote-sync.db/set-rsos-status! (map :id restored) "update" now)))))
+
(methodical/defmethod events/publish-event! ::collection-change-event
[topic event]
(let [{:keys [object]} event
@@ -313,7 +344,10 @@
should-sync?
(do
(log/infof "Creating remote sync object entry for collection %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details))
+ (create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details)
+ (when (and (= topic :event/collection-update)
+ (or (:archived object) (= "delete" (:status existing-entry))))
+ (cascade-archived-state! object)))
(and existing-entry (not should-sync?))
(do
(log/infof "Collection %s no longer needs syncing, marking as removed" (:id object))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj
index e92252e9e1f..97810b2434a 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj
@@ -3,9 +3,16 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [diehard.core :as dh]
[metabase-enterprise.remote-sync.settings :as settings]
+ [metabase-enterprise.remote-sync.source :as source]
+ [metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase.collections.models.collection :as collection]
+ [metabase.events.core :as events]
[metabase.test :as mt]
+ [metabase.util :as u]
+ [metabase.util.yaml :as yaml]
[toucan2.core :as t2]))
(use-fixtures :each
@@ -97,3 +104,96 @@
(is (false? (:is_remote_synced response)))
(is (nil? (t2/select-one-fn :parent_id :model/Collection :id (:id child-collection))))
(is (false? (t2/select-one-fn :is_remote_synced :model/Collection :id (:id child-collection)))))))))
+
+(defn- push-via-api!
+ "Pushes through `POST /api/ee/remote-sync/export` and waits for the export task to finish."
+ []
+ (let [{:keys [task_id]} (mt/user-http-request :crowberto :post 200 "ee/remote-sync/export" {:branch "main"})]
+ (dh/with-retry {:max-retries 20 :delay-ms 250}
+ (u/prog1 (t2/select-one :model/RemoteSyncTask :id task_id)
+ (when (nil? (:ended_at <>))
+ (throw (ex-info "Not finished" {:task-id task_id})))))))
+
+(defn- exported-entity-ids
+ "The entity_ids that have their own file in the current snapshot of `src`."
+ [src]
+ (let [snapshot (source.p/snapshot src)]
+ (into #{}
+ (keep (fn [path]
+ (try (:entity_id (yaml/parse-string (source.p/read-file snapshot path)))
+ (catch Exception _ nil))))
+ (source.p/list-files snapshot))))
+
+(deftest archive-collection-removes-nested-content-from-remote-test
+ (testing "GHY-4399: archiving a collection inside a synced collection removes its whole subtree from the remote
+ on the next push, not only the collection's own file"
+ (test-helpers/clean-remote-sync-state
+ (fn []
+ (test-helpers/commit-with-temp
+ (fn []
+ (mt/with-premium-features #{:remote-sync}
+ (mt/with-model-cleanup [:model/Card :model/Collection]
+ (mt/with-temp [:model/Collection {synced-id :id} {:name "Synced" :is_remote_synced true :location "/"}]
+ (let [src (test-helpers/versioned-source :current "v-remote" :trees {"v-remote" {}})]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"]
+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly src)]
+ (let [folder (mt/user-http-request :crowberto :post 200 "collection"
+ {:name "Bug Repro" :parent_id synced-id})
+ sub-folder (mt/user-http-request :crowberto :post 200 "collection"
+ {:name "Sub Folder" :parent_id (:id folder)})
+ new-card (fn [card-name coll-id]
+ (mt/user-http-request :crowberto :post 200 "card"
+ {:name card-name
+ :display "table"
+ :dataset_query (mt/mbql-query venues)
+ :visualization_settings {}
+ :collection_id coll-id}))
+ card (new-card "Card In Folder" (:id folder))
+ sub-card (new-card "Card In Sub Folder" (:id sub-folder))
+ subtree (set (map :entity_id [folder sub-folder card sub-card]))]
+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "first push pushes")
+ (is (= subtree (into #{} (filter subtree) (exported-entity-ids src)))
+ "precondition: the folder, its sub-folder, and both cards are on the remote")
+ (mt/user-http-request :crowberto :put 200 (str "collection/" (:id folder)) {:archived true})
+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "second push pushes")
+ (is (= #{} (into #{} (filter subtree) (exported-entity-ids src)))
+ "no file from the archived subtree stays on the remote"))))))))))))))
+
+(deftest archive-transforms-folder-removes-its-transforms-from-remote-test
+ (testing "GHY-4399: archiving a Transforms folder removes the transforms in it from the remote on the next push.
+ Transforms have no archived column, so only their folder being archived says they left."
+ (test-helpers/clean-remote-sync-state
+ (fn []
+ (test-helpers/commit-with-temp
+ (fn []
+ (mt/with-premium-features #{:remote-sync :transforms-basic}
+ (mt/with-model-cleanup [:model/Transform :model/Collection]
+ (let [src (test-helpers/versioned-source :current "v-remote" :trees {"v-remote" {}})]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"
+ remote-sync-transforms true]
+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly src)]
+ (let [folder (mt/user-http-request :crowberto :post 200 "collection"
+ {:name "Bug Repro" :namespace "transforms"})
+ ;; H2 cannot be a transform target, so POST /api/transform refuses it; insert the transform
+ ;; and publish the event that endpoint publishes
+ transform-id (t2/insert-returning-pk! :model/Transform
+ {:name "Transform In Folder"
+ :collection_id (:id folder)
+ :source {:type "query"
+ :query (mt/native-query {:query "SELECT 1"})}
+ :target {:type "table" :schema "PUBLIC" :name "ghy_4399"}})
+ transform (t2/select-one :model/Transform :id transform-id)
+ _ (events/publish-event! :event/transform-create
+ {:object transform :user-id (mt/user->id :crowberto)})
+ subtree (set (map :entity_id [folder transform]))]
+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "first push pushes")
+ (is (= subtree (into #{} (filter subtree) (exported-entity-ids src)))
+ "precondition: the folder and its transform are on the remote")
+ (mt/user-http-request :crowberto :put 200 (str "collection/" (:id folder)) {:archived true})
+ (is (=? {:outcome {:kind "pushed"}} (push-via-api!)) "second push pushes")
+ (is (= #{} (into #{} (filter subtree) (exported-entity-ids src)))
+ "no file from the archived folder stays on the remote")))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
index 9359f7094c5..489dc856fc0 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
@@ -416,6 +416,98 @@
:status "update"}
(first entries)))))))
+(defn- rso-statuses
+ "`{[model_type model_id] status}` for every RemoteSyncObject."
+ []
+ (into {} (map (juxt (juxt :model_type :model_id) :status)) (t2/select :model/RemoteSyncObject)))
+
+(defn- with-synced-nested-subtree!
+ "Calls `f` with `{:root :folder :sub-folder :card :sub-card :sibling-card}` ids: a remote-synced root holding a
+ sibling card and a folder, which holds a card and a sub-folder with its own card. Every entity except the root
+ has a synced RemoteSyncObject row."
+ [f]
+ (mt/with-temp [:model/Collection {root :id} {:name "Root" :is_remote_synced true :location "/"}
+ :model/Collection {folder :id} {:name "Folder" :is_remote_synced true :location (str "/" root "/")}
+ :model/Collection {sub-folder :id} {:name "Sub" :is_remote_synced true :location (str "/" root "/" folder "/")}
+ :model/Card {card :id} {:name "Card" :collection_id folder}
+ :model/Card {sub-card :id} {:name "Sub Card" :collection_id sub-folder}
+ :model/Card {sibling-card :id} {:name "Sibling Card" :collection_id root}]
+ (t2/delete! :model/RemoteSyncObject)
+ (t2/insert! :model/RemoteSyncObject
+ (for [[model-type model-id coll-id] [["Collection" folder folder]
+ ["Collection" sub-folder sub-folder]
+ ["Card" card folder]
+ ["Card" sub-card sub-folder]
+ ["Card" sibling-card root]]]
+ {:model_type model-type :model_id model-id :model_collection_id coll-id
+ :model_name "x" :status "synced" :status_changed_at (t/offset-date-time)}))
+ (f {:root root :folder folder :sub-folder sub-folder :card card :sub-card sub-card :sibling-card sibling-card})))
+
+(defn- set-subtree-archived!
+ "Sets `archived` on `folder` and its contents, as archiving and unarchiving a collection do in bulk SQL."
+ [{:keys [folder sub-folder card sub-card]} archived]
+ (t2/update! :model/Collection :id [:in [folder sub-folder]] {:archived archived})
+ (t2/update! :model/Card :id [:in [card sub-card]] {:archived archived}))
+
+(deftest collection-update-event-archived-cascades-delete-to-subtree-test
+ (testing "GHY-4399: archiving a nested collection marks its descendant collections and all contents of its subtree
+ 'delete', and leaves content outside the subtree alone"
+ (with-synced-nested-subtree!
+ (fn [{:keys [folder sub-folder card sub-card sibling-card] :as ids}]
+ (set-subtree-archived! ids true)
+ (events/publish-event! :event/collection-update
+ {:object (t2/select-one :model/Collection :id folder) :user-id (mt/user->id :rasta)})
+ (is (= {["Collection" folder] "delete"
+ ["Collection" sub-folder] "delete"
+ ["Card" card] "delete"
+ ["Card" sub-card] "delete"
+ ["Card" sibling-card] "synced"}
+ (rso-statuses)))))))
+
+(deftest collection-update-event-unarchived-restores-subtree-test
+ (testing "GHY-4399: unarchiving a collection before the next push marks its unarchived contents 'update' again, so
+ the push does not delete their files; content still archived on its own stays 'delete'"
+ (with-synced-nested-subtree!
+ (fn [{:keys [folder sub-folder card sub-card sibling-card] :as ids}]
+ (set-subtree-archived! ids true)
+ (events/publish-event! :event/collection-update
+ {:object (t2/select-one :model/Collection :id folder) :user-id (mt/user->id :rasta)})
+ (set-subtree-archived! ids false)
+ (t2/update! :model/Card :id sub-card {:archived true})
+ (events/publish-event! :event/collection-update
+ {:object (t2/select-one :model/Collection :id folder) :user-id (mt/user->id :rasta)})
+ (is (= {["Collection" folder] "update"
+ ["Collection" sub-folder] "update"
+ ["Card" card] "update"
+ ["Card" sub-card] "delete"
+ ["Card" sibling-card] "synced"}
+ (rso-statuses)))))))
+
+(deftest collection-update-event-archive-cascades-to-transforms-test
+ (testing "GHY-4399: archiving a Transforms folder marks the transforms in it 'delete', and unarchiving it marks them
+ 'update' again; transforms have no archived column, so their folder's archived state decides"
+ (mt/with-temporary-setting-values [remote-sync-transforms true]
+ (mt/with-temp [:model/Collection {folder :id} {:name "Folder" :namespace "transforms" :location "/"}
+ :model/Transform {transform :id} {:name "Transform" :collection_id folder}]
+ (t2/delete! :model/RemoteSyncObject)
+ (t2/insert! :model/RemoteSyncObject
+ (for [[model-type model-id] [["Collection" folder] ["Transform" transform]]]
+ {:model_type model-type :model_id model-id :model_collection_id folder
+ :model_name "x" :status "synced" :status_changed_at (t/offset-date-time)}))
+ (let [set-archived! (fn [archived]
+ (t2/update! :model/Collection :id folder {:archived archived})
+ (events/publish-event! :event/collection-update
+ {:object (t2/select-one :model/Collection :id folder)
+ :user-id (mt/user->id :rasta)}))]
+ (set-archived! true)
+ (is (= {["Collection" folder] "delete"
+ ["Transform" transform] "delete"}
+ (rso-statuses)))
+ (set-archived! false)
+ (is (= {["Collection" folder] "update"
+ ["Transform" transform] "update"}
+ (rso-statuses))))))))
+
(deftest collection-update-event-no-entry-for-normal-collection-test
(testing "collection-update event doesn't create entry for non-remote-synced collections"
(mt/with-temp [:model/Collection normal-collection {:name "Normal"}]