Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix git sync to remove stale YAML files from managed export directories (#71739)

Edward Paget9e576f007b56
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index f3f3e1a1c3f..26f0da00493 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -242,9 +242,7 @@
(let [snapshot-version (source.p/version snapshot)
last-imported-version (remote-sync.task/last-version)
first-import? (nil? last-imported-version)
- path-filters [#"collections/.*" #"databases/.*" #"actions/.*"
- ;; python-libraries is the old name for python_libraries, support both for backwards compatibility
- #"transforms/.*" #"python[_-]libraries/.*" #"snippets/.*"]
+ path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)
base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
has-transforms? (snapshot-has-transforms? base-ingestable)
{:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)
@@ -315,8 +313,7 @@
(if-let [models (spec/extract-entities-for-export)]
(do
(remote-sync.task/update-progress! task-id 0.3)
- (let [all-delete-prefixes (spec/build-all-removal-paths)
- written-version (source/store! models all-delete-prefixes snapshot task-id message)]
+ (let [written-version (source/store! models snapshot task-id message)]
(remote-sync.task/set-version! task-id written-version))
(t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})
{:status :success
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index e5fe3e1dc14..873b0dac70b 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -181,7 +181,7 @@
(throw (ex-info "Invalid repository URL: only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)"
{:url remote-sync-url})))
- (let [source (git/git-source remote-sync-url "HEAD" remote-sync-token)]
+ (let [source (git/git-source remote-sync-url "HEAD" remote-sync-token nil)]
(when (and (= :read-only remote-sync-type) (not (str/blank? remote-sync-branch)) (not (some #{remote-sync-branch} (git/branches source))))
(throw (ex-info "Invalid branch name" {:url remote-sync-url :branch remote-sync-branch}))))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index db71503007a..1602ae72bf0 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -54,36 +54,30 @@
(str/join File/separator (concat dirnames [basename]))))
(defn- ->file-spec
- "Converts entity from serdes stream into file spec for source write-files! "
+ "Converts entity from serdes stream into file spec for source write-files!"
[task-id count opts idx entity]
(when (instance? Exception entity)
- ;; Just short-circuit if there are errors.
(throw entity))
- (u/prog1 (if (string? entity)
- ;; when an entity is a string that means it's a path to delete
- {:path entity
- :remove? true}
- {:path (remote-sync-path opts entity)
- :content (yaml/generate-string (serialization/serialization-deep-sort entity)
- {:dumper-options {:flow-style :block :split-lines false}})})
+ (u/prog1 {:path (remote-sync-path opts entity)
+ :content (yaml/generate-string (serialization/serialization-deep-sort entity)
+ {:dumper-options {:flow-style :block :split-lines false}})}
(remote-sync.task/update-progress! task-id (-> (inc idx) (/ count) (* 0.65) (+ 0.3)))))
(defn store!
"Stores serialized entities from a stream to a remote source and commits the changes.
- Takes a stream (a sequence of serialized entities to be stored), a list of prefixes to recursively-delete at,
- a snapshot (the remote source implementing the SourceSnapshot protocol where files will be written), a task-id
- (the RemoteSyncTask identifier used to track progress updates), and a message (the commit message to use when
- writing files to the source).
+ Takes a stream (a sequence of serialized entities to be stored), a snapshot (the remote source
+ implementing the SourceSnapshot protocol where files will be written), a task-id (the RemoteSyncTask
+ identifier used to track progress updates), and a message (the commit message to use when writing
+ files to the source).
Returns the version written to the source.
Throws Exception if any entity in the stream is an Exception instance."
- [stream delete-prefixes snapshot task-id message]
+ [stream snapshot task-id message]
(let [opts (serdes/storage-base-context)
- ;; Bound the count of the items in the stream we don't accidentally realize the entire list into memory
stream-count (bounded-count 10000 stream)]
- (->> (concat stream delete-prefixes)
+ (->> stream
(map-indexed #(->file-spec task-id stream-count opts %1 %2))
(source.p/write-files! snapshot message))))
@@ -97,6 +91,7 @@
(git/git-source
(setting/get :remote-sync-url)
(or branch (setting/get :remote-sync-branch))
- (setting/get :remote-sync-token)))
+ (setting/get :remote-sync-token)
+ serialization/legal-top-level-paths))
([]
(source-from-settings (setting/get :remote-sync-branch))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index 842622c3b20..db71464ee93 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -231,21 +231,12 @@
(throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))
push-response))
-(defn- path-prefix
- "Extracts the unique collection identifier from a serialized file path.
-
- Takes a path string like \"collections/abc123_CollectionName/...\" and returns the collection prefix (e.g.,
- \"collections/abc123\") which remains stable even when collection names change.
-
- Returns the original path if no collection prefix is found."
+(defn- top-level-dir
+ "Extracts the first path segment from a file path.
+ E.g., \"databases/my_db/my_db.yaml\" => \"databases\""
[path]
- (let [matcher (re-matcher #"^(collections/[^/]{21})_[^/]+/" path)]
- (if (re-find matcher)
- (second (re-groups matcher))
- path)))
-
-(defn- matches-prefix [path prefixes]
- (some #(or (= % path) (str/starts-with? path %)) prefixes))
+ (when-let [idx (str/index-of path "/")]
+ (subs path 0 idx)))
(defn default-branch
"Retrieves the default branch name of the git repository.
@@ -267,24 +258,17 @@
(defn write-files!
"Writes multiple files to the git repository and commits the changes.
- Takes a snapshot map containing a :git Git instance and :version, a commit message string,
- and a sequence of file specs (paths should be relative to the repository root).
-
- Each file spec is a map with either:
- - :path and :content keys for writing/updating a file
- - :path and :remove? true for recursively removing all files at that path
-
- For writes within collection directories, ALL files in the same collection are replaced
- (using the collection entity_id prefix to identify the collection scope). This ensures
- that stale files don't remain when a collection's contents change.
+ Takes a snapshot map containing a :git Git instance, :version, and :managed-dirs,
+ a commit message string, and a sequence of file specs (maps with :path and :content keys,
+ paths should be relative to the repository root).
- For removals, all files matching the path as a prefix are deleted (allowing recursive
- directory deletion). Removal entries with empty paths are no-ops. Removing non-existent
- paths is also a no-op (idempotent).
+ All existing files within managed directories that are not in the write set are removed.
+ This ensures stale files (from renames, moves, or deletions) are cleaned up automatically.
+ Files outside managed directories are always preserved.
Returns the version written. Throws ExceptionInfo if the write or push
operation fails."
- [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message files]
+ [{:keys [^Git git ^String version ^String branch managed-dirs] :as snapshot} ^String message files]
(let [repo (.getRepository git)
branch-ref (qualify-branch branch)
parent-id (.resolve repo version)]
@@ -292,35 +276,25 @@
(with-open [inserter (.newObjectInserter repo)]
(let [index (DirCache/newInCore)
builder (.builder index)
- ;; Extract collection prefixes from written paths - all files in these
- ;; collections will be deleted and replaced with the new files
- write-prefixes (into #{}
- (comp
- (remove :remove?)
- (map :path)
- (remove str/blank?)
- (map path-prefix))
- files)
- ;; Collect removal paths/prefixes for explicit deletions
- removal-prefixes (into #{}
- (comp
- (filter :remove?)
- (map :path)
+ ;; Set of all paths being written in this commit
+ write-paths (into #{}
+ (comp (map :path)
(remove str/blank?))
- files)]
+ files)]
;; Add new/updated files to the index
- (doseq [{:keys [path content remove?]} files
- :when (and (not remove?) (not (str/blank? path)))]
+ (doseq [{:keys [path content]} files
+ :when (not (str/blank? path))]
(let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))
entry (doto (DirCacheEntry. ^String path)
(.setFileMode FileMode/REGULAR_FILE)
(.setObjectId blob-id))]
(.add builder entry)))
- ;; Copy existing tree entries, excluding:
- ;; 1. Files in collections being written to (using write-prefixes)
- ;; 2. Files matching explicit removal prefixes
+ ;; Copy existing tree entries that should be preserved:
+ ;; - Outside managed directories AND not being overwritten by the write set
+ ;; Files in managed dirs not in write-paths are dropped (stale file cleanup)
+ ;; Files in write-paths are skipped here (already added above with new content)
(when parent-id
(with-open [rev-walk (RevWalk. repo)
tree-walk (TreeWalk. repo)]
@@ -328,10 +302,9 @@
(.addTree tree-walk (.getTree commit))
(.setRecursive tree-walk true)
(while (.next tree-walk)
- (let [path (.getPathString tree-walk)
- existing-prefix (path-prefix path)]
- (when-not (or (contains? write-prefixes existing-prefix)
- (matches-prefix path removal-prefixes))
+ (let [path (.getPathString tree-walk)]
+ (when (and (not (contains? write-paths path))
+ (not (contains? managed-dirs (top-level-dir path))))
(let [entry (doto (DirCacheEntry. path)
(.setFileMode (.getFileMode tree-walk 0))
(.setObjectId (.getObjectId tree-walk 0)))]
@@ -425,7 +398,7 @@
(push-branch! (assoc source :branch branch-name))
branch-name))
-(defrecord GitSnapshot [git remote-url branch version token]
+(defrecord GitSnapshot [git remote-url branch version token managed-dirs]
source.p/SourceSnapshot
(list-files [this]
@@ -470,7 +443,7 @@
(fetch! source)
(let [version (commit-sha source (:branch source))]
(if version
- (->GitSnapshot (:git source) (:remote-url source) (:branch source) version (:token source))
+ (->GitSnapshot (:git source) (:remote-url source) (:branch source) version (:token source) (:managed-dirs source))
(throw (ex-info (str "Invalid branch: " (:branch source)) {})))))
(defn- snapshot
@@ -488,7 +461,7 @@
(snapshot* fresh-source)))
(throw e)))))
-(defrecord GitSource [git remote-url branch token]
+(defrecord GitSource [git remote-url branch token managed-dirs]
source.p/Source
(branches [source] (branches source))
@@ -504,10 +477,12 @@
(defn git-source
"Creates a new GitSource instance for a git repository.
- Takes a URL string (the git repository URL), a branch, and an
- optional token string (authentication token for private repositories).
+ Takes a URL string (the git repository URL), a branch, an
+ optional token string (authentication token for private repositories),
+ and a set of managed top-level directory names. Files in managed directories
+ are fully replaced during writes — any existing file not in the write set is removed.
Returns a GitSource record implementing the Source protocol."
- [url branch token]
+ [url branch token managed-dirs]
(->GitSource (get-jgit (repo-path {:remote-url url :token token}) {:remote-url url :token token})
- url branch token))
+ url branch token managed-dirs))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index aa42e1923d0..9de80c1daaa 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -51,11 +51,10 @@
"Writes multiple files to the source with a commit message.
Takes a SourceSnapshot instance implementing this protocol, a message (the commit message to use when writing files),
- and files (a sequence of file specs). Each file spec is a map with either:
- - :path and :content keys for writing/updating a file
- - :path and :remove? true for recursively removing all files at that path
+ and files (a sequence of file specs). Each file spec is a map with :path and :content keys.
- Removal entries with empty paths are no-ops. Removing non-existent paths is also a no-op (idempotent).
+ All existing files within managed directories that are not in the write set are removed.
+ Files outside managed directories are always preserved.
Returns the version of the written files.")
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 47bd61da729..87863feba0d 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -15,14 +15,12 @@
[clojure.string :as str]
[clojure.walk :as walk]
[metabase-enterprise.remote-sync.settings :as rs-settings]
- [metabase-enterprise.serialization.core :as serialization]
[metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.collections.core :as collections]
[metabase.collections.models.collection :as collection]
[metabase.models.serialization :as serdes]
[metabase.settings.core :as setting]
[metabase.util :as u]
- [metabase.util.log :as log]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -861,42 +859,6 @@
:by-path {}}
seen-paths))
-;;; --------------------------------------------- Export Path Construction ---------------------------------------------
-
-(defn- transform-entity-for-serdes
- "Transforms entity fields to serdes format for path generation.
- - Converts integer table_id to [db-name schema table-name] format
- - Converts integer collection_id to entity_id string for collection path lookup"
- [entity]
- (cond-> entity
- ;; Transform table_id for Segment (and other table-based entities)
- (and (:table_id entity) (integer? (:table_id entity)))
- (assoc :table_id (serdes/*export-table-fk* (:table_id entity)))
- ;; Transform collection_id for snippet and other collection-based entities
- (and (:collection_id entity) (integer? (:collection_id entity)))
- (assoc :collection_id (t2/select-one-fn :entity_id :model/Collection :id (:collection_id entity)))))
-
-(defn- entity->serdes-path
- "Builds the file path for an entity using serdes/storage-path.
- For Collections, returns the directory path (for recursive deletion).
- For other entities, returns the full file path.
- Returns the path as a string (without extension), or nil if path cannot be built."
- [model-type entity ctx]
- (try
- (when-let [serdes-meta (serdes/generate-path model-type entity)]
- (let [;; Transform entity fields to serdes format (e.g., table_id -> [db schema table])
- transformed-entity (transform-entity-for-serdes entity)
- entity-with-meta (assoc transformed-entity :serdes/meta serdes-meta)
- resolved (serialization/resolve-storage-path ctx entity-with-meta)]
- ;; For Collections, drop the last segment (filename) to get directory path
- (if (= model-type "Collection")
- (str/join "/" (butlast resolved))
- (str/join "/" resolved))))
- (catch Exception e
- (log/warnf "Failed to build storage path for %s %s: %s"
- model-type (:id entity) (ex-message e))
- nil)))
-
;;; -------------------------------------------- Event Helper Functions ------------------------------------------------
(defn determine-status
@@ -937,86 +899,6 @@
[:id :name :collection_id])
[:id :name :collection_id]))
-;;; -------------------------------------------- Removal Path Building --------------------------------------------------
-
-(defn- query-removed-ids
- "Queries RemoteSyncObject for model IDs marked for removal."
- [model-type statuses]
- (t2/select-fn-set :model_id :model/RemoteSyncObject
- :model_type model-type
- :status [:in (vec statuses)]))
-
-(defn- query-removed-entities
- "Queries entities marked for removal with all fields needed for serdes path generation.
- serdes/generate-path needs: entity_id, name (for label)
- serdes/storage-path needs: collection_id (for collection context), table_id (for segment paths)"
- [{:keys [model-type model-key removal]}]
- (let [{:keys [statuses]} removal
- removed-ids (query-removed-ids model-type statuses)]
- (when (seq removed-ids)
- ;; Select all columns since different models need different fields for serdes paths
- (t2/select model-key :id [:in removed-ids]))))
-
-(defn- setting-sentinel-delete?
- "Returns true if the setting's sentinel RSO exists with 'delete' status.
- Used to detect when a setting (like :remote-sync-transforms) has been disabled
- and all entities of the controlled types should be removed."
- [setting-key]
- (when (= setting-key :remote-sync-transforms)
- (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id rs-settings/transforms-root-id
- :status "delete")))
-
-(defn- build-bulk-removal-paths
- "Builds removal paths for ALL entities of a spec's model type.
- Used when the controlling setting is disabled (sentinel RSO has 'delete' status).
- Respects :removal-conditions (or :conditions) from the spec to exclude ineligible entities."
- [spec ctx]
- (let [{:keys [model-type model-key]} spec
- conditions (removal-conditions spec)]
- (for [entity (if conditions
- (apply t2/select model-key (into [] cat conditions))
- (t2/select model-key))
- :let [path (entity->serdes-path model-type entity ctx)]
- :when path]
- path)))
-
-(defn build-all-removal-paths
- "Builds full file paths for all entities marked for removal in RemoteSyncObject.
- Uses serdes/storage-path to generate paths that exactly match the file structure.
-
- Does bulk queries per model type for efficiency:
- 1. Query RemoteSyncObject for model_ids with removal statuses
- 2. Query actual entities by those IDs
- 3. Use serdes/storage-path to build the exact file path
-
- Also handles bulk removal for specs with :all-on-setting-disable when the
- controlling setting's sentinel RSO has 'delete' status.
-
- Returns paths without file extensions - the git source adds .yaml as needed."
- []
- (let [ctx (serdes/storage-base-context)
- ;; Standard removal paths from enabled specs
- standard-paths (into []
- (for [[_model-key spec] (enabled-specs)
- :when (spec-enabled? spec)
- :let [{:keys [statuses]} (:removal spec)
- model-type (:model-type spec)]
- :when (seq statuses)
- entity (query-removed-entities spec)
- :let [path (entity->serdes-path model-type entity ctx)]
- :when path]
- path))
- ;; Bulk removal paths for specs with :all-on-setting-disable
- bulk-paths (into []
- (for [[_model-key spec] remote-sync-specs
- :let [setting-key (get-in spec [:removal :all-on-setting-disable])]
- :when (and setting-key (setting-sentinel-delete? setting-key))
- path (build-bulk-removal-paths spec ctx)]
- path))]
- (into standard-paths bulk-paths)))
-
;;; ----------------------------------------- Sync Object Query Functions --------------------------------------------
(defmulti query-entities-for-sync
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
index 73449d05e0b..2ad83cdfe67 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
@@ -21,6 +21,7 @@
make-targets-of-type
extract]
[metabase-enterprise.serialization.v2.ingest
+ legal-top-level-paths
strip-labels
Ingestable
ingest-yaml
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
index 623db08741d..8c7c0c09b8d 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
@@ -406,15 +406,14 @@ is_sample: false
:model_collection_id coll-id
:status "delete"
:status_changed_at (t/offset-date-time)}]
- (let [removal-paths (spec/build-all-removal-paths)
- snippet-removal-path (first (filter #(str/includes? % "archived_snippet") removal-paths))
- _ (is (some? snippet-removal-path) "Should have a removal path for the archived snippet")
- initial-files {"main" {(str snippet-removal-path ".yaml")
+ (let [snippet-file-path (str "snippets/" snippet-eid "_archived_snippet.yaml")
+ initial-files {"main" {snippet-file-path
(test-helpers/generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
(is (= :success (:status result))
(str "Export should succeed. Result: " result))
+ ;; Archived snippet is not in the export stream, so its file gets cleaned from snippets/ (a managed dir)
(let [files-after-export (get @(:files-atom mock-source) "main")]
(is (not (some #(str/includes? % "archived_snippet") (keys files-after-export)))
"Archived snippet file should be deleted after export")))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index aafe703e784..486310ceeb4 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -80,6 +80,10 @@
remote))
+(def ^:private test-managed-dirs
+ "Managed dirs for test sources — matches v2.ingest/legal-top-level-paths."
+ #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})
+
(defn- ->source!
"Creates a (local) 'remote' repo and initializes a git source that uses it"
[branch {:keys [^Git git] :as _remote-repo}]
@@ -89,7 +93,7 @@
(.toURL)
(.toExternalForm))
local-repo (#'git/get-jgit (#'git/repo-path {:remote-url remote-url}) {:remote-url remote-url})]
- (git/->GitSource local-repo remote-url branch nil)))
+ (git/->GitSource local-repo remote-url branch nil test-managed-dirs)))
(defn- init-source!
[branch dir & config]
@@ -97,27 +101,6 @@
(let [remote-repo (apply init-remote! dir config)]
[(->source! branch remote-repo) remote-repo]))
-(deftest path-prefix
- (let [id (u/generate-nano-id "a")]
- (testing "Not in a collection"
- (doseq [path ["asdf"
- "asdf.txt"
- "dir/asdf.txt"
- "collections/asdf.txt"
- "collections/asdf/a.txt"
- "invalid/collections/asdf/a.txt"
- (str "collections/" id)
- (str "collections/" id "/but_no_name")]]
- (testing path
- (is (= path (#'git/path-prefix path)))
- (is (true? (#'git/matches-prefix path #{(str "collections/" (u/generate-nano-id)) path}))))))
- (testing "In a collection"
- (doseq [path [(str "collections/" id "_my_name/asdf")
- (str "collections/" id "_other_name/subdir/asdf.txt")]]
- (testing path
- (is (= (str "collections/" id) (#'git/path-prefix path)))
- (is (true? (#'git/matches-prefix path #{(str "collections/" id) (str "collections/" (u/generate-nano-id))}))))))))
-
(deftest qualify-branch-test
(is (= "refs/heads/main" (#'git/qualify-branch "main")))
(is (= "refs/heads/main" (#'git/qualify-branch "refs/heads/main"))))
@@ -180,7 +163,6 @@
(deftest write-files
(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")
- otherdir-path (str "collections/" "o" (subs (u/generate-nano-id "b") 1) "_otherdir/")
thirddir-path (str "collections/" "s" (subs (u/generate-nano-id "c") 1) "_thirddir/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
@@ -188,12 +170,10 @@
"master2.txt" "File 2 in master"
(str subdir-path "path.txt") "File in subdir"
(str subdir-path "path2.txt") "File 2 in subdir"
- (str otherdir-path "path.txt") "File in otherdir"
- (str otherdir-path "path2.txt") "File 2 in otherdir"
(str thirddir-path "path.txt") "File in third dir"
(str thirddir-path "path2.txt") "File 2 in third dir"}
:branches ["branch-1" "branch-2"])]
- (testing "Files in a subdir are replaced, other subdirs and root are unchanged"
+ (testing "All files in managed dirs not in write set are removed; root files outside managed dirs are preserved"
(source.p/write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}
{:path (str subdir-path "path.txt") :content "Updated subdir content"}
{:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}
@@ -201,9 +181,8 @@
{:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])
(is (= ["Update 1" "Initial commit"] (map :message (git/log master))))
(let [master-snap (source.p/snapshot master)]
- (is (= [(str otherdir-path "path.txt")
- (str otherdir-path "path2.txt")
- (str subdir-path "path.txt")
+ ;; otherdir files are removed because collections/ is a managed dir and those files weren't in the write set
+ (is (= [(str subdir-path "path.txt")
(str subdir-path "path3.txt")
(str thirddir-path "path.txt")
(str thirddir-path "path3.txt")
@@ -213,15 +192,12 @@
(is (= "Updated master content" (source.p/read-file master-snap "master.txt")))
(is (= "File 2 in master" (source.p/read-file master-snap "master2.txt")))
- (is (= "File 2 in otherdir" (source.p/read-file master-snap (str otherdir-path "path2.txt"))))
(is (= "Updated subdir content" (source.p/read-file master-snap (str subdir-path "path.txt"))))
(is (= "Updated subdir content 3" (source.p/read-file master-snap (str subdir-path "path3.txt")))))
(testing "Check remote repo directly"
(is (= "Updated master content" (git/read-file (assoc remote :version "master") "master.txt")))
- (is (= [(str otherdir-path "path.txt")
- (str otherdir-path "path2.txt")
- (str subdir-path "path.txt")
+ (is (= [(str subdir-path "path.txt")
(str subdir-path "path3.txt")
(str thirddir-path "path.txt")
(str thirddir-path "path3.txt")
@@ -230,13 +206,9 @@
(git/list-files (assoc remote :version "master"))))
(is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))))
- (testing "If no root files are touched, they all stay as-is"
+ (testing "Writing only to collections/ removes all other collection files"
(source.p/write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])
- (is (= [(str otherdir-path "path.txt")
- (str otherdir-path "path2.txt")
- (str subdir-path "path.txt")
- (str subdir-path "path3.txt")
- (str thirddir-path "path.txt")
+ (is (= [(str thirddir-path "path.txt")
"master.txt"
"master2.txt"]
(git/list-files (assoc remote :version "master")))))))))
@@ -249,58 +221,57 @@
(str subdir-path "path.txt") "File in subdir"})]))))
(deftest concurrent-access
- (let [subdir-path (str "collections/" (u/generate-nano-id "a") "_subdir")]
- (mt/with-temp-dir [remote-dir nil]
- (let [[master remote] (init-source! "master" remote-dir
- :files {"master.txt" "File in master"
- (str subdir-path "path.txt") "File in subdir"}
- :branches ["branch-1" "branch-2"])
- new-branch (->source! "new-branch" remote)]
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master remote] (init-source! "master" remote-dir
+ :files {"master.txt" "File in master"
+ "subdir/path.txt" "File in subdir"}
+ :branches ["branch-1" "branch-2"])
+ new-branch (->source! "new-branch" remote)]
- (testing "Initial clone is the same"
- (is (= ["Initial commit"] (map :message (git/log master))))
- (is (= ["Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
+ (testing "Initial clone is the same"
+ (is (= ["Initial commit"] (map :message (git/log master))))
+ (is (= ["Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
- ;; Add an extra commit to remote
- (git-working-add! remote "additional-file.txt" "Additional file content")
- (git-working-commit! remote "Added additional file")
+ ;; Add an extra commit to remote
+ (git-working-add! remote "additional-file.txt" "Additional file content")
+ (git-working-commit! remote "Added additional file")
- (testing "Source is behind remote"
- (is (= ["Initial commit"] (map :message (git/log master))))
- (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master"))))))
+ (testing "Source is behind remote"
+ (is (= ["Initial commit"] (map :message (git/log master))))
+ (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master"))))))
- (testing "After fetch, source is up to date"
- (git/fetch! master)
- (is (= ["Added additional file" "Initial commit"] (map :message (git/log master)))))
+ (testing "After fetch, source is up to date"
+ (git/fetch! master)
+ (is (= ["Added additional file" "Initial commit"] (map :message (git/log master)))))
- (testing "Writing a file to source and pushing back to remote when there is new content on remote"
- ;; Make source be behind again
- (git-working-add! remote "only-on-remote.txt" "Initially on remote")
- (git-working-commit! remote "Only on remote")
+ (testing "Writing a file to source and pushing back to remote when there is new content on remote"
+ ;; Make source be behind again
+ (git-working-add! remote "only-on-remote.txt" "Initially on remote")
+ (git-working-commit! remote "Only on remote")
- (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])
+ (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])
- (testing "Remote has the new commit with just the files committed, but only version is in history"
- (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
- (is (= ["additional-file.txt" (str subdir-path "path.txt") "initially-source.txt" "master.txt" "only-on-remote.txt"] (git/list-files (assoc remote :version "master"))))
- (is (= "Initially on source" (git/read-file (assoc remote :version "master") "initially-source.txt"))))
+ (testing "Remote has the new commit with just the files committed, but only version is in history"
+ (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
+ (is (= ["additional-file.txt" "initially-source.txt" "master.txt" "only-on-remote.txt" "subdir/path.txt"] (git/list-files (assoc remote :version "master"))))
+ (is (= "Initially on source" (git/read-file (assoc remote :version "master") "initially-source.txt"))))
- (testing "Source has the same history"
- (is (= (map :message (git/log (assoc remote :branch "master"))) (map :message (git/log master))))))
+ (testing "Source has the same history"
+ (is (= (map :message (git/log (assoc remote :branch "master"))) (map :message (git/log master))))))
- (testing "Writing to a branch local has not seen (but remote has) adds it to the history on remote"
- (git-working-checkout! remote "new-branch" true)
- (git-working-add! remote "new-branch-file.txt" "Initially on remote")
- (git-working-add! remote "new-branch-remote.txt" "Initially on remote")
- (git-working-commit! remote "New-branch on remote")
+ (testing "Writing to a branch local has not seen (but remote has) adds it to the history on remote"
+ (git-working-checkout! remote "new-branch" true)
+ (git-working-add! remote "new-branch-file.txt" "Initially on remote")
+ (git-working-add! remote "new-branch-remote.txt" "Initially on remote")
+ (git-working-commit! remote "New-branch on remote")
- (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))
- (is (nil? (git/log new-branch)))
+ (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))
+ (is (nil? (git/log new-branch)))
- (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}
- {:path "new-branch-file.txt" :content "Updated on source"}])
+ (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}
+ {:path "new-branch-file.txt" :content "Updated on source"}])
- (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))))))))
+ (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch"))))))))))
(deftest git-source-using-commit-ref
(mt/with-temp-dir [remote-dir nil]
@@ -358,67 +329,64 @@
(let [[master _remote] (init-source! "master" remote-dir)]
(is (= "master" (git/default-branch master))))))
-(deftest write-files-removal-test
- (let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")
- otherdir-path (str "collections/" "o" (subs (u/generate-nano-id "b") 1) "_otherdir/")]
+(deftest write-files-top-level-exports-replaced-test
+ (let [old-col-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_mycol/")
+ new-col-path (str "collections/" "s" (subs (u/generate-nano-id "b") 1) "_othercol/")
+ kept-col-path (str "collections/" "t" (subs (u/generate-nano-id "c") 1) "_keptcol/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
- :files {"master.txt" "File in master"
- (str subdir-path "file1.yaml") "File 1 in subdir"
- (str subdir-path "file2.yaml") "File 2 in subdir"
- (str otherdir-path "file1.yaml") "File 1 in otherdir"
- (str otherdir-path "file2.yaml") "File 2 in otherdir"})]
- (testing "Removal entry deletes all files under that path recursively"
- (source.p/write-files! (source.p/snapshot master) "Remove subdir"
- [{:path (subs subdir-path 0 (dec (count subdir-path))) :remove? true}])
+ :files {"databases/old_db/old_db.yaml" "Old database"
+ "databases/old_db/schemas/public.yaml" "Old schema"
+ "snippets/old_snippet.yaml" "Old snippet"
+ "unmanaged/keep_me.txt" "Unmanaged file"
+ (str old-col-path "cards/card1.yaml") "Card in old col"
+ (str old-col-path "cards/card2.yaml") "Card 2 in old col"
+ (str kept-col-path "dashboards/dash1.yaml") "Dashboard in kept col"})]
+ (testing "Writing to a managed dir removes all stale files in ALL managed dirs"
+ (source.p/write-files! (source.p/snapshot master) "Rename database"
+ [{:path "databases/new_db/new_db.yaml" :content "Renamed database"}
+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}
+ {:path (str old-col-path "cards/card1.yaml") :content "Card in old col"}
+ {:path (str old-col-path "cards/card2.yaml") :content "Card 2 in old col"}
+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard in kept col"}
+ {:path "snippets/old_snippet.yaml" :content "Old snippet"}])
(let [files (set (source.p/list-files (source.p/snapshot master)))]
- (is (contains? files "master.txt") "Root files should remain")
- (is (contains? files (str otherdir-path "file1.yaml")) "Other collection files should remain")
- (is (contains? files (str otherdir-path "file2.yaml")) "Other collection files should remain")
- (is (not (contains? files (str subdir-path "file1.yaml"))) "Subdir files should be removed")
- (is (not (contains? files (str subdir-path "file2.yaml"))) "Subdir files should be removed")))))))
-
-(deftest write-files-mixed-write-and-removal-test
- (let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")
- newdir-path (str "collections/" "n" (subs (u/generate-nano-id "b") 1) "_newdir/")]
+ (is (contains? files "databases/new_db/new_db.yaml") "New database file should exist")
+ (is (contains? files "databases/new_db/schemas/public.yaml") "New schema file should exist")
+ (is (not (contains? files "databases/old_db/old_db.yaml")) "Old database file should be removed")
+ (is (not (contains? files "databases/old_db/schemas/public.yaml")) "Old schema file should be removed")
+ (is (contains? files (str old-col-path "cards/card1.yaml")) "Written collection files should remain")
+ (is (contains? files "snippets/old_snippet.yaml") "Written snippet file should remain")
+ (is (contains? files "unmanaged/keep_me.txt") "Unmanaged files should be untouched")))
+
+ (testing "Entity moved between collections removes files from old collection"
+ (source.p/write-files! (source.p/snapshot master) "Move card to new collection"
+ [{:path (str new-col-path "cards/card1.yaml") :content "Card moved to new col"}
+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard still here"}
+ {:path "databases/new_db/new_db.yaml" :content "Renamed database"}
+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}])
+ (let [files (set (source.p/list-files (source.p/snapshot master)))]
+ (is (contains? files (str new-col-path "cards/card1.yaml")) "Moved card should exist in new collection")
+ (is (contains? files (str kept-col-path "dashboards/dash1.yaml")) "Kept collection files should remain")
+ (is (not (contains? files (str old-col-path "cards/card1.yaml"))) "Old collection card should be removed")
+ (is (not (contains? files (str old-col-path "cards/card2.yaml"))) "Other files in old collection should also be removed")
+ (is (not (contains? files "snippets/old_snippet.yaml")) "Snippets cleaned up when not in write set")
+ (is (contains? files "unmanaged/keep_me.txt") "Unmanaged files still untouched")))))))
+
+(deftest write-files-entity-rename-within-collection-test
+ (let [col-path (str "collections/" "u" (subs (u/generate-nano-id "d") 1) "_col/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
- :files {"master.txt" "File in master"
- (str subdir-path "old-file.yaml") "Old file in subdir"})]
- (testing "Can combine write and removal entries in same call"
- (source.p/write-files! (source.p/snapshot master) "Mixed operations"
- [{:path (subs subdir-path 0 (dec (count subdir-path))) :remove? true}
- {:path (str newdir-path "new-file.yaml") :content "New file content"}])
- (let [snap (source.p/snapshot master)
- files (set (source.p/list-files snap))]
- (is (contains? files "master.txt") "Root files should remain")
- (is (contains? files (str newdir-path "new-file.yaml")) "New files should be added")
- (is (not (contains? files (str subdir-path "old-file.yaml"))) "Old files should be removed")
- (is (= "New file content" (source.p/read-file snap (str newdir-path "new-file.yaml"))))))))))
-
-(deftest write-files-empty-removal-path-test
- (mt/with-temp-dir [remote-dir nil]
- (let [[master _remote] (init-source! "master" remote-dir
- :files {"master.txt" "File in master"
- "subdir/file.yaml" "File in subdir"})]
- (testing "Empty removal paths are ignored (no-op)"
- (source.p/write-files! (source.p/snapshot master) "Empty removal"
- [{:path "" :remove? true}
- {:path " " :remove? true}])
- (let [files (set (source.p/list-files (source.p/snapshot master)))]
- (is (= #{"master.txt" "subdir/file.yaml"} files)
- "All files should remain when removal path is empty"))))))
-
-(deftest write-files-nonexistent-removal-path-test
- (mt/with-temp-dir [remote-dir nil]
- (let [[master _remote] (init-source! "master" remote-dir
- :files {"master.txt" "File in master"})]
- (testing "Removing non-existent path is a no-op"
- (source.p/write-files! (source.p/snapshot master) "Remove nonexistent"
- [{:path "collections/nonexistent" :remove? true}])
- (let [files (set (source.p/list-files (source.p/snapshot master)))]
- (is (= #{"master.txt"} files)
- "Files should remain unchanged"))))))
+ :files {(str col-path "cards/eid123_OldCardName.yaml") "Card with old name"
+ (str col-path "cards/eid456_OtherCard.yaml") "Other card"})]
+ (testing "Entity renamed within a collection removes the old-named file"
+ (source.p/write-files! (source.p/snapshot master) "Rename card"
+ [{:path (str col-path "cards/eid123_NewCardName.yaml") :content "Card with new name"}
+ {:path (str col-path "cards/eid456_OtherCard.yaml") :content "Other card"}])
+ (let [files (set (source.p/list-files (source.p/snapshot master)))]
+ (is (contains? files (str col-path "cards/eid123_NewCardName.yaml")) "Renamed card should exist")
+ (is (contains? files (str col-path "cards/eid456_OtherCard.yaml")) "Other card should still exist")
+ (is (not (contains? files (str col-path "cards/eid123_OldCardName.yaml"))) "Old card name file should be removed")))))))
(deftest ensure-origin-configured-sets-origin-after-clone-test
(mt/with-temp-dir [remote-dir nil]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
index 89446734b9a..7bc3bea0a34 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
@@ -157,26 +157,3 @@
(is (= []
@written-files)
"Should write no files when none match"))))
-
-(deftest wrapping-source-write-files-removal-entries-test
- (testing "WrappingSource filters removal entries based on path-filters"
- (let [written-files (atom [])
- mock-snap (reify source.p/SourceSnapshot
- (list-files [_]
- [])
- (read-file [_ _path]
- nil)
- (write-files! [_ _message files]
- (reset! written-files (vec files))
- nil)
- (version [_]
- "mock-version"))
- wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*"])
- files-to-write [{:path "collections/foo.yaml" :content "foo-content"}
- {:path "collections/abc123" :remove? true}
- {:path "databases/db1" :remove? true}]]
- (source.p/write-files! wrapped-snap "test commit" files-to-write)
- (is (= [{:path "collections/foo.yaml" :content "foo-content"}
- {:path "collections/abc123" :remove? true}]
- @written-files)
- "Should pass through removal entries that match filter and filter out those that don't"))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index 21e452f8c5d..0f77acc5cf4 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -63,7 +63,7 @@
test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")
(create-test-entity "test-id-2" "entity-two" "Card")]]
- (is (= "mock-written-version" (source/store! test-entities [] (source.p/snapshot mock-source) task-id "Test commit message")))
+ (is (= "mock-written-version" (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message")))
(testing "write-files! was called with correct message"
(is (= "Test commit message" (:message @written-files))))
@@ -108,7 +108,7 @@
(let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)]
(is (nil? (:progress initial-task)) "Progress should be nil initially"))
- (source/store! test-entities [] (source.p/snapshot mock-source) task-id "Test commit")
+ (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit")
(let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)]
(is (some? (:progress final-task)) "Progress should be updated after store!")
@@ -128,7 +128,7 @@
(let [written-files (atom nil)
mock-source (->MockSource written-files)]
- (source/store! [] [] (source.p/snapshot mock-source) task-id "Empty commit")
+ (source/store! [] (source.p/snapshot mock-source) task-id "Empty commit")
(testing "write-files! was called even with empty stream"
(is (some? @written-files)))
@@ -159,8 +159,8 @@
:dataset_query {:z_field 1 :a_field 2 :database 3}
:visualization_settings {:zebra true :apple false}}]
;; Store the same entity twice
- (source/store! [test-entity] [] (source.p/snapshot mock-source-1) task-id "First commit")
- (source/store! [test-entity] [] (source.p/snapshot mock-source-2) task-id "Second commit")
+ (source/store! [test-entity] (source.p/snapshot mock-source-1) task-id "First commit")
+ (source/store! [test-entity] (source.p/snapshot mock-source-2) task-id "Second commit")
(testing "YAML content is identical between runs"
(let [content-1 (-> @written-files-1 :files first :content)
content-2 (-> @written-files-2 :files first :content)]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index d2e7b1d550f..a98a02b1b43 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -245,12 +245,6 @@
(let [spec (spec/spec-for-model-key :model/Card)]
(is (nil? (spec/build-sync-object-fields spec nil))))))
-;;; ------------------------------------------- Removal Path Building Tests ------------------------------------------
-
-;; Note: build-all-removal-paths uses serdes/storage-path to generate paths.
-;; The actual paths are tested in impl_test.clj integration tests which verify
-;; the full export/import cycle with real database entities.
-
;;; --------------------------------------------- Fields for Sync Tests -----------------------------------------------
(deftest fields-for-sync-test
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 445e60ff65a..590ae6a0928 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -163,7 +163,13 @@ width: fixed
"
name entity-id collection-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_") dashcards-yaml)))
-(defrecord MockSourceSnapshot [source-id base-url branch fail-mode files-atom]
+(defn- top-level-dir
+ "Extracts the first path segment from a file path."
+ [path]
+ (when-let [idx (str/index-of path "/")]
+ (subs path 0 idx)))
+
+(defrecord MockSourceSnapshot [source-id base-url branch fail-mode files-atom managed-dirs]
source.p/SourceSnapshot
(list-files [_this]
(case fail-mode
@@ -190,28 +196,25 @@ width: fixed
:write-files-error (throw (Exception. "Failed to write files"))
:store-error (throw (Exception. "Store failed"))
:network-error (throw (java.net.UnknownHostException. "Remote host not found"))
- ;; Default success case - handle both writes and removals
- (let [write-entries (remove #(or (:remove? %) (str/blank? (:path %))) files)
- removal-prefixes (into #{} (comp (filter :remove?)
- (map :path)
- (remove str/blank?))
- files)
+ ;; Default success case - remove all files in managed dirs, then add new files
+ (let [write-entries (remove #(str/blank? (:path %)) files)
+ write-paths (into #{} (map :path) write-entries)
current-files (get @files-atom branch {})
- ;; Remove files matching removal prefixes
- after-removals (into {}
- (remove (fn [[path _]]
- (some #(or (= path %) (str/starts-with? path %))
- removal-prefixes))
- current-files))
+ ;; Keep files outside managed dirs or in the write set
+ kept-files (into {}
+ (filter (fn [[path _]]
+ (or (not (contains? managed-dirs (top-level-dir path)))
+ (contains? write-paths path))))
+ current-files)
;; Add new files
- final-files (into after-removals (map (juxt :path :content) write-entries))]
+ final-files (into kept-files (map (juxt :path :content) write-entries))]
(swap! files-atom assoc branch final-files)))
"write-files-version")
(version [_this]
"mock-version"))
-(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom]
+(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom managed-dirs]
source.p/Source
(create-branch [_this branch _base]
(swap! branches-atom conj [branch (str branch "-ref")]))
@@ -228,14 +231,19 @@ width: fixed
"main")
(snapshot [_this]
- (->MockSourceSnapshot source-id base-url branch fail-mode files-atom)))
+ (->MockSourceSnapshot source-id base-url branch fail-mode files-atom managed-dirs)))
+
+(def ^:private default-managed-dirs
+ "Default managed dirs for mock sources, matching v2.ingest/legal-top-level-paths."
+ #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})
(defn create-mock-source
- "Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, and `:initial-files`."
- [& {:keys [branch fail-mode initial-files]
+ "Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, `:initial-files`, and `:managed-dirs`."
+ [& {:keys [branch fail-mode initial-files managed-dirs]
:or {branch "main"
fail-mode nil
- initial-files nil}}]
+ initial-files nil
+ managed-dirs default-managed-dirs}}]
(let [default-files {"main" {"collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml"
(generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")
@@ -254,7 +262,7 @@ width: fixed
files-atom (atom (or initial-files default-files))
branches-atom (atom #{["main" "main-ref"] ["develop" "develop-ref"]})]
- (->MockSource "test-source" "https://test.example.com" branch fail-mode files-atom branches-atom)))
+ (->MockSource "test-source" "https://test.example.com" branch fail-mode files-atom branches-atom managed-dirs)))
(defn clean-object
"Test fixture that resets the RemoteSyncObject table before running tests to prevent existing
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj
index 00cddf64e33..6a279468330 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj
@@ -5,64 +5,44 @@
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as th]))
-(deftest mock-source-write-files-removal-test
- (testing "MockSource handles removal entries correctly"
+(deftest mock-source-write-files-managed-dir-cleanup-test
+ (testing "MockSource removes files in managed dirs not in write set"
(let [source (th/create-mock-source
:initial-files {"main" {"collections/abc/file1.yaml" "content1"
"collections/abc/file2.yaml" "content2"
"collections/def/file3.yaml" "content3"
- "other/file4.yaml" "content4"}})
+ "other/file4.yaml" "content4"}}
+ :managed-dirs #{"collections"})
snapshot (source.p/snapshot source)]
- (source.p/write-files! snapshot "Remove abc collection"
- [{:path "collections/abc" :remove? true}])
- (is (= #{"collections/def/file3.yaml" "other/file4.yaml"}
+ (source.p/write-files! snapshot "Write only abc"
+ [{:path "collections/abc/file1.yaml" :content "new-content1"}])
+ (is (= #{"collections/abc/file1.yaml" "other/file4.yaml"}
(set (source.p/list-files snapshot)))
- "Should remove all files under collections/abc recursively"))))
+ "Only written files in managed dirs should remain; unmanaged dirs untouched"))))
-(deftest mock-source-write-files-removal-exact-path-test
- (testing "MockSource removal matches exact path"
+(deftest mock-source-write-files-unmanaged-preserved-test
+ (testing "MockSource preserves files in unmanaged directories"
(let [source (th/create-mock-source
- :initial-files {"main" {"collections/abc.yaml" "content1"
- "collections/abcdef/file.yaml" "content2"}})
- snapshot (source.p/snapshot source)]
- (source.p/write-files! snapshot "Remove abc.yaml"
- [{:path "collections/abc.yaml" :remove? true}])
- (is (= #{"collections/abcdef/file.yaml"}
- (set (source.p/list-files snapshot)))
- "Should only remove exact path match, not prefix match without slash"))))
-
-(deftest mock-source-write-files-mixed-write-and-removal-test
- (testing "MockSource handles both write and removal entries"
- (let [source (th/create-mock-source
- :initial-files {"main" {"collections/old/file1.yaml" "old-content"
- "collections/keep/file2.yaml" "keep-content"}})
- snapshot (source.p/snapshot source)]
- (source.p/write-files! snapshot "Mixed operations"
- [{:path "collections/old" :remove? true}
- {:path "collections/new/file3.yaml" :content "new-content"}])
- (is (= #{"collections/keep/file2.yaml" "collections/new/file3.yaml"}
- (set (source.p/list-files snapshot)))
- "Should remove old files and add new files"))))
-
-(deftest mock-source-write-files-empty-removal-path-test
- (testing "MockSource ignores removal entries with empty paths"
- (let [source (th/create-mock-source
- :initial-files {"main" {"collections/abc/file1.yaml" "content1"}})
+ :initial-files {"main" {"collections/abc/file1.yaml" "content1"
+ "unmanaged/file2.yaml" "content2"}}
+ :managed-dirs #{"collections"})
snapshot (source.p/snapshot source)]
- (source.p/write-files! snapshot "Empty removal"
- [{:path "" :remove? true}
- {:path " " :remove? true}])
- (is (= #{"collections/abc/file1.yaml"}
+ (source.p/write-files! snapshot "Write collections"
+ [{:path "collections/abc/file1.yaml" :content "new-content"}])
+ (is (= #{"collections/abc/file1.yaml" "unmanaged/file2.yaml"}
(set (source.p/list-files snapshot)))
- "Should not remove anything when removal path is empty or blank"))))
+ "Unmanaged directory files should be preserved"))))
-(deftest mock-source-write-files-nonexistent-removal-path-test
- (testing "MockSource handles removal of non-existent paths (no-op)"
+(deftest mock-source-write-files-empty-managed-dir-cleanup-test
+ (testing "MockSource cleans managed dir even when no files written to it"
(let [source (th/create-mock-source
- :initial-files {"main" {"collections/abc/file1.yaml" "content1"}})
+ :initial-files {"main" {"collections/abc/file1.yaml" "content1"
+ "snippets/old.yaml" "old-snippet"}}
+ :managed-dirs #{"collections" "snippets"})
snapshot (source.p/snapshot source)]
- (source.p/write-files! snapshot "Remove nonexistent"
- [{:path "collections/xyz" :remove? true}])
+ ;; Write only to collections, nothing to snippets
+ (source.p/write-files! snapshot "Write only collections"
+ [{:path "collections/abc/file1.yaml" :content "new-content"}])
(is (= #{"collections/abc/file1.yaml"}
(set (source.p/list-files snapshot)))
- "Should be a no-op when removing non-existent path"))))
+ "Snippets dir should be cleaned even though no snippet files were written"))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index 29b8a0b88ca..695e60fd771 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -281,6 +281,30 @@
(is (not (some #(str/includes? % "collections/transforms/") (keys files-after-export)))
"Export should NOT include transform files when setting is disabled"))))))))))
+(deftest export-removes-existing-transform-files-when-setting-disabled-test
+ (testing "Export removes pre-existing transform files from remote when setting is disabled"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms false
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection {rs-coll-id :id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxxxxx" :location "/"}
+ :model/RemoteSyncObject _rso {:model_type "Collection" :model_id rs-coll-id :model_name "Remote Synced" :status "create" :status_changed_at (t/offset-date-time)}]
+ ;; Start with transform files already on the remote (from a previous export when setting was enabled)
+ (let [initial-files {"main" {"transforms/my_transform/my_transform.yaml" "old transform content"
+ "transforms/my_transform/steps/step1.yaml" "old step content"
+ "python-libraries/custom_lib.yaml" "old library content"}}
+ mock-source (test-helpers/create-mock-source :initial-files initial-files)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
+ (is (= :success (:status result))
+ (str "Export should succeed. Result: " result))
+ (let [files-after-export (get @(:files-atom mock-source) "main")]
+ (is (not (some #(str/starts-with? % "transforms/") (keys files-after-export)))
+ "Transform files should be removed from remote when setting is disabled")
+ (is (not (some #(str/starts-with? % "python-libraries/") (keys files-after-export)))
+ "Python library files should be removed from remote when setting is disabled"))))))))))
+
(defn- generate-transforms-namespace-collection-yaml
"Generates YAML content for a transforms-namespace collection."
[entity-id name]
@@ -848,66 +872,6 @@ serdes/meta:
(is (not (t2/exists? :model/Transform :id local-transform-id))
"Local transform should be removed because it's not on the remote")))))))))
-;;; ------------------------------------------- build-all-removal-paths Tests -------------------------------------------
-
-(deftest build-all-removal-paths-includes-all-transforms-on-setting-disable-test
- (testing "build-all-removal-paths returns paths for all transforms content when sentinel RSO has 'delete' status"
- (mt/with-premium-features #{:transforms-basic}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
- :namespace collection/transforms-ns
- :location "/"}
- :model/Transform {transform-id :id} {:name "Test Transform"
- :collection_id coll-id}
- :model/TransformTag {tag-id :id} {:name "Test Tag"
- :built_in_type nil}]
- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
- (is (t2/exists? :model/Transform :id transform-id))
- (is (t2/exists? :model/TransformTag :id tag-id))
- (is (t2/exists? :model/PythonLibrary :id (:id library)))
- (let [paths-before (spec/build-all-removal-paths)]
- (is (not (some #(str/includes? % "test_transform") paths-before))
- "Transform should not be in removal paths before setting is disabled"))
- (settings/sync-transform-tracking! true)
- (settings/sync-transform-tracking! false)
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id
- :status "delete")
- "Sentinel RSO should exist with 'delete' status")
- (let [paths-after (spec/build-all-removal-paths)]
- (is (some #(str/includes? % "test_transform") paths-after)
- "Transform should be in removal paths after setting is disabled")
- (is (some #(str/includes? % "test_tag") paths-after)
- "TransformTag should be in removal paths after setting is disabled")
- (is (some #(str/includes? % "common.py") paths-after)
- "PythonLibrary should be in removal paths after setting is disabled"))))))))
-
-(deftest build-all-removal-paths-excludes-builtin-transform-tags-test
- (testing "build-all-removal-paths respects :conditions and excludes built-in tags"
- (mt/with-premium-features #{:transforms-basic}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-temp [:model/TransformTag {custom-tag-id :id} {:name "Custom Tag"
- :built_in_type nil}
- :model/TransformTag {builtin-tag-id :id} {:name "Built-in Tag"
- :built_in_type "target"}]
- (is (t2/exists? :model/TransformTag :id custom-tag-id))
- (is (t2/exists? :model/TransformTag :id builtin-tag-id))
- (settings/sync-transform-tracking! true)
- (settings/sync-transform-tracking! false)
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id
- :status "delete")
- "Sentinel RSO should exist with 'delete' status")
- (let [paths (spec/build-all-removal-paths)]
- (is (some #(str/includes? % "custom_tag") paths)
- "Custom tag (built_in_type nil) should be in removal paths")
- (is (not (some #(str/includes? % "built_in_tag") paths))
- "Built-in tag should NOT be in removal paths")))))))
-
(deftest export-excludes-builtin-transform-tags-test
(testing "Export excludes built-in transform tags based on :conditions"
(mt/with-premium-features #{:transforms-basic}
@@ -944,21 +908,3 @@ serdes/meta:
"Built-in PythonLibrary should be in export roots")
(is (contains? exported-ids (:id custom-lib))
"Custom PythonLibrary should be in export roots")))))))
-
-(deftest build-all-removal-paths-excludes-builtin-python-library-test
- (testing "build-all-removal-paths excludes built-in PythonLibrary via :removal-conditions"
- (mt/with-premium-features #{:transforms}
- (mt/with-temporary-setting-values [remote-sync-transforms true
- remote-sync-enabled true]
- (mt/with-temp [:model/PythonLibrary _ {:path "common.py"
- :source "# builtin"
- :entity_id transforms-python/builtin-entity-id}
- :model/PythonLibrary _ {:path "custom.py"
- :source "# custom"}]
- (settings/sync-transform-tracking! true)
- (settings/sync-transform-tracking! false)
- (let [paths (spec/build-all-removal-paths)]
- (is (some #(str/includes? % "custom.py") paths)
- "Custom PythonLibrary should be in removal paths")
- (is (not (some #(str/includes? % "common.py") paths))
- "Built-in PythonLibrary should NOT be in removal paths")))))))