The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix git sync to remove stale YAML files from managed export directories (#71739)
Edward Paget9e576f007b56
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex f3f3e1a1c3f..26f0da00493 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -242,9 +242,7 @@(let [snapshot-version (source.p/version snapshot)last-imported-version (remote-sync.task/last-version)first-import? (nil? last-imported-version)- path-filters [#"collections/.*" #"databases/.*" #"actions/.*"- ;; python-libraries is the old name for python_libraries, support both for backwards compatibility- #"transforms/.*" #"python[_-]libraries/.*" #"snippets/.*"]+ path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})has-transforms? (snapshot-has-transforms? base-ingestable){:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)@@ -315,8 +313,7 @@(if-let [models (spec/extract-entities-for-export)](do(remote-sync.task/update-progress! task-id 0.3)- (let [all-delete-prefixes (spec/build-all-removal-paths)- written-version (source/store! models all-delete-prefixes snapshot task-id message)]+ (let [written-version (source/store! models snapshot task-id message)](remote-sync.task/set-version! task-id written-version))(t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp}){:status :successdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex e5fe3e1dc14..873b0dac70b 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -181,7 +181,7 @@(throw (ex-info "Invalid repository URL: only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)"{:url remote-sync-url})))- (let [source (git/git-source remote-sync-url "HEAD" remote-sync-token)]+ (let [source (git/git-source remote-sync-url "HEAD" remote-sync-token nil)](when (and (= :read-only remote-sync-type) (not (str/blank? remote-sync-branch)) (not (some #{remote-sync-branch} (git/branches source))))(throw (ex-info "Invalid branch name" {:url remote-sync-url :branch remote-sync-branch}))))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex db71503007a..1602ae72bf0 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -54,36 +54,30 @@(str/join File/separator (concat dirnames [basename]))))(defn- ->file-spec- "Converts entity from serdes stream into file spec for source write-files! "+ "Converts entity from serdes stream into file spec for source write-files!"[task-id count opts idx entity](when (instance? Exception entity)- ;; Just short-circuit if there are errors.(throw entity))- (u/prog1 (if (string? entity)- ;; when an entity is a string that means it's a path to delete- {:path entity- :remove? true}- {:path (remote-sync-path opts entity)- :content (yaml/generate-string (serialization/serialization-deep-sort entity)- {:dumper-options {:flow-style :block :split-lines false}})})+ (u/prog1 {:path (remote-sync-path opts entity)+ :content (yaml/generate-string (serialization/serialization-deep-sort entity)+ {:dumper-options {:flow-style :block :split-lines false}})}(remote-sync.task/update-progress! task-id (-> (inc idx) (/ count) (* 0.65) (+ 0.3)))))(defn store!"Stores serialized entities from a stream to a remote source and commits the changes.- Takes a stream (a sequence of serialized entities to be stored), a list of prefixes to recursively-delete at,- a snapshot (the remote source implementing the SourceSnapshot protocol where files will be written), a task-id- (the RemoteSyncTask identifier used to track progress updates), and a message (the commit message to use when- writing files to the source).+ Takes a stream (a sequence of serialized entities to be stored), a snapshot (the remote source+ implementing the SourceSnapshot protocol where files will be written), a task-id (the RemoteSyncTask+ identifier used to track progress updates), and a message (the commit message to use when writing+ files to the source).Returns the version written to the source.Throws Exception if any entity in the stream is an Exception instance."- [stream delete-prefixes snapshot task-id message]+ [stream snapshot task-id message](let [opts (serdes/storage-base-context)- ;; Bound the count of the items in the stream we don't accidentally realize the entire list into memorystream-count (bounded-count 10000 stream)]- (->> (concat stream delete-prefixes)+ (->> stream(map-indexed #(->file-spec task-id stream-count opts %1 %2))(source.p/write-files! snapshot message))))@@ -97,6 +91,7 @@(git/git-source(setting/get :remote-sync-url)(or branch (setting/get :remote-sync-branch))- (setting/get :remote-sync-token)))+ (setting/get :remote-sync-token)+ serialization/legal-top-level-paths))([](source-from-settings (setting/get :remote-sync-branch))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex 842622c3b20..db71464ee93 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -231,21 +231,12 @@(throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))push-response))-(defn- path-prefix- "Extracts the unique collection identifier from a serialized file path.-- Takes a path string like \"collections/abc123_CollectionName/...\" and returns the collection prefix (e.g.,- \"collections/abc123\") which remains stable even when collection names change.-- Returns the original path if no collection prefix is found."+(defn- top-level-dir+ "Extracts the first path segment from a file path.+ E.g., \"databases/my_db/my_db.yaml\" => \"databases\""[path]- (let [matcher (re-matcher #"^(collections/[^/]{21})_[^/]+/" path)]- (if (re-find matcher)- (second (re-groups matcher))- path)))--(defn- matches-prefix [path prefixes]- (some #(or (= % path) (str/starts-with? path %)) prefixes))+ (when-let [idx (str/index-of path "/")]+ (subs path 0 idx)))(defn default-branch"Retrieves the default branch name of the git repository.@@ -267,24 +258,17 @@(defn write-files!"Writes multiple files to the git repository and commits the changes.- Takes a snapshot map containing a :git Git instance and :version, a commit message string,- and a sequence of file specs (paths should be relative to the repository root).-- Each file spec is a map with either:- - :path and :content keys for writing/updating a file- - :path and :remove? true for recursively removing all files at that path-- For writes within collection directories, ALL files in the same collection are replaced- (using the collection entity_id prefix to identify the collection scope). This ensures- that stale files don't remain when a collection's contents change.+ Takes a snapshot map containing a :git Git instance, :version, and :managed-dirs,+ a commit message string, and a sequence of file specs (maps with :path and :content keys,+ paths should be relative to the repository root).- For removals, all files matching the path as a prefix are deleted (allowing recursive- directory deletion). Removal entries with empty paths are no-ops. Removing non-existent- paths is also a no-op (idempotent).+ All existing files within managed directories that are not in the write set are removed.+ This ensures stale files (from renames, moves, or deletions) are cleaned up automatically.+ Files outside managed directories are always preserved.Returns the version written. Throws ExceptionInfo if the write or pushoperation fails."- [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message files]+ [{:keys [^Git git ^String version ^String branch managed-dirs] :as snapshot} ^String message files](let [repo (.getRepository git)branch-ref (qualify-branch branch)parent-id (.resolve repo version)]@@ -292,35 +276,25 @@(with-open [inserter (.newObjectInserter repo)](let [index (DirCache/newInCore)builder (.builder index)- ;; Extract collection prefixes from written paths - all files in these- ;; collections will be deleted and replaced with the new files- write-prefixes (into #{}- (comp- (remove :remove?)- (map :path)- (remove str/blank?)- (map path-prefix))- files)- ;; Collect removal paths/prefixes for explicit deletions- removal-prefixes (into #{}- (comp- (filter :remove?)- (map :path)+ ;; Set of all paths being written in this commit+ write-paths (into #{}+ (comp (map :path)(remove str/blank?))- files)]+ files)];; Add new/updated files to the index- (doseq [{:keys [path content remove?]} files- :when (and (not remove?) (not (str/blank? path)))]+ (doseq [{:keys [path content]} files+ :when (not (str/blank? path))](let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))entry (doto (DirCacheEntry. ^String path)(.setFileMode FileMode/REGULAR_FILE)(.setObjectId blob-id))](.add builder entry)))- ;; Copy existing tree entries, excluding:- ;; 1. Files in collections being written to (using write-prefixes)- ;; 2. Files matching explicit removal prefixes+ ;; Copy existing tree entries that should be preserved:+ ;; - Outside managed directories AND not being overwritten by the write set+ ;; Files in managed dirs not in write-paths are dropped (stale file cleanup)+ ;; Files in write-paths are skipped here (already added above with new content)(when parent-id(with-open [rev-walk (RevWalk. repo)tree-walk (TreeWalk. repo)]@@ -328,10 +302,9 @@(.addTree tree-walk (.getTree commit))(.setRecursive tree-walk true)(while (.next tree-walk)- (let [path (.getPathString tree-walk)- existing-prefix (path-prefix path)]- (when-not (or (contains? write-prefixes existing-prefix)- (matches-prefix path removal-prefixes))+ (let [path (.getPathString tree-walk)]+ (when (and (not (contains? write-paths path))+ (not (contains? managed-dirs (top-level-dir path))))(let [entry (doto (DirCacheEntry. path)(.setFileMode (.getFileMode tree-walk 0))(.setObjectId (.getObjectId tree-walk 0)))]@@ -425,7 +398,7 @@(push-branch! (assoc source :branch branch-name))branch-name))-(defrecord GitSnapshot [git remote-url branch version token]+(defrecord GitSnapshot [git remote-url branch version token managed-dirs]source.p/SourceSnapshot(list-files [this]@@ -470,7 +443,7 @@(fetch! source)(let [version (commit-sha source (:branch source))](if version- (->GitSnapshot (:git source) (:remote-url source) (:branch source) version (:token source))+ (->GitSnapshot (:git source) (:remote-url source) (:branch source) version (:token source) (:managed-dirs source))(throw (ex-info (str "Invalid branch: " (:branch source)) {})))))(defn- snapshot@@ -488,7 +461,7 @@(snapshot* fresh-source)))(throw e)))))-(defrecord GitSource [git remote-url branch token]+(defrecord GitSource [git remote-url branch token managed-dirs]source.p/Source(branches [source] (branches source))@@ -504,10 +477,12 @@(defn git-source"Creates a new GitSource instance for a git repository.- Takes a URL string (the git repository URL), a branch, and an- optional token string (authentication token for private repositories).+ Takes a URL string (the git repository URL), a branch, an+ optional token string (authentication token for private repositories),+ and a set of managed top-level directory names. Files in managed directories+ are fully replaced during writes — any existing file not in the write set is removed.Returns a GitSource record implementing the Source protocol."- [url branch token]+ [url branch token managed-dirs](->GitSource (get-jgit (repo-path {:remote-url url :token token}) {:remote-url url :token token})- url branch token))+ url branch token managed-dirs))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.cljindex aa42e1923d0..9de80c1daaa 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj@@ -51,11 +51,10 @@"Writes multiple files to the source with a commit message.Takes a SourceSnapshot instance implementing this protocol, a message (the commit message to use when writing files),- and files (a sequence of file specs). Each file spec is a map with either:- - :path and :content keys for writing/updating a file- - :path and :remove? true for recursively removing all files at that path+ and files (a sequence of file specs). Each file spec is a map with :path and :content keys.- Removal entries with empty paths are no-ops. Removing non-existent paths is also a no-op (idempotent).+ All existing files within managed directories that are not in the write set are removed.+ Files outside managed directories are always preserved.Returns the version of the written files.")diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 47bd61da729..87863feba0d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -15,14 +15,12 @@[clojure.string :as str][clojure.walk :as walk][metabase-enterprise.remote-sync.settings :as rs-settings]- [metabase-enterprise.serialization.core :as serialization][metabase-enterprise.transforms-python.core :as transforms-python][metabase.collections.core :as collections][metabase.collections.models.collection :as collection][metabase.models.serialization :as serdes][metabase.settings.core :as setting][metabase.util :as u]- [metabase.util.log :as log][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -861,42 +859,6 @@:by-path {}}seen-paths))-;;; --------------------------------------------- Export Path Construction -----------------------------------------------(defn- transform-entity-for-serdes- "Transforms entity fields to serdes format for path generation.- - Converts integer table_id to [db-name schema table-name] format- - Converts integer collection_id to entity_id string for collection path lookup"- [entity]- (cond-> entity- ;; Transform table_id for Segment (and other table-based entities)- (and (:table_id entity) (integer? (:table_id entity)))- (assoc :table_id (serdes/*export-table-fk* (:table_id entity)))- ;; Transform collection_id for snippet and other collection-based entities- (and (:collection_id entity) (integer? (:collection_id entity)))- (assoc :collection_id (t2/select-one-fn :entity_id :model/Collection :id (:collection_id entity)))))--(defn- entity->serdes-path- "Builds the file path for an entity using serdes/storage-path.- For Collections, returns the directory path (for recursive deletion).- For other entities, returns the full file path.- Returns the path as a string (without extension), or nil if path cannot be built."- [model-type entity ctx]- (try- (when-let [serdes-meta (serdes/generate-path model-type entity)]- (let [;; Transform entity fields to serdes format (e.g., table_id -> [db schema table])- transformed-entity (transform-entity-for-serdes entity)- entity-with-meta (assoc transformed-entity :serdes/meta serdes-meta)- resolved (serialization/resolve-storage-path ctx entity-with-meta)]- ;; For Collections, drop the last segment (filename) to get directory path- (if (= model-type "Collection")- (str/join "/" (butlast resolved))- (str/join "/" resolved))))- (catch Exception e- (log/warnf "Failed to build storage path for %s %s: %s"- model-type (:id entity) (ex-message e))- nil)))-;;; -------------------------------------------- Event Helper Functions ------------------------------------------------(defn determine-status@@ -937,86 +899,6 @@[:id :name :collection_id])[:id :name :collection_id]))-;;; -------------------------------------------- Removal Path Building ----------------------------------------------------(defn- query-removed-ids- "Queries RemoteSyncObject for model IDs marked for removal."- [model-type statuses]- (t2/select-fn-set :model_id :model/RemoteSyncObject- :model_type model-type- :status [:in (vec statuses)]))--(defn- query-removed-entities- "Queries entities marked for removal with all fields needed for serdes path generation.- serdes/generate-path needs: entity_id, name (for label)- serdes/storage-path needs: collection_id (for collection context), table_id (for segment paths)"- [{:keys [model-type model-key removal]}]- (let [{:keys [statuses]} removal- removed-ids (query-removed-ids model-type statuses)]- (when (seq removed-ids)- ;; Select all columns since different models need different fields for serdes paths- (t2/select model-key :id [:in removed-ids]))))--(defn- setting-sentinel-delete?- "Returns true if the setting's sentinel RSO exists with 'delete' status.- Used to detect when a setting (like :remote-sync-transforms) has been disabled- and all entities of the controlled types should be removed."- [setting-key]- (when (= setting-key :remote-sync-transforms)- (t2/exists? :model/RemoteSyncObject- :model_type "Collection"- :model_id rs-settings/transforms-root-id- :status "delete")))--(defn- build-bulk-removal-paths- "Builds removal paths for ALL entities of a spec's model type.- Used when the controlling setting is disabled (sentinel RSO has 'delete' status).- Respects :removal-conditions (or :conditions) from the spec to exclude ineligible entities."- [spec ctx]- (let [{:keys [model-type model-key]} spec- conditions (removal-conditions spec)]- (for [entity (if conditions- (apply t2/select model-key (into [] cat conditions))- (t2/select model-key))- :let [path (entity->serdes-path model-type entity ctx)]- :when path]- path)))--(defn build-all-removal-paths- "Builds full file paths for all entities marked for removal in RemoteSyncObject.- Uses serdes/storage-path to generate paths that exactly match the file structure.-- Does bulk queries per model type for efficiency:- 1. Query RemoteSyncObject for model_ids with removal statuses- 2. Query actual entities by those IDs- 3. Use serdes/storage-path to build the exact file path-- Also handles bulk removal for specs with :all-on-setting-disable when the- controlling setting's sentinel RSO has 'delete' status.-- Returns paths without file extensions - the git source adds .yaml as needed."- []- (let [ctx (serdes/storage-base-context)- ;; Standard removal paths from enabled specs- standard-paths (into []- (for [[_model-key spec] (enabled-specs)- :when (spec-enabled? spec)- :let [{:keys [statuses]} (:removal spec)- model-type (:model-type spec)]- :when (seq statuses)- entity (query-removed-entities spec)- :let [path (entity->serdes-path model-type entity ctx)]- :when path]- path))- ;; Bulk removal paths for specs with :all-on-setting-disable- bulk-paths (into []- (for [[_model-key spec] remote-sync-specs- :let [setting-key (get-in spec [:removal :all-on-setting-disable])]- :when (and setting-key (setting-sentinel-delete? setting-key))- path (build-bulk-removal-paths spec ctx)]- path))]- (into standard-paths bulk-paths)))-;;; ----------------------------------------- Sync Object Query Functions --------------------------------------------(defmulti query-entities-for-syncdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.cljindex 73449d05e0b..2ad83cdfe67 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj@@ -21,6 +21,7 @@make-targets-of-typeextract][metabase-enterprise.serialization.v2.ingest+ legal-top-level-pathsstrip-labelsIngestableingest-yamldiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.cljindex 623db08741d..8c7c0c09b8d 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj@@ -406,15 +406,14 @@ is_sample: false:model_collection_id coll-id:status "delete":status_changed_at (t/offset-date-time)}]- (let [removal-paths (spec/build-all-removal-paths)- snippet-removal-path (first (filter #(str/includes? % "archived_snippet") removal-paths))- _ (is (some? snippet-removal-path) "Should have a removal path for the archived snippet")- initial-files {"main" {(str snippet-removal-path ".yaml")+ (let [snippet-file-path (str "snippets/" snippet-eid "_archived_snippet.yaml")+ initial-files {"main" {snippet-file-path(test-helpers/generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}mock-source (test-helpers/create-mock-source :initial-files initial-files)result (impl/export! (source.p/snapshot mock-source) task-id "Test export")](is (= :success (:status result))(str "Export should succeed. Result: " result))+ ;; Archived snippet is not in the export stream, so its file gets cleaned from snippets/ (a managed dir)(let [files-after-export (get @(:files-atom mock-source) "main")](is (not (some #(str/includes? % "archived_snippet") (keys files-after-export)))"Archived snippet file should be deleted after export")))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex aafe703e784..486310ceeb4 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -80,6 +80,10 @@remote))+(def ^:private test-managed-dirs+ "Managed dirs for test sources — matches v2.ingest/legal-top-level-paths."+ #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})+(defn- ->source!"Creates a (local) 'remote' repo and initializes a git source that uses it"[branch {:keys [^Git git] :as _remote-repo}]@@ -89,7 +93,7 @@(.toURL)(.toExternalForm))local-repo (#'git/get-jgit (#'git/repo-path {:remote-url remote-url}) {:remote-url remote-url})]- (git/->GitSource local-repo remote-url branch nil)))+ (git/->GitSource local-repo remote-url branch nil test-managed-dirs)))(defn- init-source![branch dir & config]@@ -97,27 +101,6 @@(let [remote-repo (apply init-remote! dir config)][(->source! branch remote-repo) remote-repo]))-(deftest path-prefix- (let [id (u/generate-nano-id "a")]- (testing "Not in a collection"- (doseq [path ["asdf"- "asdf.txt"- "dir/asdf.txt"- "collections/asdf.txt"- "collections/asdf/a.txt"- "invalid/collections/asdf/a.txt"- (str "collections/" id)- (str "collections/" id "/but_no_name")]]- (testing path- (is (= path (#'git/path-prefix path)))- (is (true? (#'git/matches-prefix path #{(str "collections/" (u/generate-nano-id)) path}))))))- (testing "In a collection"- (doseq [path [(str "collections/" id "_my_name/asdf")- (str "collections/" id "_other_name/subdir/asdf.txt")]]- (testing path- (is (= (str "collections/" id) (#'git/path-prefix path)))- (is (true? (#'git/matches-prefix path #{(str "collections/" id) (str "collections/" (u/generate-nano-id))}))))))))-(deftest qualify-branch-test(is (= "refs/heads/main" (#'git/qualify-branch "main")))(is (= "refs/heads/main" (#'git/qualify-branch "refs/heads/main"))))@@ -180,7 +163,6 @@(deftest write-files(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")- otherdir-path (str "collections/" "o" (subs (u/generate-nano-id "b") 1) "_otherdir/")thirddir-path (str "collections/" "s" (subs (u/generate-nano-id "c") 1) "_thirddir/")](mt/with-temp-dir [remote-dir nil](let [[master remote] (init-source! "master" remote-dir@@ -188,12 +170,10 @@"master2.txt" "File 2 in master"(str subdir-path "path.txt") "File in subdir"(str subdir-path "path2.txt") "File 2 in subdir"- (str otherdir-path "path.txt") "File in otherdir"- (str otherdir-path "path2.txt") "File 2 in otherdir"(str thirddir-path "path.txt") "File in third dir"(str thirddir-path "path2.txt") "File 2 in third dir"}:branches ["branch-1" "branch-2"])]- (testing "Files in a subdir are replaced, other subdirs and root are unchanged"+ (testing "All files in managed dirs not in write set are removed; root files outside managed dirs are preserved"(source.p/write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}{:path (str subdir-path "path.txt") :content "Updated subdir content"}{:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}@@ -201,9 +181,8 @@{:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])(is (= ["Update 1" "Initial commit"] (map :message (git/log master))))(let [master-snap (source.p/snapshot master)]- (is (= [(str otherdir-path "path.txt")- (str otherdir-path "path2.txt")- (str subdir-path "path.txt")+ ;; otherdir files are removed because collections/ is a managed dir and those files weren't in the write set+ (is (= [(str subdir-path "path.txt")(str subdir-path "path3.txt")(str thirddir-path "path.txt")(str thirddir-path "path3.txt")@@ -213,15 +192,12 @@(is (= "Updated master content" (source.p/read-file master-snap "master.txt")))(is (= "File 2 in master" (source.p/read-file master-snap "master2.txt")))- (is (= "File 2 in otherdir" (source.p/read-file master-snap (str otherdir-path "path2.txt"))))(is (= "Updated subdir content" (source.p/read-file master-snap (str subdir-path "path.txt"))))(is (= "Updated subdir content 3" (source.p/read-file master-snap (str subdir-path "path3.txt")))))(testing "Check remote repo directly"(is (= "Updated master content" (git/read-file (assoc remote :version "master") "master.txt")))- (is (= [(str otherdir-path "path.txt")- (str otherdir-path "path2.txt")- (str subdir-path "path.txt")+ (is (= [(str subdir-path "path.txt")(str subdir-path "path3.txt")(str thirddir-path "path.txt")(str thirddir-path "path3.txt")@@ -230,13 +206,9 @@(git/list-files (assoc remote :version "master"))))(is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))))- (testing "If no root files are touched, they all stay as-is"+ (testing "Writing only to collections/ removes all other collection files"(source.p/write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])- (is (= [(str otherdir-path "path.txt")- (str otherdir-path "path2.txt")- (str subdir-path "path.txt")- (str subdir-path "path3.txt")- (str thirddir-path "path.txt")+ (is (= [(str thirddir-path "path.txt")"master.txt""master2.txt"](git/list-files (assoc remote :version "master")))))))))@@ -249,58 +221,57 @@(str subdir-path "path.txt") "File in subdir"})]))))(deftest concurrent-access- (let [subdir-path (str "collections/" (u/generate-nano-id "a") "_subdir")]- (mt/with-temp-dir [remote-dir nil]- (let [[master remote] (init-source! "master" remote-dir- :files {"master.txt" "File in master"- (str subdir-path "path.txt") "File in subdir"}- :branches ["branch-1" "branch-2"])- new-branch (->source! "new-branch" remote)]+ (mt/with-temp-dir [remote-dir nil]+ (let [[master remote] (init-source! "master" remote-dir+ :files {"master.txt" "File in master"+ "subdir/path.txt" "File in subdir"}+ :branches ["branch-1" "branch-2"])+ new-branch (->source! "new-branch" remote)]- (testing "Initial clone is the same"- (is (= ["Initial commit"] (map :message (git/log master))))- (is (= ["Initial commit"] (map :message (git/log (assoc remote :branch "master")))))+ (testing "Initial clone is the same"+ (is (= ["Initial commit"] (map :message (git/log master))))+ (is (= ["Initial commit"] (map :message (git/log (assoc remote :branch "master")))))- ;; Add an extra commit to remote- (git-working-add! remote "additional-file.txt" "Additional file content")- (git-working-commit! remote "Added additional file")+ ;; Add an extra commit to remote+ (git-working-add! remote "additional-file.txt" "Additional file content")+ (git-working-commit! remote "Added additional file")- (testing "Source is behind remote"- (is (= ["Initial commit"] (map :message (git/log master))))- (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master"))))))+ (testing "Source is behind remote"+ (is (= ["Initial commit"] (map :message (git/log master))))+ (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master"))))))- (testing "After fetch, source is up to date"- (git/fetch! master)- (is (= ["Added additional file" "Initial commit"] (map :message (git/log master)))))+ (testing "After fetch, source is up to date"+ (git/fetch! master)+ (is (= ["Added additional file" "Initial commit"] (map :message (git/log master)))))- (testing "Writing a file to source and pushing back to remote when there is new content on remote"- ;; Make source be behind again- (git-working-add! remote "only-on-remote.txt" "Initially on remote")- (git-working-commit! remote "Only on remote")+ (testing "Writing a file to source and pushing back to remote when there is new content on remote"+ ;; Make source be behind again+ (git-working-add! remote "only-on-remote.txt" "Initially on remote")+ (git-working-commit! remote "Only on remote")- (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])+ (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])- (testing "Remote has the new commit with just the files committed, but only version is in history"- (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))- (is (= ["additional-file.txt" (str subdir-path "path.txt") "initially-source.txt" "master.txt" "only-on-remote.txt"] (git/list-files (assoc remote :version "master"))))- (is (= "Initially on source" (git/read-file (assoc remote :version "master") "initially-source.txt"))))+ (testing "Remote has the new commit with just the files committed, but only version is in history"+ (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))+ (is (= ["additional-file.txt" "initially-source.txt" "master.txt" "only-on-remote.txt" "subdir/path.txt"] (git/list-files (assoc remote :version "master"))))+ (is (= "Initially on source" (git/read-file (assoc remote :version "master") "initially-source.txt"))))- (testing "Source has the same history"- (is (= (map :message (git/log (assoc remote :branch "master"))) (map :message (git/log master))))))+ (testing "Source has the same history"+ (is (= (map :message (git/log (assoc remote :branch "master"))) (map :message (git/log master))))))- (testing "Writing to a branch local has not seen (but remote has) adds it to the history on remote"- (git-working-checkout! remote "new-branch" true)- (git-working-add! remote "new-branch-file.txt" "Initially on remote")- (git-working-add! remote "new-branch-remote.txt" "Initially on remote")- (git-working-commit! remote "New-branch on remote")+ (testing "Writing to a branch local has not seen (but remote has) adds it to the history on remote"+ (git-working-checkout! remote "new-branch" true)+ (git-working-add! remote "new-branch-file.txt" "Initially on remote")+ (git-working-add! remote "new-branch-remote.txt" "Initially on remote")+ (git-working-commit! remote "New-branch on remote")- (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))- (is (nil? (git/log new-branch)))+ (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))+ (is (nil? (git/log new-branch)))- (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}- {:path "new-branch-file.txt" :content "Updated on source"}])+ (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}+ {:path "new-branch-file.txt" :content "Updated on source"}])- (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))))))))+ (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch"))))))))))(deftest git-source-using-commit-ref(mt/with-temp-dir [remote-dir nil]@@ -358,67 +329,64 @@(let [[master _remote] (init-source! "master" remote-dir)](is (= "master" (git/default-branch master))))))-(deftest write-files-removal-test- (let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")- otherdir-path (str "collections/" "o" (subs (u/generate-nano-id "b") 1) "_otherdir/")]+(deftest write-files-top-level-exports-replaced-test+ (let [old-col-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_mycol/")+ new-col-path (str "collections/" "s" (subs (u/generate-nano-id "b") 1) "_othercol/")+ kept-col-path (str "collections/" "t" (subs (u/generate-nano-id "c") 1) "_keptcol/")](mt/with-temp-dir [remote-dir nil](let [[master _remote] (init-source! "master" remote-dir- :files {"master.txt" "File in master"- (str subdir-path "file1.yaml") "File 1 in subdir"- (str subdir-path "file2.yaml") "File 2 in subdir"- (str otherdir-path "file1.yaml") "File 1 in otherdir"- (str otherdir-path "file2.yaml") "File 2 in otherdir"})]- (testing "Removal entry deletes all files under that path recursively"- (source.p/write-files! (source.p/snapshot master) "Remove subdir"- [{:path (subs subdir-path 0 (dec (count subdir-path))) :remove? true}])+ :files {"databases/old_db/old_db.yaml" "Old database"+ "databases/old_db/schemas/public.yaml" "Old schema"+ "snippets/old_snippet.yaml" "Old snippet"+ "unmanaged/keep_me.txt" "Unmanaged file"+ (str old-col-path "cards/card1.yaml") "Card in old col"+ (str old-col-path "cards/card2.yaml") "Card 2 in old col"+ (str kept-col-path "dashboards/dash1.yaml") "Dashboard in kept col"})]+ (testing "Writing to a managed dir removes all stale files in ALL managed dirs"+ (source.p/write-files! (source.p/snapshot master) "Rename database"+ [{:path "databases/new_db/new_db.yaml" :content "Renamed database"}+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}+ {:path (str old-col-path "cards/card1.yaml") :content "Card in old col"}+ {:path (str old-col-path "cards/card2.yaml") :content "Card 2 in old col"}+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard in kept col"}+ {:path "snippets/old_snippet.yaml" :content "Old snippet"}])(let [files (set (source.p/list-files (source.p/snapshot master)))]- (is (contains? files "master.txt") "Root files should remain")- (is (contains? files (str otherdir-path "file1.yaml")) "Other collection files should remain")- (is (contains? files (str otherdir-path "file2.yaml")) "Other collection files should remain")- (is (not (contains? files (str subdir-path "file1.yaml"))) "Subdir files should be removed")- (is (not (contains? files (str subdir-path "file2.yaml"))) "Subdir files should be removed")))))))--(deftest write-files-mixed-write-and-removal-test- (let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")- newdir-path (str "collections/" "n" (subs (u/generate-nano-id "b") 1) "_newdir/")]+ (is (contains? files "databases/new_db/new_db.yaml") "New database file should exist")+ (is (contains? files "databases/new_db/schemas/public.yaml") "New schema file should exist")+ (is (not (contains? files "databases/old_db/old_db.yaml")) "Old database file should be removed")+ (is (not (contains? files "databases/old_db/schemas/public.yaml")) "Old schema file should be removed")+ (is (contains? files (str old-col-path "cards/card1.yaml")) "Written collection files should remain")+ (is (contains? files "snippets/old_snippet.yaml") "Written snippet file should remain")+ (is (contains? files "unmanaged/keep_me.txt") "Unmanaged files should be untouched")))++ (testing "Entity moved between collections removes files from old collection"+ (source.p/write-files! (source.p/snapshot master) "Move card to new collection"+ [{:path (str new-col-path "cards/card1.yaml") :content "Card moved to new col"}+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard still here"}+ {:path "databases/new_db/new_db.yaml" :content "Renamed database"}+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}])+ (let [files (set (source.p/list-files (source.p/snapshot master)))]+ (is (contains? files (str new-col-path "cards/card1.yaml")) "Moved card should exist in new collection")+ (is (contains? files (str kept-col-path "dashboards/dash1.yaml")) "Kept collection files should remain")+ (is (not (contains? files (str old-col-path "cards/card1.yaml"))) "Old collection card should be removed")+ (is (not (contains? files (str old-col-path "cards/card2.yaml"))) "Other files in old collection should also be removed")+ (is (not (contains? files "snippets/old_snippet.yaml")) "Snippets cleaned up when not in write set")+ (is (contains? files "unmanaged/keep_me.txt") "Unmanaged files still untouched")))))))++(deftest write-files-entity-rename-within-collection-test+ (let [col-path (str "collections/" "u" (subs (u/generate-nano-id "d") 1) "_col/")](mt/with-temp-dir [remote-dir nil](let [[master _remote] (init-source! "master" remote-dir- :files {"master.txt" "File in master"- (str subdir-path "old-file.yaml") "Old file in subdir"})]- (testing "Can combine write and removal entries in same call"- (source.p/write-files! (source.p/snapshot master) "Mixed operations"- [{:path (subs subdir-path 0 (dec (count subdir-path))) :remove? true}- {:path (str newdir-path "new-file.yaml") :content "New file content"}])- (let [snap (source.p/snapshot master)- files (set (source.p/list-files snap))]- (is (contains? files "master.txt") "Root files should remain")- (is (contains? files (str newdir-path "new-file.yaml")) "New files should be added")- (is (not (contains? files (str subdir-path "old-file.yaml"))) "Old files should be removed")- (is (= "New file content" (source.p/read-file snap (str newdir-path "new-file.yaml"))))))))))--(deftest write-files-empty-removal-path-test- (mt/with-temp-dir [remote-dir nil]- (let [[master _remote] (init-source! "master" remote-dir- :files {"master.txt" "File in master"- "subdir/file.yaml" "File in subdir"})]- (testing "Empty removal paths are ignored (no-op)"- (source.p/write-files! (source.p/snapshot master) "Empty removal"- [{:path "" :remove? true}- {:path " " :remove? true}])- (let [files (set (source.p/list-files (source.p/snapshot master)))]- (is (= #{"master.txt" "subdir/file.yaml"} files)- "All files should remain when removal path is empty"))))))--(deftest write-files-nonexistent-removal-path-test- (mt/with-temp-dir [remote-dir nil]- (let [[master _remote] (init-source! "master" remote-dir- :files {"master.txt" "File in master"})]- (testing "Removing non-existent path is a no-op"- (source.p/write-files! (source.p/snapshot master) "Remove nonexistent"- [{:path "collections/nonexistent" :remove? true}])- (let [files (set (source.p/list-files (source.p/snapshot master)))]- (is (= #{"master.txt"} files)- "Files should remain unchanged"))))))+ :files {(str col-path "cards/eid123_OldCardName.yaml") "Card with old name"+ (str col-path "cards/eid456_OtherCard.yaml") "Other card"})]+ (testing "Entity renamed within a collection removes the old-named file"+ (source.p/write-files! (source.p/snapshot master) "Rename card"+ [{:path (str col-path "cards/eid123_NewCardName.yaml") :content "Card with new name"}+ {:path (str col-path "cards/eid456_OtherCard.yaml") :content "Other card"}])+ (let [files (set (source.p/list-files (source.p/snapshot master)))]+ (is (contains? files (str col-path "cards/eid123_NewCardName.yaml")) "Renamed card should exist")+ (is (contains? files (str col-path "cards/eid456_OtherCard.yaml")) "Other card should still exist")+ (is (not (contains? files (str col-path "cards/eid123_OldCardName.yaml"))) "Old card name file should be removed")))))))(deftest ensure-origin-configured-sets-origin-after-clone-test(mt/with-temp-dir [remote-dir nil]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.cljindex 89446734b9a..7bc3bea0a34 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj@@ -157,26 +157,3 @@(is (= []@written-files)"Should write no files when none match"))))--(deftest wrapping-source-write-files-removal-entries-test- (testing "WrappingSource filters removal entries based on path-filters"- (let [written-files (atom [])- mock-snap (reify source.p/SourceSnapshot- (list-files [_]- [])- (read-file [_ _path]- nil)- (write-files! [_ _message files]- (reset! written-files (vec files))- nil)- (version [_]- "mock-version"))- wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*"])- files-to-write [{:path "collections/foo.yaml" :content "foo-content"}- {:path "collections/abc123" :remove? true}- {:path "databases/db1" :remove? true}]]- (source.p/write-files! wrapped-snap "test commit" files-to-write)- (is (= [{:path "collections/foo.yaml" :content "foo-content"}- {:path "collections/abc123" :remove? true}]- @written-files)- "Should pass through removal entries that match filter and filter out those that don't"))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex 21e452f8c5d..0f77acc5cf4 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -63,7 +63,7 @@test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")(create-test-entity "test-id-2" "entity-two" "Card")]]- (is (= "mock-written-version" (source/store! test-entities [] (source.p/snapshot mock-source) task-id "Test commit message")))+ (is (= "mock-written-version" (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message")))(testing "write-files! was called with correct message"(is (= "Test commit message" (:message @written-files))))@@ -108,7 +108,7 @@(let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)](is (nil? (:progress initial-task)) "Progress should be nil initially"))- (source/store! test-entities [] (source.p/snapshot mock-source) task-id "Test commit")+ (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit")(let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)](is (some? (:progress final-task)) "Progress should be updated after store!")@@ -128,7 +128,7 @@(let [written-files (atom nil)mock-source (->MockSource written-files)]- (source/store! [] [] (source.p/snapshot mock-source) task-id "Empty commit")+ (source/store! [] (source.p/snapshot mock-source) task-id "Empty commit")(testing "write-files! was called even with empty stream"(is (some? @written-files)))@@ -159,8 +159,8 @@:dataset_query {:z_field 1 :a_field 2 :database 3}:visualization_settings {:zebra true :apple false}}];; Store the same entity twice- (source/store! [test-entity] [] (source.p/snapshot mock-source-1) task-id "First commit")- (source/store! [test-entity] [] (source.p/snapshot mock-source-2) task-id "Second commit")+ (source/store! [test-entity] (source.p/snapshot mock-source-1) task-id "First commit")+ (source/store! [test-entity] (source.p/snapshot mock-source-2) task-id "Second commit")(testing "YAML content is identical between runs"(let [content-1 (-> @written-files-1 :files first :content)content-2 (-> @written-files-2 :files first :content)]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.cljindex d2e7b1d550f..a98a02b1b43 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj@@ -245,12 +245,6 @@(let [spec (spec/spec-for-model-key :model/Card)](is (nil? (spec/build-sync-object-fields spec nil))))))-;;; ------------------------------------------- Removal Path Building Tests --------------------------------------------;; Note: build-all-removal-paths uses serdes/storage-path to generate paths.-;; The actual paths are tested in impl_test.clj integration tests which verify-;; the full export/import cycle with real database entities.-;;; --------------------------------------------- Fields for Sync Tests -----------------------------------------------(deftest fields-for-sync-testdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 445e60ff65a..590ae6a0928 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -163,7 +163,13 @@ width: fixed"name entity-id collection-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_") dashcards-yaml)))-(defrecord MockSourceSnapshot [source-id base-url branch fail-mode files-atom]+(defn- top-level-dir+ "Extracts the first path segment from a file path."+ [path]+ (when-let [idx (str/index-of path "/")]+ (subs path 0 idx)))++(defrecord MockSourceSnapshot [source-id base-url branch fail-mode files-atom managed-dirs]source.p/SourceSnapshot(list-files [_this](case fail-mode@@ -190,28 +196,25 @@ width: fixed:write-files-error (throw (Exception. "Failed to write files")):store-error (throw (Exception. "Store failed")):network-error (throw (java.net.UnknownHostException. "Remote host not found"))- ;; Default success case - handle both writes and removals- (let [write-entries (remove #(or (:remove? %) (str/blank? (:path %))) files)- removal-prefixes (into #{} (comp (filter :remove?)- (map :path)- (remove str/blank?))- files)+ ;; Default success case - remove all files in managed dirs, then add new files+ (let [write-entries (remove #(str/blank? (:path %)) files)+ write-paths (into #{} (map :path) write-entries)current-files (get @files-atom branch {})- ;; Remove files matching removal prefixes- after-removals (into {}- (remove (fn [[path _]]- (some #(or (= path %) (str/starts-with? path %))- removal-prefixes))- current-files))+ ;; Keep files outside managed dirs or in the write set+ kept-files (into {}+ (filter (fn [[path _]]+ (or (not (contains? managed-dirs (top-level-dir path)))+ (contains? write-paths path))))+ current-files);; Add new files- final-files (into after-removals (map (juxt :path :content) write-entries))]+ final-files (into kept-files (map (juxt :path :content) write-entries))](swap! files-atom assoc branch final-files)))"write-files-version")(version [_this]"mock-version"))-(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom]+(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom managed-dirs]source.p/Source(create-branch [_this branch _base](swap! branches-atom conj [branch (str branch "-ref")]))@@ -228,14 +231,19 @@ width: fixed"main")(snapshot [_this]- (->MockSourceSnapshot source-id base-url branch fail-mode files-atom)))+ (->MockSourceSnapshot source-id base-url branch fail-mode files-atom managed-dirs)))++(def ^:private default-managed-dirs+ "Default managed dirs for mock sources, matching v2.ingest/legal-top-level-paths."+ #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})(defn create-mock-source- "Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, and `:initial-files`."- [& {:keys [branch fail-mode initial-files]+ "Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, `:initial-files`, and `:managed-dirs`."+ [& {:keys [branch fail-mode initial-files managed-dirs]:or {branch "main"fail-mode nil- initial-files nil}}]+ initial-files nil+ managed-dirs default-managed-dirs}}](let [default-files {"main" {"collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml"(generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")@@ -254,7 +262,7 @@ width: fixedfiles-atom (atom (or initial-files default-files))branches-atom (atom #{["main" "main-ref"] ["develop" "develop-ref"]})]- (->MockSource "test-source" "https://test.example.com" branch fail-mode files-atom branches-atom)))+ (->MockSource "test-source" "https://test.example.com" branch fail-mode files-atom branches-atom managed-dirs)))(defn clean-object"Test fixture that resets the RemoteSyncObject table before running tests to prevent existingdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.cljindex 00cddf64e33..6a279468330 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj@@ -5,64 +5,44 @@[metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as th]))-(deftest mock-source-write-files-removal-test- (testing "MockSource handles removal entries correctly"+(deftest mock-source-write-files-managed-dir-cleanup-test+ (testing "MockSource removes files in managed dirs not in write set"(let [source (th/create-mock-source:initial-files {"main" {"collections/abc/file1.yaml" "content1""collections/abc/file2.yaml" "content2""collections/def/file3.yaml" "content3"- "other/file4.yaml" "content4"}})+ "other/file4.yaml" "content4"}}+ :managed-dirs #{"collections"})snapshot (source.p/snapshot source)]- (source.p/write-files! snapshot "Remove abc collection"- [{:path "collections/abc" :remove? true}])- (is (= #{"collections/def/file3.yaml" "other/file4.yaml"}+ (source.p/write-files! snapshot "Write only abc"+ [{:path "collections/abc/file1.yaml" :content "new-content1"}])+ (is (= #{"collections/abc/file1.yaml" "other/file4.yaml"}(set (source.p/list-files snapshot)))- "Should remove all files under collections/abc recursively"))))+ "Only written files in managed dirs should remain; unmanaged dirs untouched"))))-(deftest mock-source-write-files-removal-exact-path-test- (testing "MockSource removal matches exact path"+(deftest mock-source-write-files-unmanaged-preserved-test+ (testing "MockSource preserves files in unmanaged directories"(let [source (th/create-mock-source- :initial-files {"main" {"collections/abc.yaml" "content1"- "collections/abcdef/file.yaml" "content2"}})- snapshot (source.p/snapshot source)]- (source.p/write-files! snapshot "Remove abc.yaml"- [{:path "collections/abc.yaml" :remove? true}])- (is (= #{"collections/abcdef/file.yaml"}- (set (source.p/list-files snapshot)))- "Should only remove exact path match, not prefix match without slash"))))--(deftest mock-source-write-files-mixed-write-and-removal-test- (testing "MockSource handles both write and removal entries"- (let [source (th/create-mock-source- :initial-files {"main" {"collections/old/file1.yaml" "old-content"- "collections/keep/file2.yaml" "keep-content"}})- snapshot (source.p/snapshot source)]- (source.p/write-files! snapshot "Mixed operations"- [{:path "collections/old" :remove? true}- {:path "collections/new/file3.yaml" :content "new-content"}])- (is (= #{"collections/keep/file2.yaml" "collections/new/file3.yaml"}- (set (source.p/list-files snapshot)))- "Should remove old files and add new files"))))--(deftest mock-source-write-files-empty-removal-path-test- (testing "MockSource ignores removal entries with empty paths"- (let [source (th/create-mock-source- :initial-files {"main" {"collections/abc/file1.yaml" "content1"}})+ :initial-files {"main" {"collections/abc/file1.yaml" "content1"+ "unmanaged/file2.yaml" "content2"}}+ :managed-dirs #{"collections"})snapshot (source.p/snapshot source)]- (source.p/write-files! snapshot "Empty removal"- [{:path "" :remove? true}- {:path " " :remove? true}])- (is (= #{"collections/abc/file1.yaml"}+ (source.p/write-files! snapshot "Write collections"+ [{:path "collections/abc/file1.yaml" :content "new-content"}])+ (is (= #{"collections/abc/file1.yaml" "unmanaged/file2.yaml"}(set (source.p/list-files snapshot)))- "Should not remove anything when removal path is empty or blank"))))+ "Unmanaged directory files should be preserved"))))-(deftest mock-source-write-files-nonexistent-removal-path-test- (testing "MockSource handles removal of non-existent paths (no-op)"+(deftest mock-source-write-files-empty-managed-dir-cleanup-test+ (testing "MockSource cleans managed dir even when no files written to it"(let [source (th/create-mock-source- :initial-files {"main" {"collections/abc/file1.yaml" "content1"}})+ :initial-files {"main" {"collections/abc/file1.yaml" "content1"+ "snippets/old.yaml" "old-snippet"}}+ :managed-dirs #{"collections" "snippets"})snapshot (source.p/snapshot source)]- (source.p/write-files! snapshot "Remove nonexistent"- [{:path "collections/xyz" :remove? true}])+ ;; Write only to collections, nothing to snippets+ (source.p/write-files! snapshot "Write only collections"+ [{:path "collections/abc/file1.yaml" :content "new-content"}])(is (= #{"collections/abc/file1.yaml"}(set (source.p/list-files snapshot)))- "Should be a no-op when removing non-existent path"))))+ "Snippets dir should be cleaned even though no snippet files were written"))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex 29b8a0b88ca..695e60fd771 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -281,6 +281,30 @@(is (not (some #(str/includes? % "collections/transforms/") (keys files-after-export)))"Export should NOT include transform files when setting is disabled"))))))))))+(deftest export-removes-existing-transform-files-when-setting-disabled-test+ (testing "Export removes pre-existing transform files from remote when setting is disabled"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-transforms false+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection {rs-coll-id :id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxxxxx" :location "/"}+ :model/RemoteSyncObject _rso {:model_type "Collection" :model_id rs-coll-id :model_name "Remote Synced" :status "create" :status_changed_at (t/offset-date-time)}]+ ;; Start with transform files already on the remote (from a previous export when setting was enabled)+ (let [initial-files {"main" {"transforms/my_transform/my_transform.yaml" "old transform content"+ "transforms/my_transform/steps/step1.yaml" "old step content"+ "python-libraries/custom_lib.yaml" "old library content"}}+ mock-source (test-helpers/create-mock-source :initial-files initial-files)+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]+ (is (= :success (:status result))+ (str "Export should succeed. Result: " result))+ (let [files-after-export (get @(:files-atom mock-source) "main")]+ (is (not (some #(str/starts-with? % "transforms/") (keys files-after-export)))+ "Transform files should be removed from remote when setting is disabled")+ (is (not (some #(str/starts-with? % "python-libraries/") (keys files-after-export)))+ "Python library files should be removed from remote when setting is disabled"))))))))))+(defn- generate-transforms-namespace-collection-yaml"Generates YAML content for a transforms-namespace collection."[entity-id name]@@ -848,66 +872,6 @@ serdes/meta:(is (not (t2/exists? :model/Transform :id local-transform-id))"Local transform should be removed because it's not on the remote")))))))))-;;; ------------------------------------------- build-all-removal-paths Tests ---------------------------------------------(deftest build-all-removal-paths-includes-all-transforms-on-setting-disable-test- (testing "build-all-removal-paths returns paths for all transforms content when sentinel RSO has 'delete' status"- (mt/with-premium-features #{:transforms-basic}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"- :namespace collection/transforms-ns- :location "/"}- :model/Transform {transform-id :id} {:name "Test Transform"- :collection_id coll-id}- :model/TransformTag {tag-id :id} {:name "Test Tag"- :built_in_type nil}]- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]- (is (t2/exists? :model/Transform :id transform-id))- (is (t2/exists? :model/TransformTag :id tag-id))- (is (t2/exists? :model/PythonLibrary :id (:id library)))- (let [paths-before (spec/build-all-removal-paths)]- (is (not (some #(str/includes? % "test_transform") paths-before))- "Transform should not be in removal paths before setting is disabled"))- (settings/sync-transform-tracking! true)- (settings/sync-transform-tracking! false)- (is (t2/exists? :model/RemoteSyncObject- :model_type "Collection"- :model_id settings/transforms-root-id- :status "delete")- "Sentinel RSO should exist with 'delete' status")- (let [paths-after (spec/build-all-removal-paths)]- (is (some #(str/includes? % "test_transform") paths-after)- "Transform should be in removal paths after setting is disabled")- (is (some #(str/includes? % "test_tag") paths-after)- "TransformTag should be in removal paths after setting is disabled")- (is (some #(str/includes? % "common.py") paths-after)- "PythonLibrary should be in removal paths after setting is disabled"))))))))--(deftest build-all-removal-paths-excludes-builtin-transform-tags-test- (testing "build-all-removal-paths respects :conditions and excludes built-in tags"- (mt/with-premium-features #{:transforms-basic}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (mt/with-temp [:model/TransformTag {custom-tag-id :id} {:name "Custom Tag"- :built_in_type nil}- :model/TransformTag {builtin-tag-id :id} {:name "Built-in Tag"- :built_in_type "target"}]- (is (t2/exists? :model/TransformTag :id custom-tag-id))- (is (t2/exists? :model/TransformTag :id builtin-tag-id))- (settings/sync-transform-tracking! true)- (settings/sync-transform-tracking! false)- (is (t2/exists? :model/RemoteSyncObject- :model_type "Collection"- :model_id settings/transforms-root-id- :status "delete")- "Sentinel RSO should exist with 'delete' status")- (let [paths (spec/build-all-removal-paths)]- (is (some #(str/includes? % "custom_tag") paths)- "Custom tag (built_in_type nil) should be in removal paths")- (is (not (some #(str/includes? % "built_in_tag") paths))- "Built-in tag should NOT be in removal paths")))))))-(deftest export-excludes-builtin-transform-tags-test(testing "Export excludes built-in transform tags based on :conditions"(mt/with-premium-features #{:transforms-basic}@@ -944,21 +908,3 @@ serdes/meta:"Built-in PythonLibrary should be in export roots")(is (contains? exported-ids (:id custom-lib))"Custom PythonLibrary should be in export roots")))))))--(deftest build-all-removal-paths-excludes-builtin-python-library-test- (testing "build-all-removal-paths excludes built-in PythonLibrary via :removal-conditions"- (mt/with-premium-features #{:transforms}- (mt/with-temporary-setting-values [remote-sync-transforms true- remote-sync-enabled true]- (mt/with-temp [:model/PythonLibrary _ {:path "common.py"- :source "# builtin"- :entity_id transforms-python/builtin-entity-id}- :model/PythonLibrary _ {:path "custom.py"- :source "# custom"}]- (settings/sync-transform-tracking! true)- (settings/sync-transform-tracking! false)- (let [paths (spec/build-all-removal-paths)]- (is (some #(str/includes? % "custom.py") paths)- "Custom PythonLibrary should be in removal paths")- (is (not (some #(str/includes? % "common.py") paths))- "Built-in PythonLibrary should NOT be in removal paths")))))))