Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Clean up remote sync source vs. snapshot APIs (#65122)

Nathan Voxland954857c036ad
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 3e3083a4bca..e3ed7e7ba2b 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -7,7 +7,6 @@
[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.settings :as settings]
[metabase-enterprise.remote-sync.source :as source]
- [metabase-enterprise.remote-sync.source.git :as git]
[metabase-enterprise.remote-sync.source.ingestable :as source.ingestable]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.serialization.core :as serialization]
@@ -20,7 +19,8 @@
[metabase.util :as u]
[metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
(defn- all-top-level-remote-synced-collections
"Returns a vector of primary keys for all top-level remote-synced collections."
@@ -103,7 +103,7 @@
If the exception indicates cancellation (has :cancelled? in ex-data), logs an info message and returns nil.
Otherwise logs the error, increments the failed imports analytics metric, and returns a map with :status :error,
a user-friendly :message, the source :version, and error :details."
- [e source]
+ [e snapshot]
(if (:cancelled? (ex-data e))
(log/info "Import from git repository was cancelled")
(do
@@ -111,38 +111,38 @@
(analytics/inc! :metabase-remote-sync/imports-failed)
{:status :error
:message (source-error-message e)
- :version (source.p/version source)
+ :version (source.p/version snapshot)
:details {:error-type (type e)}})))
(defn import!
- "Imports and reloads Metabase entities from a remote source.
+ "Imports and reloads Metabase entities from a remote snapshot.
- Takes a Source instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:
- - :force? - forces import even when the source version matches the last imported version
+ Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:
+ - :force? - forces import even when the snapshot version matches the last imported version
Loads serialized entities, removes entities not in the import, syncs the remote-sync-object table, and
optionally creates a remote-synced collection.
Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may be
thrown during import and are caught and converted to error status maps."
- [source task-id & {:keys [force?]}]
+ [^SourceSnapshot snapshot task-id & {:keys [force?]}]
(log/info "Reloading remote entities from the remote source")
(analytics/inc! :metabase-remote-sync/imports)
(let [sync-timestamp (t/instant)]
- (if source
+ (if snapshot
(try
- (let [source-version (source.p/version source)
+ (let [snapshot-version (source.p/version snapshot)
last-imported-version (remote-sync.task/last-version)]
- (if (and (not force?) (= last-imported-version source-version))
+ (if (and (not force?) (= last-imported-version snapshot-version))
(u/prog1 {:status :success
- :version (source.p/version source)
- :message (format "Skipping import: source version %s matches last imported version" source-version)}
+ :version (source.p/version snapshot)
+ :message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}
(log/infof (:message <>)))
- (let [ingestable-source (->> (source.p/->ingestable source {:path-filters [#"collections/.*"]})
- (source.ingestable/wrap-progress-ingestable task-id 0.7))
+ (let [ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters [#"collections/.*"]})
+ (source.ingestable/wrap-progress-ingestable task-id 0.7))
load-result (serdes/with-cache
- (serialization/load-metabase! ingestable-source))
+ (serialization/load-metabase! ingestable-snapshot))
imported-entities-by-model (->> (:seen load-result)
(map last) ; Get the last element of each path (the entity itself)
(group-by :model)
@@ -158,29 +158,29 @@
(remote-sync.task/update-progress! task-id 0.95)
(remote-sync.task/set-version!
task-id
- (source.p/version source))
+ (source.p/version snapshot))
(log/info "Successfully reloaded entities from git repository")
{:status :success
- :version (source.p/version source)
+ :version (source.p/version snapshot)
:message "Successfully reloaded from git repository"})))
(catch Exception e
- (handle-import-exception e source))
+ (handle-import-exception e snapshot))
(finally
(analytics/observe! :metabase-remote-sync/import-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))
{:status :error
:message "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."})))
(defn export!
- "Exports remote-synced collections to a remote source repository. Assumes source has been fetched
+ "Exports remote-synced collections to a remote source repository.
- Takes a Source instance, a RemoteSyncTask ID for progress tracking, and a commit message string. Extracts all
+ Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and a commit message string. Extracts all
remote-synced collections, serializes their content, writes the files to the source, and updates all
RemoteSyncObject statuses to 'synced'.
Returns a map with :status (either :success or :error), :version, and optionally :message keys. Various
exceptions may be thrown during export and are caught and converted to error status maps."
- [source task-id message]
- (if source
+ [^SourceSnapshot snapshot task-id message]
+ (if snapshot
(let [sync-timestamp (t/instant)
collections (t2/select-fn-set :entity_id :model/Collection :type "remote-synced" :location "/")]
(if (empty? collections)
@@ -199,11 +199,11 @@
:continue-on-error false
:skip-archived true})]
(remote-sync.task/update-progress! task-id 0.3)
- (source/store! models source task-id message)
- (remote-sync.task/set-version! task-id (source.p/version source))
+ (let [written-version (source/store! models snapshot task-id message)]
+ (remote-sync.task/set-version! task-id written-version))
(t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})))
{:status :success
- :version (source.p/version source)}
+ :version (source.p/version snapshot)}
(catch Exception e
(if (:cancelled? (ex-data e))
(log/info "Export to git repository was cancelled")
@@ -211,7 +211,7 @@
(analytics/inc! :metabase-remote-sync/exports-failed)
(remote-sync.task/fail-sync-task! task-id (ex-message e))
{:status :error
- :version (source.p/version source)
+ :version (source.p/version snapshot)
:message (format "Failed to export to git repository: %s" (ex-message e))})))
(finally
(analytics/observe! :metabase-remote-sync/export-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))))
@@ -279,8 +279,7 @@
(throw (ex-info "There are unsaved changes in the Remote Sync collection which will be overwritten by the import. Force the import to discard these changes."
{:status-code 400
:conflicts true})))
- (git/fetch! source)
- (run-async! "import" branch (fn [task-id] (import! source task-id (assoc import-args :force? force?))))))
+ (run-async! "import" branch (fn [task-id] (import! (source.p/snapshot source) task-id (assoc import-args :force? force?))))))
(defn async-export!
"Exports the remote-synced collections to the remote source repository asynchronously.
@@ -294,13 +293,13 @@
[branch force? message]
(let [source (source/source-from-settings branch)
last-task-version (remote-sync.task/last-version)
- _ (git/fetch! source)
- current-source-version (source.p/version source)]
+ snapshot (source.p/snapshot source)
+ current-source-version (source.p/version snapshot)]
(when (and (not force?) (some? last-task-version) (not= last-task-version current-source-version))
(throw (ex-info "Cannot export changes that will overwrite new changes in the branch."
{:status-code 400
:conflicts true})))
- (run-async! "export" branch (fn [task-id] (export! source task-id message)))))
+ (run-async! "export" branch (fn [task-id] (export! snapshot task-id message)))))
(defn finish-remote-config!
"Based on the current configuration, fill in any missing settings and finalize remote sync setup.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 36b44af860e..5394538eb9a 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -12,57 +12,40 @@
[metabase.util.yaml :as yaml]
[methodical.core :as methodical])
(:import
- (java.io File)
- (metabase_enterprise.remote_sync.source.git GitSource)))
+ (java.io File)))
(set! *warn-on-reflection* true)
-;; Wrapping source accepts a list of path regexes to apply to paths in the source returning
-;; nil when they do no match
-(defrecord WrappingSource [original-source path-filters]
- source.p/Source
- (create-branch [_ branch base]
- (source.p/create-branch original-source branch base))
-
- (branches [_]
- (source.p/branches original-source))
-
- (default-branch [_]
- (source.p/default-branch original-source))
+;; Wrapping snapshot accepts a list of path regexes to apply to paths in the source returning
+;; nil when they do not match
+(defrecord WrappingSnapshot [original-snapshot path-filters]
+ source.p/SourceSnapshot
(list-files [_]
(filter (fn [file-path]
(some (fn [path-filter] (re-matches path-filter file-path)) path-filters))
- (source.p/list-files original-source)))
+ (source.p/list-files original-snapshot)))
(read-file [_ path]
(when (some (fn [path-filter] (re-matches path-filter path)) path-filters)
- (source.p/read-file original-source path)))
+ (source.p/read-file original-snapshot path)))
(write-files! [_ message files]
- (source.p/write-files! original-source message
+ (source.p/write-files! original-snapshot message
(filter (fn [file-spec]
(some (fn [path-filter] (re-matches path-filter (:path file-spec))) path-filters))
files)))
(version [_]
- (source.p/version original-source)))
+ (source.p/version original-snapshot)))
(methodical/defmethod source.p/->ingestable :default
- [source {:keys [path-filters root-dependencies]}]
- (cond->> (ingestable/->IngestableSource (cond-> source
- (seq path-filters) (->WrappingSource path-filters))
- (atom nil))
+ [snapshot {:keys [path-filters root-dependencies]}]
+ (cond->> (ingestable/->IngestableSnapshot (cond-> snapshot
+ (seq path-filters) (->WrappingSnapshot path-filters))
+ (atom nil))
(seq root-dependencies) (ingestable/wrap-root-dep-ingestable root-dependencies)))
-(methodical/defmethod source.p/->ingestable GitSource
- [{:keys [remote-url] :as source} opts]
- (git/fetch! source)
- (if-let [commit-ref (git/commit-sha source)]
- (next-method (assoc source :commit-ish commit-ref) opts)
- (throw (ex-info (str "Unable to find branch " (:commit-ish source) " to read from") {:url remote-url
- :branch (:commit-ish source)}))))
-
(defn- remote-sync-path
[opts entity]
(let [base-path (serdes/storage-path entity opts)
@@ -83,18 +66,18 @@
(defn store!
"Stores serialized entities from a stream to a remote source and commits the changes.
- Takes a stream (a sequence of serialized entities to be stored), a source (the remote source implementing the
- Source protocol where files will be written), a task-id (the RemoteSyncTask identifier used to track progress
+ Takes a stream (a sequence of serialized entities to be stored), a snapshot (the remote source implementing the
+ SourceSnapshot protocol where files will be written), a task-id (the RemoteSyncTask identifier used to track progress
updates), and a message (the commit message to use when writing files to the source).
- Returns the result of calling write-files! on the source with the serialized files.
+ Returns the version written to the source.
Throws Exception if any entity in the stream is an Exception instance."
- [stream source task-id message]
+ [stream snapshot task-id message]
(let [opts (serdes/storage-base-context)
;; Bound the count of the items in the stream we don't accidentally realize the entire list into memory
stream-count (bounded-count 10000 stream)]
- (source.p/write-files! source message (map-indexed #(->file-spec task-id stream-count opts %1 %2) stream))))
+ (source.p/write-files! snapshot message (map-indexed #(->file-spec task-id stream-count opts %1 %2) stream))))
(defn source-from-settings
"Creates a git source from the current remote sync settings.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index b50a3716fca..db6f184dd65 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -116,23 +116,20 @@
second commit-ish argument which overrides the :commit-ish from the source map.
Returns the full commit SHA string, or nil if the commit-ish cannot be resolved."
- ([{:keys [^String commit-ish] :as source}]
- (commit-sha source commit-ish))
-
- ([{:keys [^Git git]} ^String commit-ish]
- (when-let [ref (.resolve (.getRepository git) commit-ish)]
- (.name ref))))
+ [{:keys [^Git git]} ^String commit-ish]
+ (when-let [ref (.resolve (.getRepository git) commit-ish)]
+ (.name ref)))
(defn log
"Retrieves the commit history log for a branch.
- Takes a source map containing a :git Git instance and :commit-ish to retrieve logs from.
+ Takes a source OR snapshot to retrieve logs from.
Returns a sequence of commit maps, each containing :message (full commit message), :author-name (commit author's
name), :author-email (commit author's email), :id (abbreviated commit SHA, 8 characters), and :parent
(abbreviated parent commit SHA, 8 characters, or nil if no parent)."
- [{:keys [^Git git] :as source}]
- (when-let [ref (commit-sha source)]
+ [{:keys [^Git git branch version] :as source-or-snapshot}]
+ (when-let [ref (commit-sha source-or-snapshot (or version branch))]
(when-let [branch-id (.resolve (.getRepository git) ref)]
(let [log-result (call-command (-> (.log git)
(.add branch-id)))]
@@ -143,49 +140,47 @@
:parent (when (< 0 (.getParentCount commit)) (.name (.abbreviate (.getParent commit 0) 8)))}) log-result)))))
(defn list-files
- "Lists all files in the git repository at the current commit.
+ "Lists all files in the git repository at the snapshot.
- Takes a source map containing a :git Git instance and :commit-ish specifying which commit to list files from.
+ Takes a GitSnapshot containing a :git Git instance and :version specifying which commit to list files from.
Returns a sorted sequence of relative file path strings from the repository root."
- [{:keys [^Git git] :as source}]
- (when-let [ref (commit-sha source)]
- (let [repo (.getRepository git)
- rev-walk (RevWalk. repo)
- commit-id (.resolve repo ref)
- commit (.parseCommit rev-walk commit-id)
- tree-walk (TreeWalk. repo)]
- (.addTree tree-walk (.getTree commit))
- (.setRecursive tree-walk true)
- (sort (loop [files []]
- (if (.next tree-walk)
- (recur (conj files (.getPathString tree-walk)))
- files))))))
+ [{:keys [^Git git ^String version]}]
+ (let [repo (.getRepository git)
+ rev-walk (RevWalk. repo)
+ commit-id (.resolve repo version)
+ commit (.parseCommit rev-walk commit-id)
+ tree-walk (TreeWalk. repo)]
+ (.addTree tree-walk (.getTree commit))
+ (.setRecursive tree-walk true)
+ (sort (loop [files []]
+ (if (.next tree-walk)
+ (recur (conj files (.getPathString tree-walk)))
+ files)))))
(defn read-file
- "Reads the contents of a specific file from the git repository.
+ "Reads the contents of a specific file from the git snapshot.
- Takes a source map containing a :git Git instance and :commit-ish specifying which commit to read from, and a path
+ Takes a GitSnapshot containing a :git Git instance and :version specifying which commit to read from, and a path
string indicating the relative path to the file from the repository root.
Returns the file contents as a UTF-8 string, or nil if the file does not exist at the specified path."
- [{:keys [^Git git] :as source} ^String path]
+ [{:keys [^Git git ^String version]} ^String path]
(let [repo (.getRepository git)]
- (when-let [ref (commit-sha source)]
- (when-let [object-id (.resolve repo (str ref ":" path))]
- (let [loader (.open repo object-id)]
- (String. (.getBytes loader) "UTF-8"))))))
+ (when-let [object-id (.resolve repo (str version ":" path))]
+ (let [loader (.open repo object-id)]
+ (String. (.getBytes loader) "UTF-8")))))
(defn push-branch!
"Pushes a local branch to the remote repository.
- Takes a git-source map containing a :git Git instance, :commit-ish (branch name), and optional :token for
+ Takes a git-source map containing a :git Git instance, :branch, and optional :token for
authentication.
Returns the push response from JGit. Throws ExceptionInfo if the push operation fails or returns a
non-OK/UP_TO_DATE status."
- [{:keys [^Git git ^String commit-ish] :as git-source}]
- (let [branch-name (qualify-branch commit-ish)
+ [{:keys [^Git git ^String branch] :as git-source}]
+ (let [branch-name (qualify-branch branch)
push-response (call-remote-command
(-> (.push git)
(.setRemote "origin")
@@ -232,19 +227,17 @@
(defn write-files!
"Writes multiple files to the git repository and commits the changes.
- Takes a source map containing a :git Git instance and :commit-ish (target branch name), a commit message string,
+ Takes a snapshot map containing a :git Git instance and :version, a commit message string,
and a sequence of file maps each with :path and :content keys (paths should be relative to the repository root).
Replaces all files in the branch organized by collection prefix - files not in the provided list but in the same
collection prefix will be deleted.
- Returns the result of pushing the branch to the remote repository. Throws ExceptionInfo if the write or push
+ Returns the version written. Throws ExceptionInfo if the write or push
operation fails."
- [{:keys [^Git git ^String commit-ish] :as source} ^String message files]
- (fetch! source)
+ [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message files]
(let [repo (.getRepository git)
- branch-ref (qualify-branch commit-ish)
- parent-id (or (.resolve repo branch-ref)
- (.resolve repo (qualify-branch (default-branch source))))]
+ branch-ref (qualify-branch branch)
+ parent-id (.resolve repo version)]
(with-open [inserter (.newObjectInserter repo)]
(let [index (DirCache/newInCore)
@@ -287,8 +280,9 @@
(.flush inserter)
(doto (.updateRef repo branch-ref)
(.setNewObjectId commit-id)
- (.update))))))
- (push-branch! source)))
+ (.update))
+ (push-branch! snapshot)
+ (.name commit-id)))))))
(defn branches
"Retrieves all branch names from the remote repository.
@@ -321,7 +315,8 @@
(when (not (remote-branches branch-name))
branch-name)))
local-refs)]
- (log/info "Deleting local branches without remote:" {:branches branches-to-delete})
+ (when (seq branches-to-delete)
+ (log/info "Deleting local branches without remote:" {:branches branches-to-delete}))
(doseq [branch-name branches-to-delete]
(call-command (-> (.branchDelete git)
(.setBranchNames ^"[Ljava.lang.String;" (into-array String [branch-name]))
@@ -333,7 +328,7 @@
"Creates a new branch in the git repository from an existing branch.
Takes a source map containing a :git Git instance and optional :token for authentication, a branch-name string for
- the new branch, and a base-commit-ish string (branch name, tag, or SHA) to use as the base for the new branch.
+ the new branch, and a base-branch to use as the base for the new branch.
Returns the name of the newly created branch.
@@ -345,26 +340,19 @@
new-branch-ref (qualify-branch branch-name)
base-commit-id (.resolve repo base-commit-ish)]
(when-not base-commit-id
- (throw (ex-info (format "Base branch '%s' not found" base-commit-ish)
- {:base-branch base-commit-ish})))
+ (throw (ex-info (format "Branch base '%s' not found" base-commit-ish)
+ {:base-commit-ish base-commit-ish})))
(when (.resolve repo new-branch-ref)
(throw (ex-info (format "Branch '%s' already exists" branch-name)
{:branch branch-name})))
(doto (.updateRef repo new-branch-ref)
(.setNewObjectId base-commit-id)
(.update))
- (push-branch! (assoc source :commit-ish branch-name))
+ (push-branch! (assoc source :branch branch-name))
branch-name))
-(defrecord GitSource [git remote-url commit-ish token]
- source.p/Source
- (branches [source] (branches source))
-
- (create-branch [source branch-name base-branch]
- (create-branch source branch-name base-branch))
-
- (default-branch [this]
- (default-branch this))
+(defrecord GitSnapshot [git remote-url branch version token]
+ source.p/SourceSnapshot
(list-files [this]
(list-files this))
@@ -376,7 +364,28 @@
(write-files! this message files))
(version [this]
- (commit-sha this)))
+ (:version this)))
+
+(defn- snapshot
+ [source]
+ (fetch! source)
+ (let [version (commit-sha source (:branch source))]
+ (if version
+ (->GitSnapshot (:git source) (:remote-url source) (:branch source) version (:token source))
+ (throw (ex-info (str "Invalid branch: " (:branch source)) {})))))
+
+(defrecord GitSource [git remote-url branch token]
+ source.p/Source
+ (branches [source] (branches source))
+
+ (create-branch [source branch-name base-commit-ish]
+ (create-branch source branch-name base-commit-ish))
+
+ (default-branch [this]
+ (default-branch this))
+
+ (snapshot [this]
+ (snapshot this)))
(def ^:private jgit (atom {}))
@@ -393,10 +402,10 @@
(defn git-source
"Creates a new GitSource instance for a git repository.
- Takes a URL string (the git repository URL), a commit-ish string (branch name, tag, or commit SHA to use), and an
+ Takes a URL string (the git repository URL), a branch, and an
optional token string (authentication token for private repositories).
Returns a GitSource record implementing the Source protocol."
- [url commit-ish token]
+ [url branch token]
(->GitSource (get-jgit (repo-path {:url url :token token}) {:url url :token token})
- url commit-ish token))
+ url branch token))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
index f64f07e2d34..b377b82f4f8 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
@@ -9,19 +9,20 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.yaml :as yaml]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
(defn- ingest-content
[file-content]
(serialization/read-timestamps (yaml/parse-string file-content {:key-fn serialization/parse-key})))
(defn- ingest-all
- [source]
- (into {} (for [path (source.p/list-files source)
+ [snapshot]
+ (into {} (for [path (source.p/list-files snapshot)
:when (and (not (str/starts-with? path "."))
(str/ends-with? path ".yaml"))
:let [content (try
- (source.p/read-file source path)
+ (source.p/read-file snapshot path)
(catch Exception e
(log/error e "Error reading file" path)))
loaded (try
@@ -95,16 +96,15 @@
[root-dependencies ingestable]
(->RootDependencyIngestable ingestable root-dependencies (atom {})))
-;; Wraps a source object providing the ingestable interface for serdes
-(defrecord IngestableSource [source cache]
+;; Wraps a snapshot object providing the ingestable interface for serdes
+(defrecord IngestableSnapshot [^SourceSnapshot snapshot cache]
serialization/Ingestable
(ingest-list [_]
- (keys (or @cache (reset! cache (ingest-all source)))))
+ (keys (or @cache (reset! cache (ingest-all snapshot)))))
(ingest-one [_ serdes-path]
- source
- (when-not cache
- (reset! cache (ingest-all source)))
+ (when-not @cache
+ (reset! cache (ingest-all snapshot)))
(when-let [target (get @cache (serialization/strip-labels serdes-path))]
(try
(ingest-content (second target))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index ca3220d9444..ad360e4e11b 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -10,11 +10,11 @@
Returns a collection of branch name strings available in the source.")
- (create-branch [source branch-name base-branch]
+ (create-branch [source branch-name base-commit-ish]
"Creates a new branch from an existing branch.
Takes a source instance implementing this protocol, a branch-name (the name for the new branch to create), and
- a base-branch (the name of the existing branch to use as the base).
+ a base-commit-ish (the name of the existing branch or commit hash to use as the base).
Returns the name of the newly created branch.")
@@ -25,44 +25,52 @@
Returns the default branch name as a string, or nil if no default branch is found.")
- (list-files [source]
- "Lists all files available in the source.
+ (snapshot [source]
+ "Creates a snapshot from the source for consistent reads and writes.
- Takes a source instance implementing this protocol.
+ Takes a source instance implementing this protocol.
+
+ Returns a SourceSnapshot instance representing a point-in-time view of the source."))
+
+(defprotocol SourceSnapshot
+ (list-files [snapshot]
+ "Lists all files available in the snapshot.
+
+ Takes a SourceSnapshot instance implementing this protocol.
Returns a collection of file path strings.")
- (read-file [source path]
- "Reads the contents of a file from the source.
+ (read-file [snapshot path]
+ "Reads the contents of a file from the snapshot.
- Takes a source instance implementing this protocol and a path (the relative path to the file to read).
+ Takes a SourceSnapshot instance implementing this protocol and a path (the relative path to the file to read).
Returns the file contents as a string, or nil if the file doesn't exist.")
- (write-files! [source message files]
+ (write-files! [snapshot message files]
"Writes multiple files to the source with a commit message.
- Takes a source instance implementing this protocol, a message (the commit message to use when writing files),
+ Takes a SourceSnapshot instance implementing this protocol, a message (the commit message to use when writing files),
and files (a sequence of file specs, which are maps with :path and :content keys).
- Returns the result of the write operation.")
+ Returns the version of the written files.")
- (version [source]
- "Gets a version identifier for the current state of the source.
+ (version [snapshot]
+ "Gets a version identifier for the current state of the snapshot.
- Takes a source instance implementing this protocol.
+ Takes a SourceSnapshot instance implementing this protocol.
Returns a version identifier string (e.g., a git SHA)."))
(methodical/defmulti ->ingestable
- "Creates an ingestable source for remote sync operations.
+ "Creates an ingestable snapshot for remote sync operations.
- Takes a source (the source configuration for remote sync) and an options map containing:
+ Takes a source snapshot and an options map containing:
- :root-dependencies - a sequence of serdes dependencies in the format [{:model MODEL_NAME :id ENTITY_ID}] that
filters the items returned to only ones with these dependencies
- :path-filters - a sequence of regexes that filter allowed paths to read
- :task-id - a RemoteSyncTask identifier used to update progress
- Returns an IngestableSource instance with the provided source and empty atom state."
- {:arglists '([source opts])}
- (fn [source _opts] (type source)))
+ Returns an IngestableSnapshot instance with the provided source and empty atom state."
+ {:arglists '([snapshot opts])}
+ (fn [snapshot _opts] (type snapshot)))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj
index d16815c7a40..e76e0035adc 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj
@@ -9,7 +9,6 @@
[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.settings :as settings]
[metabase-enterprise.remote-sync.source :as source]
- [metabase-enterprise.remote-sync.source.git :as git]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase.task-history.core :as task-history]
[metabase.task.core :as task]
@@ -24,11 +23,11 @@
(settings/remote-sync-auto-import))
(let [branch (settings/remote-sync-branch)
source (source/source-from-settings branch)
- _ (git/fetch! source)
- source-version (source.p/version source)
+ snapshot (source.p/snapshot source)
+ snapshot-version (source.p/version snapshot)
last-version (remote-sync.task/last-version)]
- (if (= last-version source-version)
- (log/infof "Skipping auto-import: source version %s matches last imported version" source-version)
+ (if (= last-version snapshot-version)
+ (log/infof "Skipping auto-import: source version %s matches last imported version" snapshot-version)
(let [{task-id :id existing? :existing?} (impl/create-task-with-lock! "import")]
(if existing?
(log/info "Remote sync already in progress, not auto-importing")
@@ -37,7 +36,7 @@
(dh/with-timeout {:interrupt? true
:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}
(log/info "Auto-importing remote-sync collections")
- (impl/handle-task-result! (impl/import! source task-id) task-id)))))))))
+ (impl/handle-task-result! (impl/import! snapshot task-id) task-id)))))))))
(task/defjob ^{:doc "Auto-imports any remote collections."} AutoImport [_]
(auto-import!))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index b1c2c4e3929..ef94e2d69e5 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -29,10 +29,8 @@
branches))
(default-branch [_]
"main")
- (list-files [_] [])
- (read-file [_ _] "")
- (write-files! [_ _ _] nil)
- (version [_] "mock-version")))
+ (snapshot [_]
+ nil)))
(use-fixtures :once
(fixtures/initialize :db)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index e79e7bee0ef..5b3d8c2259b 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -27,7 +27,7 @@
;; import! tests
(deftest import!-with-no-source-configured-test
- (testing "import! with no source configured"
+ (testing "import! with no snapshot configured"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
result (impl/import! nil task-id)]
(is (= :error (:status result)))
@@ -37,7 +37,7 @@
(testing "import! successful without collections (imports all remote-synced)"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}]
- (let [result (impl/import! (test-helpers/create-mock-source) task-id)]
+ (let [result (impl/import! (source.p/snapshot (test-helpers/create-mock-source)) task-id)]
(is (= :success (:status result))))))))
(deftest import!-with-branch-parameter-test
@@ -45,13 +45,13 @@
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
custom-files {"custom-branch" {"collections/custom-collection.yaml"
(test-helpers/generate-collection-yaml "custom-collection-idx" "Custom Collection")}}
- result (impl/import! (test-helpers/create-mock-source :initial-files custom-files) task-id)]
+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :initial-files custom-files)) task-id)]
(is (= :success (:status result))))))
(deftest import!-handles-network-errors-test
(testing "import! handles network errors"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! (test-helpers/create-mock-source :fail-mode :network-error)
+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :network-error))
task-id)]
(is (= :error (:status result)))
(is (re-find #"Network error" (:message result))))))
@@ -59,28 +59,28 @@
(deftest import!-handles-authentication-errors-test
(testing "import! handles authentication errors"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! (test-helpers/create-mock-source :fail-mode :auth-error) task-id)]
+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :auth-error)) task-id)]
(is (= :error (:status result)))
(is (re-find #"Authentication failed" (:message result))))))
(deftest import!-handles-repository-not-found-errors-test
(testing "import! handles repository not found errors"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! (test-helpers/create-mock-source :fail-mode :repo-not-found) task-id)]
+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :repo-not-found)) task-id)]
(is (= :error (:status result)))
(is (re-find #"Repository not found" (:message result))))))
(deftest import!-handles-branch-errors-test
(testing "import! handles branch errors"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! (test-helpers/create-mock-source :fail-mode :branch-error) task-id)]
+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :branch-error)) task-id)]
(is (= :error (:status result)))
(is (re-find #"Branch error:" (:message result))))))
(deftest import!-handles-generic-errors-test
(testing "import! handles generic errors"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! (test-helpers/create-mock-source :fail-mode :list-files-error) task-id)]
+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :list-files-error)) task-id)]
(is (= :error (:status result)))
(is (re-find #"Failed to reload from git repository" (:message result))))))
@@ -90,7 +90,7 @@
(testing "import! skips import when source version matches last imported version and force? is false"
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [mock-source (test-helpers/create-mock-source)
- source-version (source.p/version mock-source)
+ source-version (source.p/version (source.p/snapshot mock-source))
task-defaults (mt/with-temp-defaults :model/RemoteSyncTask)]
(t2/insert! :model/RemoteSyncTask (merge task-defaults
{:sync_task_type "import"
@@ -100,7 +100,7 @@
(let [task-id-2 (t2/insert-returning-pk! :model/RemoteSyncTask (merge task-defaults
{:sync_task_type "import"
:initiated_by (mt/user->id :rasta)}))
- result (impl/import! mock-source task-id-2 :force? false)]
+ result (impl/import! (source.p/snapshot mock-source) task-id-2 :force? false)]
(is (= :success (:status result)))
(is (= source-version (:version result)))
(is (re-find #"Skipping import.*matches last imported version" (:message result))))))))
@@ -109,7 +109,7 @@
(testing "import! proceeds with import when source version matches last imported version but force? is true"
(mt/with-model-cleanup [:model/Collection :model/RemoteSyncTask]
(let [mock-source (test-helpers/create-mock-source)
- source-version (source.p/version mock-source)
+ source-version (source.p/version (source.p/snapshot mock-source))
coll-defaults (mt/with-temp-defaults :model/Collection)
task-defaults (mt/with-temp-defaults :model/RemoteSyncTask)]
(t2/insert! :model/Collection (merge coll-defaults
@@ -125,7 +125,7 @@
(let [task-id-2 (t2/insert-returning-pk! :model/RemoteSyncTask (merge task-defaults
{:sync_task_type "import"
:initiated_by (mt/user->id :rasta)}))
- result (impl/import! mock-source task-id-2 :force? true)]
+ result (impl/import! (source.p/snapshot mock-source) task-id-2 :force? true)]
(is (= :success (:status result)))
(is (= source-version (:version result)))
(is (= "Successfully reloaded from git repository" (:message result))))))))
@@ -147,7 +147,7 @@
;; Create a regular collection (not remote-synced) to verify it's not included
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}]
(let [mock-source (test-helpers/create-mock-source)
- result (impl/export! mock-source task-id "Test commit")]
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :error (:status result)))
(is (= "No remote-synced collections available to sync." (:message result)))))))))
@@ -157,7 +157,7 @@
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}]
(let [mock-source (test-helpers/create-mock-source)
- result (impl/export! mock-source task-id "Test commit message")]
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")]
(is (= :success (:status result)))))))))
(deftest export!-handles-store-failure-test
@@ -166,7 +166,7 @@
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}]
(let [mock-source (test-helpers/create-mock-source :fail-mode :store-error)
- result (impl/export! mock-source task-id "Test commit message")]
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")]
(is (= :error (:status result)))
(is (re-find #"Failed to export to git repository" (:message result)))))))))
@@ -176,7 +176,7 @@
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}]
(let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)
- result (impl/export! mock-source task-id "Test commit message")]
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")]
(is (= :error (:status result)))
(is (re-find #"Failed to export to git repository" (:message result)))))))))
@@ -189,7 +189,7 @@
:model/Card {card-id :id} {:name "Test Card" :collection_id coll-id :entity_id "test-card-1xxxxxxxxxx"}]
(let [mock-main (test-helpers/create-mock-source :branch "test-branch")
export-task (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
- export-result (impl/export! mock-main (:id export-task) "Test export")]
+ export-result (impl/export! (source.p/snapshot mock-main) (:id export-task) "Test export")]
(remote-sync.task/complete-sync-task! (:id export-task))
(is (= :success (:status export-result)))
@@ -204,7 +204,7 @@
(t2/delete! :model/RemoteSyncTask :id (:id export-task))
;; Then import - verify it succeeds and processes the exported files
(let [import-task (t2/with-connection [_conn (app-db/app-db) (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})])
- import-result (impl/import! mock-main (:id import-task))]
+ import-result (impl/import! (source.p/snapshot mock-main) (:id import-task))]
(remote-sync.task/complete-sync-task! (:id import-task))
(is (= :success (:status import-result)))
(is (= "Successfully reloaded from git repository" (:message import-result)))
@@ -235,7 +235,7 @@
"collections/test-collection-1xxxx-_/cards/test-card-1.yaml"
(test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card 1" "test-collection-1xxxx")}}
mock-main (test-helpers/create-mock-source :initial-files test-files :branch "test-branch")
- result (impl/import! mock-main (:id import-task))]
+ result (impl/import! (source.p/snapshot mock-main) (:id import-task))]
(is (= :success (:status result)))
;; Verify the entities still exist (real cleanup would require more complex setup)
@@ -247,7 +247,7 @@
(testing "error handling propagation through private functions"
(let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)
task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! mock-source task-id)]
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :error (:status result)))
(is (re-find #"Network error" (:message result))))))
@@ -260,7 +260,7 @@
(with-redefs [remote-sync.task/update-progress!
(fn [task-id progress]
(swap! progress-calls conj {:task-id task-id :progress progress}))]
- (let [result (impl/import! mock-source task-id)]
+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
;; Verify progress was called with expected values
(is (= 5 (count @progress-calls)))
@@ -285,7 +285,7 @@
(with-redefs [remote-sync.task/update-progress!
(fn [task-id progress]
(swap! progress-calls conj {:task-id task-id :progress progress}))]
- (let [result (impl/export! mock-source task-id "Test commit")]
+ (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
;; Verify progress was called with expected values
(is (= 4 (count @progress-calls)))
@@ -309,7 +309,7 @@
"collections/test-collection-1xxxx-_/cards/test-card-1.yaml"
(test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card" "test-collection-1xxxx")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
- result (impl/import! mock-source task-id)]
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
(let [entries (t2/select :model/RemoteSyncObject)]
(is (= 2 (count entries)))
@@ -336,7 +336,7 @@
(is (= "created" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id card-id))))
(is (= "removed" (:status (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id dash-id))))
(let [mock-source (test-helpers/create-mock-source)
- result (impl/export! mock-source task-id "Test commit message")]
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")]
(is (= :success (:status result)))
(let [entries (t2/select :model/RemoteSyncObject)]
(is (= 3 (count entries)))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index 51621a7bc4f..7df5f2ef0c3 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -135,16 +135,26 @@
;; add extra branch to remote to check it is picked up
(git-working-create-branch! _remote "branch-3")
(is (= ["branch-1" "branch-2" "branch-3" "master"] (source.p/branches source))))))
-;
+
+(deftest snapshot
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"master.txt" "File in master"
+ "subdir/path.txt" "File in subdir"}
+ :branches ["branch-1" "branch-2"])
+ master-snapshot (source.p/snapshot master)]
+ (is (= (git/commit-sha master "master") (:version master-snapshot))))))
+
(deftest list-files
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"}
:branches ["branch-1" "branch-2"])
- branch-1 (->source! "branch-1" remote)
- branch-2 (->source! "branch-2" remote)]
- (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files master)))
+ master-snap (source.p/snapshot master)
+ branch-1 (source.p/snapshot (->source! "branch-1" remote))
+ branch-2 (source.p/snapshot (->source! "branch-2" remote))]
+ (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files master-snap)))
(is (= ["file-in-branch-1.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-1)))
(is (= ["file-in-branch-2.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-2))))))
@@ -154,26 +164,22 @@
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"}
:branches ["branch-1" "branch-2"])
- branch-1 (->source! "branch-1" _remote)
- invalid-branch (->source! "invalid-branch" _remote)]
+ master-snap (source.p/snapshot master)
+ branch-1 (source.p/snapshot (->source! "branch-1" _remote))]
(testing "Reading master"
- (is (= "File in master" (source.p/read-file master "master.txt")))
- (is (= "File in subdir" (source.p/read-file master "subdir/path.txt")))
- (is (nil? (source.p/read-file master "file-in-branch-1.txt"))))
+ (is (= "File in master" (source.p/read-file master-snap "master.txt")))
+ (is (= "File in subdir" (source.p/read-file master-snap "subdir/path.txt")))
+ (is (nil? (source.p/read-file master-snap "file-in-branch-1.txt"))))
(testing "Reading branch-1"
(is (= "File in master" (source.p/read-file branch-1 "master.txt")))
(is (= "File in branch-1" (source.p/read-file branch-1 "file-in-branch-1.txt")))
- (is (nil? (source.p/read-file branch-1 "file-in-branch-2.txt"))))
-
- (testing "Reading invalid branch"
- (is (nil? (source.p/read-file invalid-branch "master.txt")))))))
+ (is (nil? (source.p/read-file branch-1 "file-in-branch-2.txt")))))))
(deftest write-files
(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")
otherdir-path (str "collections/" "o" (subs (u/generate-nano-id "b") 1) "_otherdir/")
- thirddir-path (str "collections/" "s" (subs (u/generate-nano-id "c") 1) "_thirddir/")
- branched-path (str "collections/" "b" (subs (u/generate-nano-id "d") 1) "_branched/")]
+ thirddir-path (str "collections/" "s" (subs (u/generate-nano-id "c") 1) "_thirddir/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
@@ -184,33 +190,33 @@
(str otherdir-path "path2.txt") "File 2 in otherdir"
(str thirddir-path "path.txt") "File in third dir"
(str thirddir-path "path2.txt") "File 2 in third dir"}
- :branches ["branch-1" "branch-2"])
- new-branch (->source! "new-branch" remote)]
+ :branches ["branch-1" "branch-2"])]
(testing "Files in a subdir are replaced, other subdirs and root are unchanged"
- (source.p/write-files! master "Update 1" [{:path "master.txt" :content "Updated master content"}
- {:path (str subdir-path "path.txt") :content "Updated subdir content"}
- {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}
- {:path (str thirddir-path "path.txt") :content "Updated third dir content"}
- {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])
+ (source.p/write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}
+ {:path (str subdir-path "path.txt") :content "Updated subdir content"}
+ {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}
+ {:path (str thirddir-path "path.txt") :content "Updated third dir content"}
+ {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])
(is (= ["Update 1" "Initial commit"] (map :message (git/log master))))
- (is (= [(str otherdir-path "path.txt")
- (str otherdir-path "path2.txt")
- (str subdir-path "path.txt")
- (str subdir-path "path3.txt")
- (str thirddir-path "path.txt")
- (str thirddir-path "path3.txt")
- "master.txt"
- "master2.txt"]
- (source.p/list-files master)))
+ (let [master-snap (source.p/snapshot master)]
+ (is (= [(str otherdir-path "path.txt")
+ (str otherdir-path "path2.txt")
+ (str subdir-path "path.txt")
+ (str subdir-path "path3.txt")
+ (str thirddir-path "path.txt")
+ (str thirddir-path "path3.txt")
+ "master.txt"
+ "master2.txt"]
+ (source.p/list-files master-snap)))
- (is (= "Updated master content" (source.p/read-file master "master.txt")))
- (is (= "File 2 in master" (source.p/read-file master "master2.txt")))
- (is (= "File 2 in otherdir" (source.p/read-file master (str otherdir-path "path2.txt"))))
- (is (= "Updated subdir content" (source.p/read-file master (str subdir-path "path.txt"))))
- (is (= "Updated subdir content 3" (source.p/read-file master (str subdir-path "path3.txt"))))
+ (is (= "Updated master content" (source.p/read-file master-snap "master.txt")))
+ (is (= "File 2 in master" (source.p/read-file master-snap "master2.txt")))
+ (is (= "File 2 in otherdir" (source.p/read-file master-snap (str otherdir-path "path2.txt"))))
+ (is (= "Updated subdir content" (source.p/read-file master-snap (str subdir-path "path.txt"))))
+ (is (= "Updated subdir content 3" (source.p/read-file master-snap (str subdir-path "path3.txt")))))
(testing "Check remote repo directly"
- (is (= "Updated master content" (git/read-file (assoc remote :commit-ish "master") "master.txt")))
+ (is (= "Updated master content" (git/read-file (assoc remote :version "master") "master.txt")))
(is (= [(str otherdir-path "path.txt")
(str otherdir-path "path2.txt")
(str subdir-path "path.txt")
@@ -219,11 +225,11 @@
(str thirddir-path "path3.txt")
"master.txt"
"master2.txt"]
- (git/list-files (assoc remote :commit-ish "master"))))
- (is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))))
+ (git/list-files (assoc remote :version "master"))))
+ (is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))))
(testing "If no root fils are touched, they all stay as-is"
- (source.p/write-files! master "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])
+ (source.p/write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])
(is (= [(str otherdir-path "path.txt")
(str otherdir-path "path2.txt")
(str subdir-path "path.txt")
@@ -231,56 +237,7 @@
(str thirddir-path "path.txt")
"master.txt"
"master2.txt"]
- (git/list-files (assoc remote :commit-ish "master")))))
-
- (testing "Writing a a new branch"
- (source.p/write-files! new-branch "New Branch" [{:path (str branched-path "branched-file.txt") :content "File added to branch"}
- {:path (str branched-path "branched-file2.txt") :content "File 2 added to branch"}
- {:path (str otherdir-path "path.txt") :content "Updated collections/otherdir/path in branch"}
- {:path (str otherdir-path "path3.txt") :content "Updated collections/otherdir/path in branch"}
- {:path "new-file.txt" :content "Updated file in branch"}])
-
- (is (= "File added to branch" (source.p/read-file new-branch (str branched-path "branched-file.txt"))))
- (is (= "Updated file in branch" (source.p/read-file new-branch "new-file.txt")))
- (is (= [(str branched-path "branched-file.txt")
- (str branched-path "branched-file2.txt")
- (str otherdir-path "path.txt")
- (str otherdir-path "path3.txt")
- (str subdir-path "path.txt")
- (str subdir-path "path3.txt")
- (str thirddir-path "path.txt")
- "master.txt"
- "master2.txt"
- "new-file.txt"]
- (source.p/list-files new-branch)))
- (is (= ["New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log new-branch))))
-
- (testing "Check remote repo"
- (is (= ["New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))))
-
- (testing "Updating a branch"
- (source.p/write-files! new-branch "Updating Branch" [{:path (str branched-path "branched-file.txt") :content "File updated in branch"}
- {:path (str branched-path "branched-file3.txt") :content "File 3 updated in branch"}
- {:path "another-file.txt" :content "Added in 2nd commit"}])
-
- (is (= "File updated in branch" (source.p/read-file new-branch (str branched-path "branched-file.txt"))))
- (is (= "Added in 2nd commit" (source.p/read-file new-branch "another-file.txt")))
- (is (= ["another-file.txt"
- (str branched-path "branched-file.txt")
- (str branched-path "branched-file3.txt")
- (str otherdir-path "path.txt")
- (str otherdir-path "path3.txt")
- (str subdir-path "path.txt")
- (str subdir-path "path3.txt")
- (str thirddir-path "path.txt")
- "master.txt"
- "master2.txt"
- "new-file.txt"]
- (source.p/list-files new-branch)))
- (is (= ["Updating Branch" "New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log new-branch))))
-
- (testing "Check remote repo"
- (is (= ["Updating Branch" "New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))))))))
+ (git/list-files (assoc remote :version "master")))))))))
(deftest write-special-collections
(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]
@@ -300,7 +257,7 @@
(testing "Initial clone is the same"
(is (= ["Initial commit"] (map :message (git/log master))))
- (is (= ["Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))
+ (is (= ["Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
;; Add an extra commit to remote
(git-working-add! remote "additional-file.txt" "Additional file content")
@@ -308,14 +265,10 @@
(testing "Source is behind remote"
(is (= ["Initial commit"] (map :message (git/log master))))
- (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))
-
- (is (= "File in master" (source.p/read-file master "master.txt")))
- (is (nil? (source.p/read-file master "additional-file.txt"))))
+ (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master"))))))
(testing "After fetch, source is up to date"
(git/fetch! master)
- (is (= "Additional file content" (source.p/read-file master "additional-file.txt")))
(is (= ["Added additional file" "Initial commit"] (map :message (git/log master)))))
(testing "Writing a file to source and pushing back to remote when there is new content on remote"
@@ -323,15 +276,15 @@
(git-working-add! remote "only-on-remote.txt" "Initially on remote")
(git-working-commit! remote "Only on remote")
- (source.p/write-files! master "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])
+ (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])
(testing "Remote has the new commit with just the files committed, but only version is in history"
- (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))
- (is (= ["additional-file.txt" (str subdir-path "path.txt") "initially-source.txt" "master.txt" "only-on-remote.txt"] (git/list-files (assoc remote :commit-ish "master"))))
- (is (= "Initially on source" (git/read-file (assoc remote :commit-ish "master") "initially-source.txt"))))
+ (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
+ (is (= ["additional-file.txt" (str subdir-path "path.txt") "initially-source.txt" "master.txt" "only-on-remote.txt"] (git/list-files (assoc remote :version "master"))))
+ (is (= "Initially on source" (git/read-file (assoc remote :version "master") "initially-source.txt"))))
(testing "Source has the same history"
- (is (= (map :message (git/log (assoc remote :commit-ish "master"))) (map :message (git/log master))))))
+ (is (= (map :message (git/log (assoc remote :branch "master"))) (map :message (git/log master))))))
(testing "Writing to a branch local has not seen (but remote has) adds it to the history on remote"
(git-working-checkout! remote "new-branch" true)
@@ -339,64 +292,63 @@
(git-working-add! remote "new-branch-remote.txt" "Initially on remote")
(git-working-commit! remote "New-branch on remote")
- (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))
+ (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))
(is (nil? (git/log new-branch)))
- (source.p/write-files! new-branch "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}
- {:path "new-branch-file.txt" :content "Updated on source"}])
+ (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}
+ {:path "new-branch-file.txt" :content "Updated on source"}])
- (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))))))))
+ (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))))))))
(deftest git-source-using-commit-ref
(mt/with-temp-dir [remote-dir nil]
- (let [[master remote] (init-source! "master" remote-dir
- :files {"master.txt" "File in master"
- "subdir/path.txt" "File in subdir"})
- original-ref (git/commit-sha master "master")]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"master.txt" "File in master"
+ "subdir/path.txt" "File in subdir"})
+ old-master (source.p/snapshot master)]
- (source.p/write-files! master "Update file" [{:path "master.txt" :content "Updated file in master"}
- {:path "new-file.txt" :content "New file in master"}])
- (let [old-master (->source! original-ref remote)]
- (is (= "File in master" (source.p/read-file old-master "master.txt")))
- (is (= "Updated file in master" (source.p/read-file master "master.txt")))
- (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files old-master)))))))
+ (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated file in master"}
+ {:path "new-file.txt" :content "New file in master"}])
+ (is (= "File in master" (source.p/read-file old-master "master.txt")))
+ (is (= "Updated file in master" (source.p/read-file (source.p/snapshot master) "master.txt")))
+ (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files old-master))))))
(deftest version-test
(testing "version returns the commit id for the current state"
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"})
- initial-version (source.p/version master)]
+ initial-version (source.p/version (source.p/snapshot master))]
(is (string? initial-version) "version should return a string")
(is (= 40 (count initial-version)) "version should be a full SHA-1 hash (40 characters)")
(is (= (git/commit-sha master "master") initial-version)
"version should match the commit id for the branch")
(testing "version changes after writing files"
- (source.p/write-files! master "Update file" [{:path "master.txt" :content "Updated content"}])
- (let [new-version (source.p/version master)]
+ (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated content"}])
+ (let [new-version (source.p/version (source.p/snapshot master))]
(is (not= initial-version new-version) "version should change after commit")
(is (= 40 (count new-version)) "new version should also be a full SHA-1 hash")
(is (= (git/commit-sha master "master") new-version)
"new version should match the new commit id")))
(testing "version is consistent across multiple calls"
- (let [version-1 (source.p/version master)
- version-2 (source.p/version master)]
+ (let [version-1 (source.p/version (source.p/snapshot master))
+ version-2 (source.p/version (source.p/snapshot master))]
(is (= version-1 version-2) "version should be consistent without changes")))
(testing "version differs for different branches"
(git-working-create-branch! remote "branch-1")
(let [branch-1 (->source! "branch-1" remote)
- master-version (source.p/version master)
- branch-version (source.p/version branch-1)]
+ master-version (source.p/version (source.p/snapshot master))
+ branch-version (source.p/version (source.p/snapshot branch-1))]
(is (not= master-version branch-version)
"different branches should have different versions")))
(testing "version matches specific commit ref"
(let [commit-ref (git/commit-sha master "master")
source-with-ref (->source! commit-ref remote)]
- (is (= commit-ref (source.p/version source-with-ref))
+ (is (= commit-ref (source.p/version (source.p/snapshot source-with-ref)))
"version should work with explicit commit refs")))))))
(deftest default-branch
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj
index e9cb1d3e6ba..720cf207416 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.remote-sync.source.ingestable :as ingestable]
+ [metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase-enterprise.serialization.core :as serialization]
[metabase.test.fixtures :as fixtures]
@@ -12,10 +13,10 @@
(use-fixtures :once (fixtures/initialize :db))
-(deftest ingestable-source-test
- (testing "IngestableSource wraps a source and provides Ingestable interface"
+(deftest ingestable-snapshot-test
+ (testing "IngestableSnapshot wraps a snapshot and provides Ingestable interface"
(let [mock-source (test-helpers/create-mock-source)
- ingestable (ingestable/->IngestableSource mock-source (atom nil))]
+ ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))]
(testing "ingest-list returns list of serdes paths"
(let [paths (serialization/ingest-list ingestable)]
@@ -31,7 +32,7 @@
(is (contains? entity :serdes/meta) "Should have serdes/meta key")))
(testing "cache is populated after first ingest-list call"
- (let [ingestable (ingestable/->IngestableSource mock-source (atom nil))]
+ (let [ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))]
(is (nil? @(:cache ingestable)) "Cache should be empty initially")
(serialization/ingest-list ingestable)
(is (map? @(:cache ingestable)) "Cache should be populated after ingest-list")
@@ -40,7 +41,7 @@
(deftest callback-ingestable-test
(testing "CallbackIngestable wraps an ingestable and calls callback on ingest-one"
(let [mock-source (test-helpers/create-mock-source)
- base-ingestable (ingestable/->IngestableSource mock-source (atom nil))
+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))
calls (atom [])
callback (fn [_ path] (swap! calls conj path))
wrapped (ingestable/->CallbackIngestable base-ingestable callback)]
@@ -78,7 +79,7 @@
:initiated_by (:id user)}))
task-id (:id task)
mock-source (test-helpers/create-mock-source)
- base-ingestable (ingestable/->IngestableSource mock-source (atom nil))
+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))
normalize 100]
(testing "creates a CallbackIngestable"
@@ -123,7 +124,7 @@
(deftest root-dependency-ingestable-test
(testing "RootDependencyIngestable filters items based on root dependencies"
(let [mock-source (test-helpers/create-mock-source)
- base-ingestable (ingestable/->IngestableSource mock-source (atom nil))]
+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))]
(testing "with no root dependencies, returns empty list"
(let [wrapped (ingestable/wrap-root-dep-ingestable [] base-ingestable)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
index 556581911cf..7bc3bea0a34 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
@@ -4,6 +4,21 @@
[metabase-enterprise.remote-sync.source :as source]
[metabase-enterprise.remote-sync.source.protocol :as source.p]))
+(defrecord MockSourceSnapshot [files]
+ source.p/SourceSnapshot
+ (list-files [_]
+ (keys files))
+
+ (read-file [_ path]
+ (get files path))
+
+ (write-files! [_ _message new-files]
+ (into {} (map (juxt :path :content) new-files))
+ "written-files-version")
+
+ (version [_]
+ "mock-version"))
+
(defrecord MockSource [files]
source.p/Source
(create-branch [_ _branch _base]
@@ -15,17 +30,8 @@
(default-branch [_]
"main")
- (list-files [_]
- (keys files))
-
- (read-file [_ path]
- (get files path))
-
- (write-files! [_ _message new-files]
- (into {} (map (juxt :path :content) new-files)))
-
- (version [_]
- "mock-version"))
+ (snapshot [_]
+ (->MockSourceSnapshot files)))
(deftest wrapping-source-list-files-single-filter-test
(testing "WrappingSource filters list-files based on path-filters"
@@ -34,9 +40,9 @@
"databases/db1.yaml" "db1-content"
"databases/db2.yaml" "db2-content"
"dashboards/dash1.yaml" "dash1-content"})
- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])]
+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])]
(is (= ["collections/foo.yaml" "collections/bar.yaml"]
- (source.p/list-files wrapped-source))
+ (source.p/list-files wrapped-snap))
"Should only include files matching the collections pattern"))))
(deftest wrapping-source-list-files-multiple-filters-test
@@ -46,18 +52,18 @@
"databases/db1.yaml" "db1-content"
"databases/db2.yaml" "db2-content"
"dashboards/dash1.yaml" "dash1-content"})
- wrapped-source (source/->WrappingSource mock-source [#"collections/.*" #"databases/.*"])]
+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*" #"databases/.*"])]
(is (= ["collections/foo.yaml" "collections/bar.yaml" "databases/db1.yaml" "databases/db2.yaml"]
- (source.p/list-files wrapped-source))
+ (source.p/list-files wrapped-snap))
"Should include files matching any of the patterns"))))
(deftest wrapping-source-list-files-no-match-test
(testing "WrappingSource with no matching files"
(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content"
"dashboards/dash1.yaml" "dash1-content"})
- wrapped-source (source/->WrappingSource mock-source [#"databases/.*"])]
+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"databases/.*"])]
(is (= []
- (source.p/list-files wrapped-source))
+ (source.p/list-files wrapped-snap))
"Should return empty list when no files match"))))
(deftest wrapping-source-read-file-single-filter-test
@@ -65,11 +71,11 @@
(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content"
"collections/bar.yaml" "bar-content"
"databases/db1.yaml" "db1-content"})
- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])]
+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])]
(is (= "foo-content"
- (source.p/read-file wrapped-source "collections/foo.yaml"))
+ (source.p/read-file wrapped-snap "collections/foo.yaml"))
"Should read file that matches filter")
- (is (nil? (source.p/read-file wrapped-source "databases/db1.yaml"))
+ (is (nil? (source.p/read-file wrapped-snap "databases/db1.yaml"))
"Should return nil for file that doesn't match filter"))))
(deftest wrapping-source-read-file-multiple-filters-test
@@ -77,40 +83,34 @@
(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content"
"databases/db1.yaml" "db1-content"
"dashboards/dash1.yaml" "dash1-content"})
- wrapped-source (source/->WrappingSource mock-source [#"collections/.*" #"dashboards/.*"])]
+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*" #"dashboards/.*"])]
(is (= "foo-content"
- (source.p/read-file wrapped-source "collections/foo.yaml"))
+ (source.p/read-file wrapped-snap "collections/foo.yaml"))
"Should read file matching first pattern")
(is (= "dash1-content"
- (source.p/read-file wrapped-source "dashboards/dash1.yaml"))
+ (source.p/read-file wrapped-snap "dashboards/dash1.yaml"))
"Should read file matching second pattern")
- (is (nil? (source.p/read-file wrapped-source "databases/db1.yaml"))
+ (is (nil? (source.p/read-file wrapped-snap "databases/db1.yaml"))
"Should return nil for file not matching any pattern"))))
(deftest wrapping-source-write-files-single-filter-test
(testing "WrappingSource filters write-files! based on path-filters"
(let [written-files (atom [])
- mock-source (reify source.p/Source
- (create-branch [_ _branch _base]
- nil)
- (branches [_]
- [])
- (default-branch [_]
- "main")
- (list-files [_]
- [])
- (read-file [_ _path]
- nil)
- (write-files! [_ _message files]
- (reset! written-files files)
- nil)
- (version [_]
- "mock-version"))
- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])
+ mock-snap (reify source.p/SourceSnapshot
+ (list-files [_]
+ [])
+ (read-file [_ _path]
+ nil)
+ (write-files! [_ _message files]
+ (reset! written-files files)
+ nil)
+ (version [_]
+ "mock-version"))
+ wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*"])
files-to-write [{:path "collections/foo.yaml" :content "foo-content"}
{:path "collections/bar.yaml" :content "bar-content"}
{:path "databases/db1.yaml" :content "db1-content"}]]
- (source.p/write-files! wrapped-source "test commit" files-to-write)
+ (source.p/write-files! wrapped-snap "test commit" files-to-write)
(is (= [{:path "collections/foo.yaml" :content "foo-content"}
{:path "collections/bar.yaml" :content "bar-content"}]
@written-files)
@@ -119,27 +119,21 @@
(deftest wrapping-source-write-files-multiple-filters-test
(testing "WrappingSource write-files! with multiple filters"
(let [written-files (atom [])
- mock-source (reify source.p/Source
- (create-branch [_ _branch _base]
- nil)
- (branches [_]
- [])
- (default-branch [_]
- "main")
- (list-files [_]
- [])
- (read-file [_ _path]
- nil)
- (write-files! [_ _message files]
- (reset! written-files files)
- nil)
- (version [_]
- "mock-version"))
- wrapped-source (source/->WrappingSource mock-source [#"collections/.*" #"dashboards/.*"])
+ mock-snap (reify source.p/SourceSnapshot
+ (list-files [_]
+ [])
+ (read-file [_ _path]
+ nil)
+ (write-files! [_ _message files]
+ (reset! written-files files)
+ nil)
+ (version [_]
+ "mock-version"))
+ wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*" #"dashboards/.*"])
files-to-write [{:path "collections/foo.yaml" :content "foo-content"}
{:path "databases/db1.yaml" :content "db1-content"}
{:path "dashboards/dash1.yaml" :content "dash1-content"}]]
- (source.p/write-files! wrapped-source "test commit" files-to-write)
+ (source.p/write-files! wrapped-snap "test commit" files-to-write)
(is (= [{:path "collections/foo.yaml" :content "foo-content"}
{:path "dashboards/dash1.yaml" :content "dash1-content"}]
@written-files)
@@ -148,34 +142,18 @@
(deftest wrapping-source-write-files-no-match-test
(testing "WrappingSource write-files! with no matching files"
(let [written-files (atom [])
- mock-source (reify source.p/Source
- (create-branch [_ _branch _base]
- nil)
- (branches [_]
- [])
- (default-branch [_]
- "main")
- (list-files [_]
- [])
- (read-file [_ _path]
- nil)
- (write-files! [_ _message files]
- (reset! written-files files)
- nil)
- (version [_]
- "mock-version"))
- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])
+ mock-snapshot (reify source.p/SourceSnapshot
+ (list-files [_]
+ [])
+ (read-file [_this _path]
+ nil)
+ (write-files! [_this _message _files]
+ nil)
+ (version [_this]
+ nil))
files-to-write [{:path "databases/db1.yaml" :content "db1-content"}
{:path "dashboards/dash1.yaml" :content "dash1-content"}]]
- (source.p/write-files! wrapped-source "test commit" files-to-write)
+ (source.p/write-files! mock-snapshot "test commit" files-to-write)
(is (= []
@written-files)
"Should write no files when none match"))))
-
-(deftest wrapping-source-branches-test
- (testing "WrappingSource delegates branches to original source"
- (let [mock-source (->MockSource {})
- wrapped-source (source/->WrappingSource mock-source [#".*"])]
- (is (= ["main"]
- (source.p/branches wrapped-source))
- "Should return branches from original source"))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index a481417e2c3..9b34f0150cd 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -13,17 +13,8 @@
(use-fixtures :each th/clean-remote-sync-state)
-(defrecord MockSource [written-files]
- source.p/Source
- (create-branch [_ _branch _base]
- nil)
-
- (branches [_]
- ["main"])
-
- (default-branch [_]
- "main")
-
+(defrecord MockSourceSnapshot [written-files]
+ source.p/SourceSnapshot
(list-files [_]
[])
@@ -32,11 +23,26 @@
(write-files! [_ message files]
(let [realized-files (vec files)]
- (reset! written-files {:message message :files realized-files})))
+ (reset! written-files {:message message :files realized-files}))
+ "mock-written-version")
(version [_]
"mock-version"))
+(defrecord MockSource [written-files]
+ source.p/Source
+ (create-branch [_ _branch _base]
+ nil)
+
+ (branches [_]
+ ["main"])
+
+ (default-branch [_]
+ "main")
+
+ (snapshot [_]
+ (->MockSourceSnapshot written-files)))
+
(defn- create-test-entity
"Creates a test entity with serdes/meta for testing"
[id label model-name]
@@ -48,7 +54,7 @@
(testing "store! writes files to source with correct structure"
(mt/with-temp [:model/User user {:first_name "Test"
:last_name "User"
:password "password123"}
:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"
:initiated_by (:id user)}]
@@ -57,7 +63,7 @@
test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")
(create-test-entity "test-id-2" "entity-two" "Card")]]
- (source/store! test-entities mock-source task-id "Test commit message")
+ (is (= "mock-written-version" (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message")))
(testing "write-files! was called with correct message"
(is (= "Test commit message" (:message @written-files))))
@@ -89,7 +95,7 @@
(testing "store! updates task progress as files are written"
(mt/with-temp [:model/User user {:first_name "Test"
:last_name "User"
:password "password123"}
:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"
:initiated_by (:id user)}]
@@ -102,7 +108,7 @@
(let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)]
(is (nil? (:progress initial-task)) "Progress should be nil initially"))
- (source/store! test-entities mock-source task-id "Test commit")
+ (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit")
(let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)]
(is (some? (:progress final-task)) "Progress should be updated after store!")
@@ -115,14 +121,14 @@
(testing "store! handles empty stream correctly"
(mt/with-temp [:model/User user {:first_name "Test"
:last_name "User"
:password "password123"}
:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"
:initiated_by (:id user)}]
(let [written-files (atom nil)
mock-source (->MockSource written-files)]
- (source/store! [] mock-source task-id "Empty commit")
+ (source/store! [] (source.p/snapshot mock-source) task-id "Empty commit")
(testing "write-files! was called even with empty stream"
(is (some? @written-files)))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index a9488e9b144..ae7d8e9a874 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -155,22 +155,8 @@ width: fixed
"
name entity-id collection-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_") dashcards-yaml)))
-(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom]
- source.p/Source
- (create-branch [_this branch _base]
- (swap! branches-atom conj [branch (str branch "-ref")]))
-
- (branches [_this]
- (case fail-mode
- :branches-error (throw (java.net.UnknownHostException. "Network error"))
- :auth-error (throw (Exception. "Authentication failed"))
- :repo-not-found (throw (Exception. "Repository not found"))
- ;; Default success case
- @branches-atom))
-
- (default-branch [_this]
- "main")
-
+(defrecord MockSourceSnapshot [source-id base-url branch fail-mode files-atom]
+ source.p/SourceSnapshot
(list-files [_this]
(case fail-mode
:list-files-error (throw (Exception. "Failed to list files"))
@@ -197,11 +183,31 @@ width: fixed
:store-error (throw (Exception. "Store failed"))
:network-error (throw (java.net.UnknownHostException. "Remote host not found"))
;; Default success case - write files to atom
- (swap! files-atom assoc branch (into {} (map (juxt :path :content) files)))))
+ (swap! files-atom assoc branch (into {} (map (juxt :path :content) files))))
+ "write-files-version")
(version [_this]
"mock-version"))
+(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom]
+ source.p/Source
+ (create-branch [_this branch _base]
+ (swap! branches-atom conj [branch (str branch "-ref")]))
+
+ (branches [_this]
+ (case fail-mode
+ :branches-error (throw (java.net.UnknownHostException. "Network error"))
+ :auth-error (throw (Exception. "Authentication failed"))
+ :repo-not-found (throw (Exception. "Repository not found"))
+ ;; Default success case
+ @branches-atom))
+
+ (default-branch [_this]
+ "main")
+
+ (snapshot [_this]
+ (->MockSourceSnapshot source-id base-url branch fail-mode files-atom)))
+
(defn create-mock-source
"Create a mock Source for testing"
[& {:keys [branch fail-mode initial-files]