The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Clean up remote sync source vs. snapshot APIs (#65122)
Nathan Voxland954857c036ad
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 3e3083a4bca..e3ed7e7ba2b 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -7,7 +7,6 @@[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.source :as source]- [metabase-enterprise.remote-sync.source.git :as git][metabase-enterprise.remote-sync.source.ingestable :as source.ingestable][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.serialization.core :as serialization]@@ -20,7 +19,8 @@[metabase.util :as u][metabase.util.jvm :as u.jvm][metabase.util.log :as log]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))(defn- all-top-level-remote-synced-collections"Returns a vector of primary keys for all top-level remote-synced collections."@@ -103,7 +103,7 @@If the exception indicates cancellation (has :cancelled? in ex-data), logs an info message and returns nil.Otherwise logs the error, increments the failed imports analytics metric, and returns a map with :status :error,a user-friendly :message, the source :version, and error :details."- [e source]+ [e snapshot](if (:cancelled? (ex-data e))(log/info "Import from git repository was cancelled")(do@@ -111,38 +111,38 @@(analytics/inc! :metabase-remote-sync/imports-failed){:status :error:message (source-error-message e)- :version (source.p/version source)+ :version (source.p/version snapshot):details {:error-type (type e)}})))(defn import!- "Imports and reloads Metabase entities from a remote source.+ "Imports and reloads Metabase entities from a remote snapshot.- Takes a Source instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:- - :force? - forces import even when the source version matches the last imported version+ Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:+ - :force? - forces import even when the snapshot version matches the last imported versionLoads serialized entities, removes entities not in the import, syncs the remote-sync-object table, andoptionally creates a remote-synced collection.Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may bethrown during import and are caught and converted to error status maps."- [source task-id & {:keys [force?]}]+ [^SourceSnapshot snapshot task-id & {:keys [force?]}](log/info "Reloading remote entities from the remote source")(analytics/inc! :metabase-remote-sync/imports)(let [sync-timestamp (t/instant)]- (if source+ (if snapshot(try- (let [source-version (source.p/version source)+ (let [snapshot-version (source.p/version snapshot)last-imported-version (remote-sync.task/last-version)]- (if (and (not force?) (= last-imported-version source-version))+ (if (and (not force?) (= last-imported-version snapshot-version))(u/prog1 {:status :success- :version (source.p/version source)- :message (format "Skipping import: source version %s matches last imported version" source-version)}+ :version (source.p/version snapshot)+ :message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}(log/infof (:message <>)))- (let [ingestable-source (->> (source.p/->ingestable source {:path-filters [#"collections/.*"]})- (source.ingestable/wrap-progress-ingestable task-id 0.7))+ (let [ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters [#"collections/.*"]})+ (source.ingestable/wrap-progress-ingestable task-id 0.7))load-result (serdes/with-cache- (serialization/load-metabase! ingestable-source))+ (serialization/load-metabase! ingestable-snapshot))imported-entities-by-model (->> (:seen load-result)(map last) ; Get the last element of each path (the entity itself)(group-by :model)@@ -158,29 +158,29 @@(remote-sync.task/update-progress! task-id 0.95)(remote-sync.task/set-version!task-id- (source.p/version source))+ (source.p/version snapshot))(log/info "Successfully reloaded entities from git repository"){:status :success- :version (source.p/version source)+ :version (source.p/version snapshot):message "Successfully reloaded from git repository"})))(catch Exception e- (handle-import-exception e source))+ (handle-import-exception e snapshot))(finally(analytics/observe! :metabase-remote-sync/import-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis)))){:status :error:message "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."})))(defn export!- "Exports remote-synced collections to a remote source repository. Assumes source has been fetched+ "Exports remote-synced collections to a remote source repository.- Takes a Source instance, a RemoteSyncTask ID for progress tracking, and a commit message string. Extracts all+ Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and a commit message string. Extracts allremote-synced collections, serializes their content, writes the files to the source, and updates allRemoteSyncObject statuses to 'synced'.Returns a map with :status (either :success or :error), :version, and optionally :message keys. Variousexceptions may be thrown during export and are caught and converted to error status maps."- [source task-id message]- (if source+ [^SourceSnapshot snapshot task-id message]+ (if snapshot(let [sync-timestamp (t/instant)collections (t2/select-fn-set :entity_id :model/Collection :type "remote-synced" :location "/")](if (empty? collections)@@ -199,11 +199,11 @@:continue-on-error false:skip-archived true})](remote-sync.task/update-progress! task-id 0.3)- (source/store! models source task-id message)- (remote-sync.task/set-version! task-id (source.p/version source))+ (let [written-version (source/store! models snapshot task-id message)]+ (remote-sync.task/set-version! task-id written-version))(t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp}))){:status :success- :version (source.p/version source)}+ :version (source.p/version snapshot)}(catch Exception e(if (:cancelled? (ex-data e))(log/info "Export to git repository was cancelled")@@ -211,7 +211,7 @@(analytics/inc! :metabase-remote-sync/exports-failed)(remote-sync.task/fail-sync-task! task-id (ex-message e)){:status :error- :version (source.p/version source)+ :version (source.p/version snapshot):message (format "Failed to export to git repository: %s" (ex-message e))})))(finally(analytics/observe! :metabase-remote-sync/export-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))))@@ -279,8 +279,7 @@(throw (ex-info "There are unsaved changes in the Remote Sync collection which will be overwritten by the import. Force the import to discard these changes."{:status-code 400:conflicts true})))- (git/fetch! source)- (run-async! "import" branch (fn [task-id] (import! source task-id (assoc import-args :force? force?))))))+ (run-async! "import" branch (fn [task-id] (import! (source.p/snapshot source) task-id (assoc import-args :force? force?))))))(defn async-export!"Exports the remote-synced collections to the remote source repository asynchronously.@@ -294,13 +293,13 @@[branch force? message](let [source (source/source-from-settings branch)last-task-version (remote-sync.task/last-version)- _ (git/fetch! source)- current-source-version (source.p/version source)]+ snapshot (source.p/snapshot source)+ current-source-version (source.p/version snapshot)](when (and (not force?) (some? last-task-version) (not= last-task-version current-source-version))(throw (ex-info "Cannot export changes that will overwrite new changes in the branch."{:status-code 400:conflicts true})))- (run-async! "export" branch (fn [task-id] (export! source task-id message)))))+ (run-async! "export" branch (fn [task-id] (export! snapshot task-id message)))))(defn finish-remote-config!"Based on the current configuration, fill in any missing settings and finalize remote sync setup.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex 36b44af860e..5394538eb9a 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -12,57 +12,40 @@[metabase.util.yaml :as yaml][methodical.core :as methodical])(:import- (java.io File)- (metabase_enterprise.remote_sync.source.git GitSource)))+ (java.io File)))(set! *warn-on-reflection* true)-;; Wrapping source accepts a list of path regexes to apply to paths in the source returning-;; nil when they do no match-(defrecord WrappingSource [original-source path-filters]- source.p/Source- (create-branch [_ branch base]- (source.p/create-branch original-source branch base))-- (branches [_]- (source.p/branches original-source))-- (default-branch [_]- (source.p/default-branch original-source))+;; Wrapping snapshot accepts a list of path regexes to apply to paths in the source returning+;; nil when they do not match+(defrecord WrappingSnapshot [original-snapshot path-filters]+ source.p/SourceSnapshot(list-files [_](filter (fn [file-path](some (fn [path-filter] (re-matches path-filter file-path)) path-filters))- (source.p/list-files original-source)))+ (source.p/list-files original-snapshot)))(read-file [_ path](when (some (fn [path-filter] (re-matches path-filter path)) path-filters)- (source.p/read-file original-source path)))+ (source.p/read-file original-snapshot path)))(write-files! [_ message files]- (source.p/write-files! original-source message+ (source.p/write-files! original-snapshot message(filter (fn [file-spec](some (fn [path-filter] (re-matches path-filter (:path file-spec))) path-filters))files)))(version [_]- (source.p/version original-source)))+ (source.p/version original-snapshot)))(methodical/defmethod source.p/->ingestable :default- [source {:keys [path-filters root-dependencies]}]- (cond->> (ingestable/->IngestableSource (cond-> source- (seq path-filters) (->WrappingSource path-filters))- (atom nil))+ [snapshot {:keys [path-filters root-dependencies]}]+ (cond->> (ingestable/->IngestableSnapshot (cond-> snapshot+ (seq path-filters) (->WrappingSnapshot path-filters))+ (atom nil))(seq root-dependencies) (ingestable/wrap-root-dep-ingestable root-dependencies)))-(methodical/defmethod source.p/->ingestable GitSource- [{:keys [remote-url] :as source} opts]- (git/fetch! source)- (if-let [commit-ref (git/commit-sha source)]- (next-method (assoc source :commit-ish commit-ref) opts)- (throw (ex-info (str "Unable to find branch " (:commit-ish source) " to read from") {:url remote-url- :branch (:commit-ish source)}))))-(defn- remote-sync-path[opts entity](let [base-path (serdes/storage-path entity opts)@@ -83,18 +66,18 @@(defn store!"Stores serialized entities from a stream to a remote source and commits the changes.- Takes a stream (a sequence of serialized entities to be stored), a source (the remote source implementing the- Source protocol where files will be written), a task-id (the RemoteSyncTask identifier used to track progress+ Takes a stream (a sequence of serialized entities to be stored), a snapshot (the remote source implementing the+ SourceSnapshot protocol where files will be written), a task-id (the RemoteSyncTask identifier used to track progressupdates), and a message (the commit message to use when writing files to the source).- Returns the result of calling write-files! on the source with the serialized files.+ Returns the version written to the source.Throws Exception if any entity in the stream is an Exception instance."- [stream source task-id message]+ [stream snapshot task-id message](let [opts (serdes/storage-base-context);; Bound the count of the items in the stream we don't accidentally realize the entire list into memorystream-count (bounded-count 10000 stream)]- (source.p/write-files! source message (map-indexed #(->file-spec task-id stream-count opts %1 %2) stream))))+ (source.p/write-files! snapshot message (map-indexed #(->file-spec task-id stream-count opts %1 %2) stream))))(defn source-from-settings"Creates a git source from the current remote sync settings.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex b50a3716fca..db6f184dd65 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -116,23 +116,20 @@second commit-ish argument which overrides the :commit-ish from the source map.Returns the full commit SHA string, or nil if the commit-ish cannot be resolved."- ([{:keys [^String commit-ish] :as source}]- (commit-sha source commit-ish))-- ([{:keys [^Git git]} ^String commit-ish]- (when-let [ref (.resolve (.getRepository git) commit-ish)]- (.name ref))))+ [{:keys [^Git git]} ^String commit-ish]+ (when-let [ref (.resolve (.getRepository git) commit-ish)]+ (.name ref)))(defn log"Retrieves the commit history log for a branch.- Takes a source map containing a :git Git instance and :commit-ish to retrieve logs from.+ Takes a source OR snapshot to retrieve logs from.Returns a sequence of commit maps, each containing :message (full commit message), :author-name (commit author'sname), :author-email (commit author's email), :id (abbreviated commit SHA, 8 characters), and :parent(abbreviated parent commit SHA, 8 characters, or nil if no parent)."- [{:keys [^Git git] :as source}]- (when-let [ref (commit-sha source)]+ [{:keys [^Git git branch version] :as source-or-snapshot}]+ (when-let [ref (commit-sha source-or-snapshot (or version branch))](when-let [branch-id (.resolve (.getRepository git) ref)](let [log-result (call-command (-> (.log git)(.add branch-id)))]@@ -143,49 +140,47 @@:parent (when (< 0 (.getParentCount commit)) (.name (.abbreviate (.getParent commit 0) 8)))}) log-result)))))(defn list-files- "Lists all files in the git repository at the current commit.+ "Lists all files in the git repository at the snapshot.- Takes a source map containing a :git Git instance and :commit-ish specifying which commit to list files from.+ Takes a GitSnapshot containing a :git Git instance and :version specifying which commit to list files from.Returns a sorted sequence of relative file path strings from the repository root."- [{:keys [^Git git] :as source}]- (when-let [ref (commit-sha source)]- (let [repo (.getRepository git)- rev-walk (RevWalk. repo)- commit-id (.resolve repo ref)- commit (.parseCommit rev-walk commit-id)- tree-walk (TreeWalk. repo)]- (.addTree tree-walk (.getTree commit))- (.setRecursive tree-walk true)- (sort (loop [files []]- (if (.next tree-walk)- (recur (conj files (.getPathString tree-walk)))- files))))))+ [{:keys [^Git git ^String version]}]+ (let [repo (.getRepository git)+ rev-walk (RevWalk. repo)+ commit-id (.resolve repo version)+ commit (.parseCommit rev-walk commit-id)+ tree-walk (TreeWalk. repo)]+ (.addTree tree-walk (.getTree commit))+ (.setRecursive tree-walk true)+ (sort (loop [files []]+ (if (.next tree-walk)+ (recur (conj files (.getPathString tree-walk)))+ files)))))(defn read-file- "Reads the contents of a specific file from the git repository.+ "Reads the contents of a specific file from the git snapshot.- Takes a source map containing a :git Git instance and :commit-ish specifying which commit to read from, and a path+ Takes a GitSnapshot containing a :git Git instance and :version specifying which commit to read from, and a pathstring indicating the relative path to the file from the repository root.Returns the file contents as a UTF-8 string, or nil if the file does not exist at the specified path."- [{:keys [^Git git] :as source} ^String path]+ [{:keys [^Git git ^String version]} ^String path](let [repo (.getRepository git)]- (when-let [ref (commit-sha source)]- (when-let [object-id (.resolve repo (str ref ":" path))]- (let [loader (.open repo object-id)]- (String. (.getBytes loader) "UTF-8"))))))+ (when-let [object-id (.resolve repo (str version ":" path))]+ (let [loader (.open repo object-id)]+ (String. (.getBytes loader) "UTF-8")))))(defn push-branch!"Pushes a local branch to the remote repository.- Takes a git-source map containing a :git Git instance, :commit-ish (branch name), and optional :token for+ Takes a git-source map containing a :git Git instance, :branch, and optional :token forauthentication.Returns the push response from JGit. Throws ExceptionInfo if the push operation fails or returns anon-OK/UP_TO_DATE status."- [{:keys [^Git git ^String commit-ish] :as git-source}]- (let [branch-name (qualify-branch commit-ish)+ [{:keys [^Git git ^String branch] :as git-source}]+ (let [branch-name (qualify-branch branch)push-response (call-remote-command(-> (.push git)(.setRemote "origin")@@ -232,19 +227,17 @@(defn write-files!"Writes multiple files to the git repository and commits the changes.- Takes a source map containing a :git Git instance and :commit-ish (target branch name), a commit message string,+ Takes a snapshot map containing a :git Git instance and :version, a commit message string,and a sequence of file maps each with :path and :content keys (paths should be relative to the repository root).Replaces all files in the branch organized by collection prefix - files not in the provided list but in the samecollection prefix will be deleted.- Returns the result of pushing the branch to the remote repository. Throws ExceptionInfo if the write or push+ Returns the version written. Throws ExceptionInfo if the write or pushoperation fails."- [{:keys [^Git git ^String commit-ish] :as source} ^String message files]- (fetch! source)+ [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message files](let [repo (.getRepository git)- branch-ref (qualify-branch commit-ish)- parent-id (or (.resolve repo branch-ref)- (.resolve repo (qualify-branch (default-branch source))))]+ branch-ref (qualify-branch branch)+ parent-id (.resolve repo version)](with-open [inserter (.newObjectInserter repo)](let [index (DirCache/newInCore)@@ -287,8 +280,9 @@(.flush inserter)(doto (.updateRef repo branch-ref)(.setNewObjectId commit-id)- (.update))))))- (push-branch! source)))+ (.update))+ (push-branch! snapshot)+ (.name commit-id)))))))(defn branches"Retrieves all branch names from the remote repository.@@ -321,7 +315,8 @@(when (not (remote-branches branch-name))branch-name)))local-refs)]- (log/info "Deleting local branches without remote:" {:branches branches-to-delete})+ (when (seq branches-to-delete)+ (log/info "Deleting local branches without remote:" {:branches branches-to-delete}))(doseq [branch-name branches-to-delete](call-command (-> (.branchDelete git)(.setBranchNames ^"[Ljava.lang.String;" (into-array String [branch-name]))@@ -333,7 +328,7 @@"Creates a new branch in the git repository from an existing branch.Takes a source map containing a :git Git instance and optional :token for authentication, a branch-name string for- the new branch, and a base-commit-ish string (branch name, tag, or SHA) to use as the base for the new branch.+ the new branch, and a base-branch to use as the base for the new branch.Returns the name of the newly created branch.@@ -345,26 +340,19 @@new-branch-ref (qualify-branch branch-name)base-commit-id (.resolve repo base-commit-ish)](when-not base-commit-id- (throw (ex-info (format "Base branch '%s' not found" base-commit-ish)- {:base-branch base-commit-ish})))+ (throw (ex-info (format "Branch base '%s' not found" base-commit-ish)+ {:base-commit-ish base-commit-ish})))(when (.resolve repo new-branch-ref)(throw (ex-info (format "Branch '%s' already exists" branch-name){:branch branch-name})))(doto (.updateRef repo new-branch-ref)(.setNewObjectId base-commit-id)(.update))- (push-branch! (assoc source :commit-ish branch-name))+ (push-branch! (assoc source :branch branch-name))branch-name))-(defrecord GitSource [git remote-url commit-ish token]- source.p/Source- (branches [source] (branches source))-- (create-branch [source branch-name base-branch]- (create-branch source branch-name base-branch))-- (default-branch [this]- (default-branch this))+(defrecord GitSnapshot [git remote-url branch version token]+ source.p/SourceSnapshot(list-files [this](list-files this))@@ -376,7 +364,28 @@(write-files! this message files))(version [this]- (commit-sha this)))+ (:version this)))++(defn- snapshot+ [source]+ (fetch! source)+ (let [version (commit-sha source (:branch source))]+ (if version+ (->GitSnapshot (:git source) (:remote-url source) (:branch source) version (:token source))+ (throw (ex-info (str "Invalid branch: " (:branch source)) {})))))++(defrecord GitSource [git remote-url branch token]+ source.p/Source+ (branches [source] (branches source))++ (create-branch [source branch-name base-commit-ish]+ (create-branch source branch-name base-commit-ish))++ (default-branch [this]+ (default-branch this))++ (snapshot [this]+ (snapshot this)))(def ^:private jgit (atom {}))@@ -393,10 +402,10 @@(defn git-source"Creates a new GitSource instance for a git repository.- Takes a URL string (the git repository URL), a commit-ish string (branch name, tag, or commit SHA to use), and an+ Takes a URL string (the git repository URL), a branch, and anoptional token string (authentication token for private repositories).Returns a GitSource record implementing the Source protocol."- [url commit-ish token]+ [url branch token](->GitSource (get-jgit (repo-path {:url url :token token}) {:url url :token token})- url commit-ish token))+ url branch token))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.cljindex f64f07e2d34..b377b82f4f8 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj@@ -9,19 +9,20 @@[metabase.util :as u][metabase.util.log :as log][metabase.util.yaml :as yaml]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))(defn- ingest-content[file-content](serialization/read-timestamps (yaml/parse-string file-content {:key-fn serialization/parse-key})))(defn- ingest-all- [source]- (into {} (for [path (source.p/list-files source)+ [snapshot]+ (into {} (for [path (source.p/list-files snapshot):when (and (not (str/starts-with? path "."))(str/ends-with? path ".yaml")):let [content (try- (source.p/read-file source path)+ (source.p/read-file snapshot path)(catch Exception e(log/error e "Error reading file" path)))loaded (try@@ -95,16 +96,15 @@[root-dependencies ingestable](->RootDependencyIngestable ingestable root-dependencies (atom {})))-;; Wraps a source object providing the ingestable interface for serdes-(defrecord IngestableSource [source cache]+;; Wraps a snapshot object providing the ingestable interface for serdes+(defrecord IngestableSnapshot [^SourceSnapshot snapshot cache]serialization/Ingestable(ingest-list [_]- (keys (or @cache (reset! cache (ingest-all source)))))+ (keys (or @cache (reset! cache (ingest-all snapshot)))))(ingest-one [_ serdes-path]- source- (when-not cache- (reset! cache (ingest-all source)))+ (when-not @cache+ (reset! cache (ingest-all snapshot)))(when-let [target (get @cache (serialization/strip-labels serdes-path))](try(ingest-content (second target))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.cljindex ca3220d9444..ad360e4e11b 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj@@ -10,11 +10,11 @@Returns a collection of branch name strings available in the source.")- (create-branch [source branch-name base-branch]+ (create-branch [source branch-name base-commit-ish]"Creates a new branch from an existing branch.Takes a source instance implementing this protocol, a branch-name (the name for the new branch to create), and- a base-branch (the name of the existing branch to use as the base).+ a base-commit-ish (the name of the existing branch or commit hash to use as the base).Returns the name of the newly created branch.")@@ -25,44 +25,52 @@Returns the default branch name as a string, or nil if no default branch is found.")- (list-files [source]- "Lists all files available in the source.+ (snapshot [source]+ "Creates a snapshot from the source for consistent reads and writes.- Takes a source instance implementing this protocol.+ Takes a source instance implementing this protocol.++ Returns a SourceSnapshot instance representing a point-in-time view of the source."))++(defprotocol SourceSnapshot+ (list-files [snapshot]+ "Lists all files available in the snapshot.++ Takes a SourceSnapshot instance implementing this protocol.Returns a collection of file path strings.")- (read-file [source path]- "Reads the contents of a file from the source.+ (read-file [snapshot path]+ "Reads the contents of a file from the snapshot.- Takes a source instance implementing this protocol and a path (the relative path to the file to read).+ Takes a SourceSnapshot instance implementing this protocol and a path (the relative path to the file to read).Returns the file contents as a string, or nil if the file doesn't exist.")- (write-files! [source message files]+ (write-files! [snapshot message files]"Writes multiple files to the source with a commit message.- Takes a source instance implementing this protocol, a message (the commit message to use when writing files),+ Takes a SourceSnapshot instance implementing this protocol, a message (the commit message to use when writing files),and files (a sequence of file specs, which are maps with :path and :content keys).- Returns the result of the write operation.")+ Returns the version of the written files.")- (version [source]- "Gets a version identifier for the current state of the source.+ (version [snapshot]+ "Gets a version identifier for the current state of the snapshot.- Takes a source instance implementing this protocol.+ Takes a SourceSnapshot instance implementing this protocol.Returns a version identifier string (e.g., a git SHA)."))(methodical/defmulti ->ingestable- "Creates an ingestable source for remote sync operations.+ "Creates an ingestable snapshot for remote sync operations.- Takes a source (the source configuration for remote sync) and an options map containing:+ Takes a source snapshot and an options map containing:- :root-dependencies - a sequence of serdes dependencies in the format [{:model MODEL_NAME :id ENTITY_ID}] thatfilters the items returned to only ones with these dependencies- :path-filters - a sequence of regexes that filter allowed paths to read- :task-id - a RemoteSyncTask identifier used to update progress- Returns an IngestableSource instance with the provided source and empty atom state."- {:arglists '([source opts])}- (fn [source _opts] (type source)))+ Returns an IngestableSnapshot instance with the provided source and empty atom state."+ {:arglists '([snapshot opts])}+ (fn [snapshot _opts] (type snapshot)))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.cljindex d16815c7a40..e76e0035adc 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj@@ -9,7 +9,6 @@[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.source :as source]- [metabase-enterprise.remote-sync.source.git :as git][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase.task-history.core :as task-history][metabase.task.core :as task]@@ -24,11 +23,11 @@(settings/remote-sync-auto-import))(let [branch (settings/remote-sync-branch)source (source/source-from-settings branch)- _ (git/fetch! source)- source-version (source.p/version source)+ snapshot (source.p/snapshot source)+ snapshot-version (source.p/version snapshot)last-version (remote-sync.task/last-version)]- (if (= last-version source-version)- (log/infof "Skipping auto-import: source version %s matches last imported version" source-version)+ (if (= last-version snapshot-version)+ (log/infof "Skipping auto-import: source version %s matches last imported version" snapshot-version)(let [{task-id :id existing? :existing?} (impl/create-task-with-lock! "import")](if existing?(log/info "Remote sync already in progress, not auto-importing")@@ -37,7 +36,7 @@(dh/with-timeout {:interrupt? true:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}(log/info "Auto-importing remote-sync collections")- (impl/handle-task-result! (impl/import! source task-id) task-id)))))))))+ (impl/handle-task-result! (impl/import! snapshot task-id) task-id)))))))))(task/defjob ^{:doc "Auto-imports any remote collections."} AutoImport [_](auto-import!))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex b1c2c4e3929..ef94e2d69e5 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -29,10 +29,8 @@branches))(default-branch [_]"main")- (list-files [_] [])- (read-file [_ _] "")- (write-files! [_ _ _] nil)- (version [_] "mock-version")))+ (snapshot [_]+ nil)))(use-fixtures :once(fixtures/initialize :db)diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex e79e7bee0ef..5b3d8c2259b 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -27,7 +27,7 @@;; import! tests(deftest import!-with-no-source-configured-test- (testing "import! with no source configured"+ (testing "import! with no snapshot configured"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})result (impl/import! nil task-id)](is (= :error (:status result)))@@ -37,7 +37,7 @@(testing "import! successful without collections (imports all remote-synced)"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}]- (let [result (impl/import! (test-helpers/create-mock-source) task-id)]+ (let [result (impl/import! (source.p/snapshot (test-helpers/create-mock-source)) task-id)](is (= :success (:status result))))))))(deftest import!-with-branch-parameter-test@@ -45,13 +45,13 @@(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})custom-files {"custom-branch" {"collections/custom-collection.yaml"(test-helpers/generate-collection-yaml "custom-collection-idx" "Custom Collection")}}- result (impl/import! (test-helpers/create-mock-source :initial-files custom-files) task-id)]+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :initial-files custom-files)) task-id)](is (= :success (:status result))))))(deftest import!-handles-network-errors-test(testing "import! handles network errors"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- result (impl/import! (test-helpers/create-mock-source :fail-mode :network-error)+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :network-error))task-id)](is (= :error (:status result)))(is (re-find #"Network error" (:message result))))))@@ -59,28 +59,28 @@(deftest import!-handles-authentication-errors-test(testing "import! handles authentication errors"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- result (impl/import! (test-helpers/create-mock-source :fail-mode :auth-error) task-id)]+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :auth-error)) task-id)](is (= :error (:status result)))(is (re-find #"Authentication failed" (:message result))))))(deftest import!-handles-repository-not-found-errors-test(testing "import! handles repository not found errors"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- result (impl/import! (test-helpers/create-mock-source :fail-mode :repo-not-found) task-id)]+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :repo-not-found)) task-id)](is (= :error (:status result)))(is (re-find #"Repository not found" (:message result))))))(deftest import!-handles-branch-errors-test(testing "import! handles branch errors"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- result (impl/import! (test-helpers/create-mock-source :fail-mode :branch-error) task-id)]+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :branch-error)) task-id)](is (= :error (:status result)))(is (re-find #"Branch error:" (:message result))))))(deftest import!-handles-generic-errors-test(testing "import! handles generic errors"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- result (impl/import! (test-helpers/create-mock-source :fail-mode :list-files-error) task-id)]+ result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :fail-mode :list-files-error)) task-id)](is (= :error (:status result)))(is (re-find #"Failed to reload from git repository" (:message result))))))@@ -90,7 +90,7 @@(testing "import! skips import when source version matches last imported version and force? is false"(mt/with-model-cleanup [:model/RemoteSyncTask](let [mock-source (test-helpers/create-mock-source)- source-version (source.p/version mock-source)+ source-version (source.p/version (source.p/snapshot mock-source))task-defaults (mt/with-temp-defaults :model/RemoteSyncTask)](t2/insert! :model/RemoteSyncTask (merge task-defaults{:sync_task_type "import"@@ -100,7 +100,7 @@(let [task-id-2 (t2/insert-returning-pk! :model/RemoteSyncTask (merge task-defaults{:sync_task_type "import":initiated_by (mt/user->id :rasta)}))- result (impl/import! mock-source task-id-2 :force? false)]+ result (impl/import! (source.p/snapshot mock-source) task-id-2 :force? false)](is (= :success (:status result)))(is (= source-version (:version result)))(is (re-find #"Skipping import.*matches last imported version" (:message result))))))))@@ -109,7 +109,7 @@(testing "import! proceeds with import when source version matches last imported version but force? is true"(mt/with-model-cleanup [:model/Collection :model/RemoteSyncTask](let [mock-source (test-helpers/create-mock-source)- source-version (source.p/version mock-source)+ source-version (source.p/version (source.p/snapshot mock-source))coll-defaults (mt/with-temp-defaults :model/Collection)task-defaults (mt/with-temp-defaults :model/RemoteSyncTask)](t2/insert! :model/Collection (merge coll-defaults@@ -125,7 +125,7 @@(let [task-id-2 (t2/insert-returning-pk! :model/RemoteSyncTask (merge task-defaults{:sync_task_type "import":initiated_by (mt/user->id :rasta)}))- result (impl/import! mock-source task-id-2 :force? true)]+ result (impl/import! (source.p/snapshot mock-source) task-id-2 :force? true)](is (= :success (:status result)))(is (= source-version (:version result)))(is (= "Successfully reloaded from git repository" (:message result))))))))@@ -147,7 +147,7 @@;; Create a regular collection (not remote-synced) to verify it's not included(mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}](let [mock-source (test-helpers/create-mock-source)- result (impl/export! mock-source task-id "Test commit")]+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")](is (= :error (:status result)))(is (= "No remote-synced collections available to sync." (:message result)))))))))@@ -157,7 +157,7 @@(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}](let [mock-source (test-helpers/create-mock-source)- result (impl/export! mock-source task-id "Test commit message")]+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")](is (= :success (:status result)))))))))(deftest export!-handles-store-failure-test@@ -166,7 +166,7 @@(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}](let [mock-source (test-helpers/create-mock-source :fail-mode :store-error)- result (impl/export! mock-source task-id "Test commit message")]+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")](is (= :error (:status result)))(is (re-find #"Failed to export to git repository" (:message result)))))))))@@ -176,7 +176,7 @@(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :type "remote-synced" :entity_id "test-collection-1xxxx" :location "/"}](let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)- result (impl/export! mock-source task-id "Test commit message")]+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")](is (= :error (:status result)))(is (re-find #"Failed to export to git repository" (:message result)))))))))@@ -189,7 +189,7 @@:model/Card {card-id :id} {:name "Test Card" :collection_id coll-id :entity_id "test-card-1xxxxxxxxxx"}](let [mock-main (test-helpers/create-mock-source :branch "test-branch")export-task (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})- export-result (impl/export! mock-main (:id export-task) "Test export")]+ export-result (impl/export! (source.p/snapshot mock-main) (:id export-task) "Test export")](remote-sync.task/complete-sync-task! (:id export-task))(is (= :success (:status export-result)))@@ -204,7 +204,7 @@(t2/delete! :model/RemoteSyncTask :id (:id export-task));; Then import - verify it succeeds and processes the exported files(let [import-task (t2/with-connection [_conn (app-db/app-db) (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})])- import-result (impl/import! mock-main (:id import-task))]+ import-result (impl/import! (source.p/snapshot mock-main) (:id import-task))](remote-sync.task/complete-sync-task! (:id import-task))(is (= :success (:status import-result)))(is (= "Successfully reloaded from git repository" (:message import-result)))@@ -235,7 +235,7 @@"collections/test-collection-1xxxx-_/cards/test-card-1.yaml"(test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card 1" "test-collection-1xxxx")}}mock-main (test-helpers/create-mock-source :initial-files test-files :branch "test-branch")- result (impl/import! mock-main (:id import-task))]+ result (impl/import! (source.p/snapshot mock-main) (:id import-task))](is (= :success (:status result)));; Verify the entities still exist (real cleanup would require more complex setup)@@ -247,7 +247,7 @@(testing "error handling propagation through private functions"(let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- result (impl/import! mock-source task-id)]+ result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :error (:status result)))(is (re-find #"Network error" (:message result))))))@@ -260,7 +260,7 @@(with-redefs [remote-sync.task/update-progress!(fn [task-id progress](swap! progress-calls conj {:task-id task-id :progress progress}))]- (let [result (impl/import! mock-source task-id)]+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)));; Verify progress was called with expected values(is (= 5 (count @progress-calls)))@@ -285,7 +285,7 @@(with-redefs [remote-sync.task/update-progress!(fn [task-id progress](swap! progress-calls conj {:task-id task-id :progress progress}))]- (let [result (impl/export! mock-source task-id "Test commit")]+ (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")](is (= :success (:status result)));; Verify progress was called with expected values(is (= 4 (count @progress-calls)))@@ -309,7 +309,7 @@"collections/test-collection-1xxxx-_/cards/test-card-1.yaml"(test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card" "test-collection-1xxxx")}}mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! mock-source task-id)]+ result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))(let [entries (t2/select :model/RemoteSyncObject)](is (= 2 (count entries)))@@ -336,7 +336,7 @@(is (= "created" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id card-id))))(is (= "removed" (:status (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id dash-id))))(let [mock-source (test-helpers/create-mock-source)- result (impl/export! mock-source task-id "Test commit message")]+ result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")](is (= :success (:status result)))(let [entries (t2/select :model/RemoteSyncObject)](is (= 3 (count entries)))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 51621a7bc4f..7df5f2ef0c3 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -135,16 +135,26 @@;; add extra branch to remote to check it is picked up(git-working-create-branch! _remote "branch-3")(is (= ["branch-1" "branch-2" "branch-3" "master"] (source.p/branches source))))))-;++(deftest snapshot+ (mt/with-temp-dir [remote-dir nil]+ (let [[master _remote] (init-source! "master" remote-dir+ :files {"master.txt" "File in master"+ "subdir/path.txt" "File in subdir"}+ :branches ["branch-1" "branch-2"])+ master-snapshot (source.p/snapshot master)]+ (is (= (git/commit-sha master "master") (:version master-snapshot))))))+(deftest list-files(mt/with-temp-dir [remote-dir nil](let [[master remote] (init-source! "master" remote-dir:files {"master.txt" "File in master""subdir/path.txt" "File in subdir"}:branches ["branch-1" "branch-2"])- branch-1 (->source! "branch-1" remote)- branch-2 (->source! "branch-2" remote)]- (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files master)))+ master-snap (source.p/snapshot master)+ branch-1 (source.p/snapshot (->source! "branch-1" remote))+ branch-2 (source.p/snapshot (->source! "branch-2" remote))]+ (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files master-snap)))(is (= ["file-in-branch-1.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-1)))(is (= ["file-in-branch-2.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-2))))))@@ -154,26 +164,22 @@:files {"master.txt" "File in master""subdir/path.txt" "File in subdir"}:branches ["branch-1" "branch-2"])- branch-1 (->source! "branch-1" _remote)- invalid-branch (->source! "invalid-branch" _remote)]+ master-snap (source.p/snapshot master)+ branch-1 (source.p/snapshot (->source! "branch-1" _remote))](testing "Reading master"- (is (= "File in master" (source.p/read-file master "master.txt")))- (is (= "File in subdir" (source.p/read-file master "subdir/path.txt")))- (is (nil? (source.p/read-file master "file-in-branch-1.txt"))))+ (is (= "File in master" (source.p/read-file master-snap "master.txt")))+ (is (= "File in subdir" (source.p/read-file master-snap "subdir/path.txt")))+ (is (nil? (source.p/read-file master-snap "file-in-branch-1.txt"))))(testing "Reading branch-1"(is (= "File in master" (source.p/read-file branch-1 "master.txt")))(is (= "File in branch-1" (source.p/read-file branch-1 "file-in-branch-1.txt")))- (is (nil? (source.p/read-file branch-1 "file-in-branch-2.txt"))))-- (testing "Reading invalid branch"- (is (nil? (source.p/read-file invalid-branch "master.txt")))))))+ (is (nil? (source.p/read-file branch-1 "file-in-branch-2.txt")))))))(deftest write-files(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")otherdir-path (str "collections/" "o" (subs (u/generate-nano-id "b") 1) "_otherdir/")- thirddir-path (str "collections/" "s" (subs (u/generate-nano-id "c") 1) "_thirddir/")- branched-path (str "collections/" "b" (subs (u/generate-nano-id "d") 1) "_branched/")]+ thirddir-path (str "collections/" "s" (subs (u/generate-nano-id "c") 1) "_thirddir/")](mt/with-temp-dir [remote-dir nil](let [[master remote] (init-source! "master" remote-dir:files {"master.txt" "File in master"@@ -184,33 +190,33 @@(str otherdir-path "path2.txt") "File 2 in otherdir"(str thirddir-path "path.txt") "File in third dir"(str thirddir-path "path2.txt") "File 2 in third dir"}- :branches ["branch-1" "branch-2"])- new-branch (->source! "new-branch" remote)]+ :branches ["branch-1" "branch-2"])](testing "Files in a subdir are replaced, other subdirs and root are unchanged"- (source.p/write-files! master "Update 1" [{:path "master.txt" :content "Updated master content"}- {:path (str subdir-path "path.txt") :content "Updated subdir content"}- {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}- {:path (str thirddir-path "path.txt") :content "Updated third dir content"}- {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])+ (source.p/write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}+ {:path (str subdir-path "path.txt") :content "Updated subdir content"}+ {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}+ {:path (str thirddir-path "path.txt") :content "Updated third dir content"}+ {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])(is (= ["Update 1" "Initial commit"] (map :message (git/log master))))- (is (= [(str otherdir-path "path.txt")- (str otherdir-path "path2.txt")- (str subdir-path "path.txt")- (str subdir-path "path3.txt")- (str thirddir-path "path.txt")- (str thirddir-path "path3.txt")- "master.txt"- "master2.txt"]- (source.p/list-files master)))+ (let [master-snap (source.p/snapshot master)]+ (is (= [(str otherdir-path "path.txt")+ (str otherdir-path "path2.txt")+ (str subdir-path "path.txt")+ (str subdir-path "path3.txt")+ (str thirddir-path "path.txt")+ (str thirddir-path "path3.txt")+ "master.txt"+ "master2.txt"]+ (source.p/list-files master-snap)))- (is (= "Updated master content" (source.p/read-file master "master.txt")))- (is (= "File 2 in master" (source.p/read-file master "master2.txt")))- (is (= "File 2 in otherdir" (source.p/read-file master (str otherdir-path "path2.txt"))))- (is (= "Updated subdir content" (source.p/read-file master (str subdir-path "path.txt"))))- (is (= "Updated subdir content 3" (source.p/read-file master (str subdir-path "path3.txt"))))+ (is (= "Updated master content" (source.p/read-file master-snap "master.txt")))+ (is (= "File 2 in master" (source.p/read-file master-snap "master2.txt")))+ (is (= "File 2 in otherdir" (source.p/read-file master-snap (str otherdir-path "path2.txt"))))+ (is (= "Updated subdir content" (source.p/read-file master-snap (str subdir-path "path.txt"))))+ (is (= "Updated subdir content 3" (source.p/read-file master-snap (str subdir-path "path3.txt")))))(testing "Check remote repo directly"- (is (= "Updated master content" (git/read-file (assoc remote :commit-ish "master") "master.txt")))+ (is (= "Updated master content" (git/read-file (assoc remote :version "master") "master.txt")))(is (= [(str otherdir-path "path.txt")(str otherdir-path "path2.txt")(str subdir-path "path.txt")@@ -219,11 +225,11 @@(str thirddir-path "path3.txt")"master.txt""master2.txt"]- (git/list-files (assoc remote :commit-ish "master"))))- (is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))))+ (git/list-files (assoc remote :version "master"))))+ (is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))))(testing "If no root fils are touched, they all stay as-is"- (source.p/write-files! master "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])+ (source.p/write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])(is (= [(str otherdir-path "path.txt")(str otherdir-path "path2.txt")(str subdir-path "path.txt")@@ -231,56 +237,7 @@(str thirddir-path "path.txt")"master.txt""master2.txt"]- (git/list-files (assoc remote :commit-ish "master")))))-- (testing "Writing a a new branch"- (source.p/write-files! new-branch "New Branch" [{:path (str branched-path "branched-file.txt") :content "File added to branch"}- {:path (str branched-path "branched-file2.txt") :content "File 2 added to branch"}- {:path (str otherdir-path "path.txt") :content "Updated collections/otherdir/path in branch"}- {:path (str otherdir-path "path3.txt") :content "Updated collections/otherdir/path in branch"}- {:path "new-file.txt" :content "Updated file in branch"}])-- (is (= "File added to branch" (source.p/read-file new-branch (str branched-path "branched-file.txt"))))- (is (= "Updated file in branch" (source.p/read-file new-branch "new-file.txt")))- (is (= [(str branched-path "branched-file.txt")- (str branched-path "branched-file2.txt")- (str otherdir-path "path.txt")- (str otherdir-path "path3.txt")- (str subdir-path "path.txt")- (str subdir-path "path3.txt")- (str thirddir-path "path.txt")- "master.txt"- "master2.txt"- "new-file.txt"]- (source.p/list-files new-branch)))- (is (= ["New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log new-branch))))-- (testing "Check remote repo"- (is (= ["New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))))-- (testing "Updating a branch"- (source.p/write-files! new-branch "Updating Branch" [{:path (str branched-path "branched-file.txt") :content "File updated in branch"}- {:path (str branched-path "branched-file3.txt") :content "File 3 updated in branch"}- {:path "another-file.txt" :content "Added in 2nd commit"}])-- (is (= "File updated in branch" (source.p/read-file new-branch (str branched-path "branched-file.txt"))))- (is (= "Added in 2nd commit" (source.p/read-file new-branch "another-file.txt")))- (is (= ["another-file.txt"- (str branched-path "branched-file.txt")- (str branched-path "branched-file3.txt")- (str otherdir-path "path.txt")- (str otherdir-path "path3.txt")- (str subdir-path "path.txt")- (str subdir-path "path3.txt")- (str thirddir-path "path.txt")- "master.txt"- "master2.txt"- "new-file.txt"]- (source.p/list-files new-branch)))- (is (= ["Updating Branch" "New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log new-branch))))-- (testing "Check remote repo"- (is (= ["Updating Branch" "New Branch" "Update 2" "Update 1" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))))))))+ (git/list-files (assoc remote :version "master")))))))))(deftest write-special-collections(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]@@ -300,7 +257,7 @@(testing "Initial clone is the same"(is (= ["Initial commit"] (map :message (git/log master))))- (is (= ["Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))+ (is (= ["Initial commit"] (map :message (git/log (assoc remote :branch "master")))));; Add an extra commit to remote(git-working-add! remote "additional-file.txt" "Additional file content")@@ -308,14 +265,10 @@(testing "Source is behind remote"(is (= ["Initial commit"] (map :message (git/log master))))- (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))-- (is (= "File in master" (source.p/read-file master "master.txt")))- (is (nil? (source.p/read-file master "additional-file.txt"))))+ (is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master"))))))(testing "After fetch, source is up to date"(git/fetch! master)- (is (= "Additional file content" (source.p/read-file master "additional-file.txt")))(is (= ["Added additional file" "Initial commit"] (map :message (git/log master)))))(testing "Writing a file to source and pushing back to remote when there is new content on remote"@@ -323,15 +276,15 @@(git-working-add! remote "only-on-remote.txt" "Initially on remote")(git-working-commit! remote "Only on remote")- (source.p/write-files! master "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])+ (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])(testing "Remote has the new commit with just the files committed, but only version is in history"- (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "master")))))- (is (= ["additional-file.txt" (str subdir-path "path.txt") "initially-source.txt" "master.txt" "only-on-remote.txt"] (git/list-files (assoc remote :commit-ish "master"))))- (is (= "Initially on source" (git/read-file (assoc remote :commit-ish "master") "initially-source.txt"))))+ (is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))+ (is (= ["additional-file.txt" (str subdir-path "path.txt") "initially-source.txt" "master.txt" "only-on-remote.txt"] (git/list-files (assoc remote :version "master"))))+ (is (= "Initially on source" (git/read-file (assoc remote :version "master") "initially-source.txt"))))(testing "Source has the same history"- (is (= (map :message (git/log (assoc remote :commit-ish "master"))) (map :message (git/log master))))))+ (is (= (map :message (git/log (assoc remote :branch "master"))) (map :message (git/log master))))))(testing "Writing to a branch local has not seen (but remote has) adds it to the history on remote"(git-working-checkout! remote "new-branch" true)@@ -339,64 +292,63 @@(git-working-add! remote "new-branch-remote.txt" "Initially on remote")(git-working-commit! remote "New-branch on remote")- (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))+ (is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))(is (nil? (git/log new-branch)))- (source.p/write-files! new-branch "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}- {:path "new-branch-file.txt" :content "Updated on source"}])+ (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}+ {:path "new-branch-file.txt" :content "Updated on source"}])- (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :commit-ish "new-branch")))))))))))+ (is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))))))))(deftest git-source-using-commit-ref(mt/with-temp-dir [remote-dir nil]- (let [[master remote] (init-source! "master" remote-dir- :files {"master.txt" "File in master"- "subdir/path.txt" "File in subdir"})- original-ref (git/commit-sha master "master")]+ (let [[master _remote] (init-source! "master" remote-dir+ :files {"master.txt" "File in master"+ "subdir/path.txt" "File in subdir"})+ old-master (source.p/snapshot master)]- (source.p/write-files! master "Update file" [{:path "master.txt" :content "Updated file in master"}- {:path "new-file.txt" :content "New file in master"}])- (let [old-master (->source! original-ref remote)]- (is (= "File in master" (source.p/read-file old-master "master.txt")))- (is (= "Updated file in master" (source.p/read-file master "master.txt")))- (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files old-master)))))))+ (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated file in master"}+ {:path "new-file.txt" :content "New file in master"}])+ (is (= "File in master" (source.p/read-file old-master "master.txt")))+ (is (= "Updated file in master" (source.p/read-file (source.p/snapshot master) "master.txt")))+ (is (= ["master.txt" "subdir/path.txt"] (source.p/list-files old-master))))))(deftest version-test(testing "version returns the commit id for the current state"(mt/with-temp-dir [remote-dir nil](let [[master remote] (init-source! "master" remote-dir:files {"master.txt" "File in master"})- initial-version (source.p/version master)]+ initial-version (source.p/version (source.p/snapshot master))](is (string? initial-version) "version should return a string")(is (= 40 (count initial-version)) "version should be a full SHA-1 hash (40 characters)")(is (= (git/commit-sha master "master") initial-version)"version should match the commit id for the branch")(testing "version changes after writing files"- (source.p/write-files! master "Update file" [{:path "master.txt" :content "Updated content"}])- (let [new-version (source.p/version master)]+ (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated content"}])+ (let [new-version (source.p/version (source.p/snapshot master))](is (not= initial-version new-version) "version should change after commit")(is (= 40 (count new-version)) "new version should also be a full SHA-1 hash")(is (= (git/commit-sha master "master") new-version)"new version should match the new commit id")))(testing "version is consistent across multiple calls"- (let [version-1 (source.p/version master)- version-2 (source.p/version master)]+ (let [version-1 (source.p/version (source.p/snapshot master))+ version-2 (source.p/version (source.p/snapshot master))](is (= version-1 version-2) "version should be consistent without changes")))(testing "version differs for different branches"(git-working-create-branch! remote "branch-1")(let [branch-1 (->source! "branch-1" remote)- master-version (source.p/version master)- branch-version (source.p/version branch-1)]+ master-version (source.p/version (source.p/snapshot master))+ branch-version (source.p/version (source.p/snapshot branch-1))](is (not= master-version branch-version)"different branches should have different versions")))(testing "version matches specific commit ref"(let [commit-ref (git/commit-sha master "master")source-with-ref (->source! commit-ref remote)]- (is (= commit-ref (source.p/version source-with-ref))+ (is (= commit-ref (source.p/version (source.p/snapshot source-with-ref)))"version should work with explicit commit refs")))))))(deftest default-branchdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.cljindex e9cb1d3e6ba..720cf207416 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase-enterprise.remote-sync.source.ingestable :as ingestable]+ [metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase-enterprise.serialization.core :as serialization][metabase.test.fixtures :as fixtures]@@ -12,10 +13,10 @@(use-fixtures :once (fixtures/initialize :db))-(deftest ingestable-source-test- (testing "IngestableSource wraps a source and provides Ingestable interface"+(deftest ingestable-snapshot-test+ (testing "IngestableSnapshot wraps a snapshot and provides Ingestable interface"(let [mock-source (test-helpers/create-mock-source)- ingestable (ingestable/->IngestableSource mock-source (atom nil))]+ ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))](testing "ingest-list returns list of serdes paths"(let [paths (serialization/ingest-list ingestable)]@@ -31,7 +32,7 @@(is (contains? entity :serdes/meta) "Should have serdes/meta key")))(testing "cache is populated after first ingest-list call"- (let [ingestable (ingestable/->IngestableSource mock-source (atom nil))]+ (let [ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))](is (nil? @(:cache ingestable)) "Cache should be empty initially")(serialization/ingest-list ingestable)(is (map? @(:cache ingestable)) "Cache should be populated after ingest-list")@@ -40,7 +41,7 @@(deftest callback-ingestable-test(testing "CallbackIngestable wraps an ingestable and calls callback on ingest-one"(let [mock-source (test-helpers/create-mock-source)- base-ingestable (ingestable/->IngestableSource mock-source (atom nil))+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))calls (atom [])callback (fn [_ path] (swap! calls conj path))wrapped (ingestable/->CallbackIngestable base-ingestable callback)]@@ -78,7 +79,7 @@:initiated_by (:id user)}))task-id (:id task)mock-source (test-helpers/create-mock-source)- base-ingestable (ingestable/->IngestableSource mock-source (atom nil))+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))normalize 100](testing "creates a CallbackIngestable"@@ -123,7 +124,7 @@(deftest root-dependency-ingestable-test(testing "RootDependencyIngestable filters items based on root dependencies"(let [mock-source (test-helpers/create-mock-source)- base-ingestable (ingestable/->IngestableSource mock-source (atom nil))]+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil))](testing "with no root dependencies, returns empty list"(let [wrapped (ingestable/wrap-root-dep-ingestable [] base-ingestable)diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.cljindex 556581911cf..7bc3bea0a34 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj@@ -4,6 +4,21 @@[metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.source.protocol :as source.p]))+(defrecord MockSourceSnapshot [files]+ source.p/SourceSnapshot+ (list-files [_]+ (keys files))++ (read-file [_ path]+ (get files path))++ (write-files! [_ _message new-files]+ (into {} (map (juxt :path :content) new-files))+ "written-files-version")++ (version [_]+ "mock-version"))+(defrecord MockSource [files]source.p/Source(create-branch [_ _branch _base]@@ -15,17 +30,8 @@(default-branch [_]"main")- (list-files [_]- (keys files))-- (read-file [_ path]- (get files path))-- (write-files! [_ _message new-files]- (into {} (map (juxt :path :content) new-files)))-- (version [_]- "mock-version"))+ (snapshot [_]+ (->MockSourceSnapshot files)))(deftest wrapping-source-list-files-single-filter-test(testing "WrappingSource filters list-files based on path-filters"@@ -34,9 +40,9 @@"databases/db1.yaml" "db1-content""databases/db2.yaml" "db2-content""dashboards/dash1.yaml" "dash1-content"})- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])]+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])](is (= ["collections/foo.yaml" "collections/bar.yaml"]- (source.p/list-files wrapped-source))+ (source.p/list-files wrapped-snap))"Should only include files matching the collections pattern"))))(deftest wrapping-source-list-files-multiple-filters-test@@ -46,18 +52,18 @@"databases/db1.yaml" "db1-content""databases/db2.yaml" "db2-content""dashboards/dash1.yaml" "dash1-content"})- wrapped-source (source/->WrappingSource mock-source [#"collections/.*" #"databases/.*"])]+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*" #"databases/.*"])](is (= ["collections/foo.yaml" "collections/bar.yaml" "databases/db1.yaml" "databases/db2.yaml"]- (source.p/list-files wrapped-source))+ (source.p/list-files wrapped-snap))"Should include files matching any of the patterns"))))(deftest wrapping-source-list-files-no-match-test(testing "WrappingSource with no matching files"(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content""dashboards/dash1.yaml" "dash1-content"})- wrapped-source (source/->WrappingSource mock-source [#"databases/.*"])]+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"databases/.*"])](is (= []- (source.p/list-files wrapped-source))+ (source.p/list-files wrapped-snap))"Should return empty list when no files match"))))(deftest wrapping-source-read-file-single-filter-test@@ -65,11 +71,11 @@(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content""collections/bar.yaml" "bar-content""databases/db1.yaml" "db1-content"})- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])]+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])](is (= "foo-content"- (source.p/read-file wrapped-source "collections/foo.yaml"))+ (source.p/read-file wrapped-snap "collections/foo.yaml"))"Should read file that matches filter")- (is (nil? (source.p/read-file wrapped-source "databases/db1.yaml"))+ (is (nil? (source.p/read-file wrapped-snap "databases/db1.yaml"))"Should return nil for file that doesn't match filter"))))(deftest wrapping-source-read-file-multiple-filters-test@@ -77,40 +83,34 @@(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content""databases/db1.yaml" "db1-content""dashboards/dash1.yaml" "dash1-content"})- wrapped-source (source/->WrappingSource mock-source [#"collections/.*" #"dashboards/.*"])]+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*" #"dashboards/.*"])](is (= "foo-content"- (source.p/read-file wrapped-source "collections/foo.yaml"))+ (source.p/read-file wrapped-snap "collections/foo.yaml"))"Should read file matching first pattern")(is (= "dash1-content"- (source.p/read-file wrapped-source "dashboards/dash1.yaml"))+ (source.p/read-file wrapped-snap "dashboards/dash1.yaml"))"Should read file matching second pattern")- (is (nil? (source.p/read-file wrapped-source "databases/db1.yaml"))+ (is (nil? (source.p/read-file wrapped-snap "databases/db1.yaml"))"Should return nil for file not matching any pattern"))))(deftest wrapping-source-write-files-single-filter-test(testing "WrappingSource filters write-files! based on path-filters"(let [written-files (atom [])- mock-source (reify source.p/Source- (create-branch [_ _branch _base]- nil)- (branches [_]- [])- (default-branch [_]- "main")- (list-files [_]- [])- (read-file [_ _path]- nil)- (write-files! [_ _message files]- (reset! written-files files)- nil)- (version [_]- "mock-version"))- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])+ mock-snap (reify source.p/SourceSnapshot+ (list-files [_]+ [])+ (read-file [_ _path]+ nil)+ (write-files! [_ _message files]+ (reset! written-files files)+ nil)+ (version [_]+ "mock-version"))+ wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*"])files-to-write [{:path "collections/foo.yaml" :content "foo-content"}{:path "collections/bar.yaml" :content "bar-content"}{:path "databases/db1.yaml" :content "db1-content"}]]- (source.p/write-files! wrapped-source "test commit" files-to-write)+ (source.p/write-files! wrapped-snap "test commit" files-to-write)(is (= [{:path "collections/foo.yaml" :content "foo-content"}{:path "collections/bar.yaml" :content "bar-content"}]@written-files)@@ -119,27 +119,21 @@(deftest wrapping-source-write-files-multiple-filters-test(testing "WrappingSource write-files! with multiple filters"(let [written-files (atom [])- mock-source (reify source.p/Source- (create-branch [_ _branch _base]- nil)- (branches [_]- [])- (default-branch [_]- "main")- (list-files [_]- [])- (read-file [_ _path]- nil)- (write-files! [_ _message files]- (reset! written-files files)- nil)- (version [_]- "mock-version"))- wrapped-source (source/->WrappingSource mock-source [#"collections/.*" #"dashboards/.*"])+ mock-snap (reify source.p/SourceSnapshot+ (list-files [_]+ [])+ (read-file [_ _path]+ nil)+ (write-files! [_ _message files]+ (reset! written-files files)+ nil)+ (version [_]+ "mock-version"))+ wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*" #"dashboards/.*"])files-to-write [{:path "collections/foo.yaml" :content "foo-content"}{:path "databases/db1.yaml" :content "db1-content"}{:path "dashboards/dash1.yaml" :content "dash1-content"}]]- (source.p/write-files! wrapped-source "test commit" files-to-write)+ (source.p/write-files! wrapped-snap "test commit" files-to-write)(is (= [{:path "collections/foo.yaml" :content "foo-content"}{:path "dashboards/dash1.yaml" :content "dash1-content"}]@written-files)@@ -148,34 +142,18 @@(deftest wrapping-source-write-files-no-match-test(testing "WrappingSource write-files! with no matching files"(let [written-files (atom [])- mock-source (reify source.p/Source- (create-branch [_ _branch _base]- nil)- (branches [_]- [])- (default-branch [_]- "main")- (list-files [_]- [])- (read-file [_ _path]- nil)- (write-files! [_ _message files]- (reset! written-files files)- nil)- (version [_]- "mock-version"))- wrapped-source (source/->WrappingSource mock-source [#"collections/.*"])+ mock-snapshot (reify source.p/SourceSnapshot+ (list-files [_]+ [])+ (read-file [_this _path]+ nil)+ (write-files! [_this _message _files]+ nil)+ (version [_this]+ nil))files-to-write [{:path "databases/db1.yaml" :content "db1-content"}{:path "dashboards/dash1.yaml" :content "dash1-content"}]]- (source.p/write-files! wrapped-source "test commit" files-to-write)+ (source.p/write-files! mock-snapshot "test commit" files-to-write)(is (= []@written-files)"Should write no files when none match"))))--(deftest wrapping-source-branches-test- (testing "WrappingSource delegates branches to original source"- (let [mock-source (->MockSource {})- wrapped-source (source/->WrappingSource mock-source [#".*"])]- (is (= ["main"]- (source.p/branches wrapped-source))- "Should return branches from original source"))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex a481417e2c3..9b34f0150cd 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -13,17 +13,8 @@(use-fixtures :each th/clean-remote-sync-state)-(defrecord MockSource [written-files]- source.p/Source- (create-branch [_ _branch _base]- nil)-- (branches [_]- ["main"])-- (default-branch [_]- "main")-+(defrecord MockSourceSnapshot [written-files]+ source.p/SourceSnapshot(list-files [_][])@@ -32,11 +23,26 @@(write-files! [_ message files](let [realized-files (vec files)]- (reset! written-files {:message message :files realized-files})))+ (reset! written-files {:message message :files realized-files}))+ "mock-written-version")(version [_]"mock-version"))+(defrecord MockSource [written-files]+ source.p/Source+ (create-branch [_ _branch _base]+ nil)++ (branches [_]+ ["main"])++ (default-branch [_]+ "main")++ (snapshot [_]+ (->MockSourceSnapshot written-files)))+(defn- create-test-entity"Creates a test entity with serdes/meta for testing"[id label model-name]@@ -48,7 +54,7 @@(testing "store! writes files to source with correct structure"(mt/with-temp [:model/User user {:first_name "Test":last_name "User"- :email "[email protected]"+ :email "[email protected]":password "password123"}:model/RemoteSyncTask {task-id :id} {:sync_task_type "export":initiated_by (:id user)}]@@ -57,7 +63,7 @@test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")(create-test-entity "test-id-2" "entity-two" "Card")]]- (source/store! test-entities mock-source task-id "Test commit message")+ (is (= "mock-written-version" (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message")))(testing "write-files! was called with correct message"(is (= "Test commit message" (:message @written-files))))@@ -89,7 +95,7 @@(testing "store! updates task progress as files are written"(mt/with-temp [:model/User user {:first_name "Test":last_name "User"- :email "[email protected]"+ :email "[email protected]":password "password123"}:model/RemoteSyncTask {task-id :id} {:sync_task_type "export":initiated_by (:id user)}]@@ -102,7 +108,7 @@(let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)](is (nil? (:progress initial-task)) "Progress should be nil initially"))- (source/store! test-entities mock-source task-id "Test commit")+ (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit")(let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)](is (some? (:progress final-task)) "Progress should be updated after store!")@@ -115,14 +121,14 @@(testing "store! handles empty stream correctly"(mt/with-temp [:model/User user {:first_name "Test":last_name "User"- :email "[email protected]"+ :email "[email protected]":password "password123"}:model/RemoteSyncTask {task-id :id} {:sync_task_type "export":initiated_by (:id user)}](let [written-files (atom nil)mock-source (->MockSource written-files)]- (source/store! [] mock-source task-id "Empty commit")+ (source/store! [] (source.p/snapshot mock-source) task-id "Empty commit")(testing "write-files! was called even with empty stream"(is (some? @written-files)))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex a9488e9b144..ae7d8e9a874 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -155,22 +155,8 @@ width: fixed"name entity-id collection-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_") dashcards-yaml)))-(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom]- source.p/Source- (create-branch [_this branch _base]- (swap! branches-atom conj [branch (str branch "-ref")]))-- (branches [_this]- (case fail-mode- :branches-error (throw (java.net.UnknownHostException. "Network error"))- :auth-error (throw (Exception. "Authentication failed"))- :repo-not-found (throw (Exception. "Repository not found"))- ;; Default success case- @branches-atom))-- (default-branch [_this]- "main")-+(defrecord MockSourceSnapshot [source-id base-url branch fail-mode files-atom]+ source.p/SourceSnapshot(list-files [_this](case fail-mode:list-files-error (throw (Exception. "Failed to list files"))@@ -197,11 +183,31 @@ width: fixed:store-error (throw (Exception. "Store failed")):network-error (throw (java.net.UnknownHostException. "Remote host not found"));; Default success case - write files to atom- (swap! files-atom assoc branch (into {} (map (juxt :path :content) files)))))+ (swap! files-atom assoc branch (into {} (map (juxt :path :content) files))))+ "write-files-version")(version [_this]"mock-version"))+(defrecord MockSource [source-id base-url branch fail-mode files-atom branches-atom]+ source.p/Source+ (create-branch [_this branch _base]+ (swap! branches-atom conj [branch (str branch "-ref")]))++ (branches [_this]+ (case fail-mode+ :branches-error (throw (java.net.UnknownHostException. "Network error"))+ :auth-error (throw (Exception. "Authentication failed"))+ :repo-not-found (throw (Exception. "Repository not found"))+ ;; Default success case+ @branches-atom))++ (default-branch [_this]+ "main")++ (snapshot [_this]+ (->MockSourceSnapshot source-id base-url branch fail-mode files-atom)))+(defn create-mock-source"Create a mock Source for testing"[& {:keys [branch fail-mode initial-files]