Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Skip empty remote-sync commits after a no-op export (GHY-3934) (#76753)

Eric Normand6cc57802d45e
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 07b27426cc7..df246daa051 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -629,55 +629,86 @@
(finally
(analytics/observe! :metabase-remote-sync/import-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))))
+(defn- commit-staged!
+ "Open a commit on `snapshot`, stage into it via `stage-fn`, then finish it.
+
+ `stage-fn` will be passed the open commit. It should return the synced write-rows.
+
+ Will abort the commit if it is empty.
+
+ Returns `[version, write-rows]` where version is the new commit SHA or `:remote-sync/empty-commit` if empty.
+
+ Will abort the commit and throw on any Throwable."
+ [snapshot message stage-fn]
+ (let [commit (source.p/open-commit snapshot)]
+ (try
+ (let [synced (stage-fn commit)
+ version (if (source.p/empty-commit? commit)
+ (do (source.p/abort-commit! commit) :remote-sync/empty-commit)
+ (source.p/finish-commit! commit message))]
+ [synced version])
+ (catch Throwable e
+ (source.p/abort-commit! commit)
+ (throw e)))))
+
(defn- export-merged!
- "Export path taken when the remote branch has advanced beyond the last synced version (`base-snapshot`
- is the merge base). Performs an entity-identity 3-way merge of local state against the remote tip:
- - on a genuine conflict (same entity changed on both sides) returns a `:conflict` result;
+ "Export when the remote branch has advanced beyond the last synced version. Runs an entity-identity 3-way merge of
+ local state (`models`) against the remote tip:
+
+ - on a genuine conflict (same entity changed on both sides) returns a `:conflict` result without writing
- on a clean merge, writes the merged set (fast-forwarding onto the remote tip), then reconciles the
local app DB by loading the merged result (the 'pull' half), so local now contains the remote's
- changes. Returns a `:success` result with a `:merge-summary`."
+ changes.
+
+ Returns a `:success` result with a `:merge-summary`."
[source snapshot base-snapshot task-id message sync-timestamp models]
(let [pushed-count (count (remote-sync.object/dirty-rows))
- {:keys [status version conflicts summary]}
- (source/merge-and-store! models snapshot base-snapshot task-id message)]
- (case status
- :conflict
- (let [conflicts (mapv remote-sync.merge/conflict-label conflicts)]
- (log/infof "Export merge conflict on %d entit(ies): %s"
- (count conflicts) (str/join ", " conflicts))
+ {:keys [merged conflicts summary]} (source/compute-merge models snapshot base-snapshot task-id)]
+ (if (seq conflicts)
+ (let [labels (mapv remote-sync.merge/conflict-label conflicts)]
+ (log/infof "Export merge conflict on %d entit(ies): %s" (count labels) (str/join ", " labels))
{:status :conflict
:version (source.p/version snapshot)
- :conflicts conflicts
+ :conflicts labels
:merge-summary summary
:message "Export blocked: the same content was changed both locally and on the remote branch."})
-
- :success
- ;; Fold-in pull: the merge brought remote changes that aren't in the local app DB yet. Load the
- ;; merged result so local state matches what we just pushed, marking everything synced and advancing
- ;; the version atomically with the reconcile (set-version! runs only after the load, so a crash leaves
- ;; the version at the old base and a retry re-merges — the push is idempotent — rather than advancing
- ;; the pointer past un-reconciled local state).
- (if-let [merged-snapshot (source.p/snapshot-at source version)]
- (do
- (load-snapshot! merged-snapshot task-id sync-timestamp
- :finalize! (fn []
- (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})
- (remote-sync.task/set-version! task-id version)))
- (log/infof "Exported with merge: folded in %d remote change(s) (added %d, updated %d, removed %d)"
- (apply + (vals summary)) (:added summary) (:updated summary) (:removed summary))
- {:status :success :version version :merge-summary summary
- :outcome {:kind "merged"
- :pulled (apply + (vals summary))
- :pushed pushed-count
- :branch (settings/remote-sync-branch)}})
- ;; The merge was pushed to `version`, but its commit can't be resolved locally (should not happen —
- ;; finish-commit! updates the local ref before returning). Fail loudly rather than silently advancing
- ;; the version while skipping the reconcile, which would leave local missing the folded-in remote
- ;; changes. The push already landed, so a retry sees the divergence and re-merges + reconciles.
- (throw (ex-info (format (str "Merge pushed to %s but its commit could not be resolved locally to "
- "reconcile the app DB; re-run the export to pull the merged changes.")
- version)
- {:version version}))))))
+ (let [[_ version] (commit-staged! snapshot message
+ (fn [commit]
+ (source.p/replace-all! commit) ; merged set replaces the managed dirs wholesale
+ (run! #(source.p/stage-upsert! commit %) merged)))
+ ;; An empty merge means the merged set already matched the remote tip: nothing was pushed, so
+ ;; reconcile (and advance) against the tip rather than a non-existent merge commit.
+ empty? (= version :remote-sync/empty-commit)
+ version (if empty? (source.p/version snapshot) version)]
+ ;; Fold-in pull: the merge brought remote changes that aren't in the local app DB yet. Load the
+ ;; merged result so local state matches what we just pushed, marking everything synced and advancing
+ ;; the version atomically with the reconcile (set-version! runs only after the load, so a crash leaves
+ ;; the version at the old base and a retry re-merges — the push is idempotent — rather than advancing
+ ;; the pointer past un-reconciled local state).
+ (if-let [merged-snapshot (source.p/snapshot-at source version)]
+ (let [pulled (apply + (vals summary))]
+ (load-snapshot! merged-snapshot task-id sync-timestamp
+ :finalize! (fn []
+ (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})
+ (remote-sync.task/set-version! task-id version)))
+ (log/infof "Exported with merge: folded in %d remote change(s) (added %d, updated %d, removed %d); pushed %d"
+ pulled (:added summary) (:updated summary) (:removed summary) (if empty? 0 pushed-count))
+ {:status :success :version version :merge-summary summary
+ ;; An empty merge pushed nothing: it's a pull when remote changes were folded in, or a no-op
+ ;; when nothing changed on either side.
+ :outcome (cond
+ (not empty?) {:kind "merged" :pulled pulled :pushed pushed-count
+ :branch (settings/remote-sync-branch)}
+ (pos? pulled) {:kind "pulled" :count pulled :branch (settings/remote-sync-branch)}
+ :else {:kind "push-skipped"})})
+ ;; The merge was pushed to `version`, but its commit can't be resolved locally (should not happen —
+ ;; finish-commit! updates the local ref before returning). Fail loudly rather than silently advancing
+ ;; the version while skipping the reconcile, which would leave local missing the folded-in remote
+ ;; changes. The push already landed, so a retry sees the divergence and re-merges + reconciles.
+ (throw (ex-info (format (str "Merge pushed to %s but its commit could not be resolved locally to "
+ "reconcile the app DB; re-run the export to pull the merged changes.")
+ version)
+ {:version version})))))))
;;; ------------------------------------------- Incremental Export Fast-Path -------------------------------------------
@@ -699,10 +730,10 @@
(its transitive `serdes/descendants` + `serdes/required`, including the entity itself)."
[model-type model-id]
;; serdes works in [model-type id] tuples; map the closure keys to {:model_type :model_id} on the way out
- (map (fn [[mt id]] {:model_type mt :model_id id})
- (keys (merge-with into
- (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))
- (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %)))))))
+ (-> #{}
+ (into (keys (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))))
+ (into (keys (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %)))))
+ (->> (map (fn [[mt id]] {:model_type mt :model_id id})))))
(defn- untracked-content-deps
"The `{:model_type :model_id}` entities in `[model-type model-id]`'s export closure that are remote-sync
@@ -996,48 +1027,51 @@
(when (empty? export-rows)
(throw (ex-info "No remote-syncable content available." {})))
(remote-sync.task/update-progress! task-id 0.3)
- (let [opts (serdes/storage-base-context)
- commit (source.p/open-commit snapshot)
- [synced version] (try
- (source.p/replace-all! commit) ; replace the managed dirs wholesale
- [(stage-writes commit opts export-rows)
- (source.p/finish-commit! commit message)]
- (catch Throwable e
- (source.p/abort-commit! commit)
- (throw e)))]
+ (let [opts (serdes/storage-base-context)
+ [synced version] (commit-staged! snapshot message
+ (fn [commit]
+ (source.p/replace-all! commit) ; replace the managed dirs wholesale
+ (stage-writes commit opts export-rows)))]
(t2/with-transaction [_]
- (remote-sync.task/set-version! task-id version)
+ (when-not (= version :remote-sync/empty-commit)
+ (remote-sync.task/set-version! task-id version))
(doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))]
(t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
(mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp))
- {:status :success
- :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}})))
+ (if (= version :remote-sync/empty-commit)
+ (do
+ (log/info "Remote sync full export: re-serialized content matches remote; skipped empty commit")
+ {:status :success :outcome {:kind "push-skipped"}})
+ {:status :success
+ :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}}))))
(defn- incremental-export!
[plan disabled-files task-id snapshot message sync-timestamp]
(let [{:keys [writes delete-paths removed-ids]} plan
delete-paths (into (vec delete-paths) disabled-files)]
(remote-sync.task/update-progress! task-id 0.3)
- (let [opts (serdes/storage-base-context)
- commit (source.p/open-commit snapshot)
- [synced version] (try
- (let [synced (stage-writes commit opts writes)]
- (stage-deletes commit delete-paths)
- [synced (source.p/finish-commit! commit message)])
- (catch Throwable e
- (source.p/abort-commit! commit)
- (throw e)))]
+ (let [opts (serdes/storage-base-context)
+ [synced version] (commit-staged! snapshot message
+ (fn [commit]
+ (let [synced (stage-writes commit opts writes)]
+ (stage-deletes commit delete-paths)
+ synced)))]
(t2/with-transaction [_]
- (remote-sync.task/set-version! task-id version)
+ (when-not (= version :remote-sync/empty-commit)
+ (remote-sync.task/set-version! task-id version))
;; delete departed rows first, then update RSO metadata — same order as full-export!
(doseq [removed-ids (partition-all 500 removed-ids)]
(t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
- (mark-rows-synced! (map :id synced) synced sync-timestamp)))
- (log/infof "Remote sync incremental export: wrote %d, deleted %d" (count writes) (count delete-paths))
- {:status :success
- :outcome {:kind "pushed"
- :count (+ (count writes) (count delete-paths))
- :branch (settings/remote-sync-branch)}}))
+ (mark-rows-synced! (map :id synced) synced sync-timestamp))
+ (if (= version :remote-sync/empty-commit)
+ (do (log/info "Remote sync incremental export: nothing changed; skipped empty commit")
+ {:status :success :outcome {:kind "push-skipped"}})
+ (do
+ (log/infof "Remote sync incremental export: wrote %d, deleted %d" (count writes) (count delete-paths))
+ {:status :success
+ :outcome {:kind "pushed"
+ :count (+ (count writes) (count delete-paths))
+ :branch (settings/remote-sync-branch)}})))))
(defn export!
"Exports remote-synced collections to a remote source repository.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 7e1be88f6b5..93652085417 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -143,7 +143,7 @@
(version [_] nil))))
(defn preview-merge
- "Dry-run of [[merge-and-store!]]: computes the 3-way merge without writing anything. Returns
+ "Dry-run of the export merge: computes the 3-way merge without writing anything. Returns
`{:clean? bool :conflicts [labels] :summary {:added :updated :removed}
:force-push-casualties {:deleted [labels] :overwritten [labels]}}`. The casualties are the remote
content a force push (rather than a merge) would discard. Pass nil for `task-id` to skip progress
@@ -166,38 +166,6 @@
[stream snapshot]
(remote-sync.merge/force-push-casualties [] (serialize-specs stream nil) (snapshot->specs snapshot)))
-(defn- replace-all-files!
- "Wholesale-commit `file-specs` ({:path :content}) to `snapshot`: clear the managed dirs, stage every spec,
- and push. Returns the new version; aborts the commit on any error."
- [snapshot message file-specs]
- (let [commit (source.p/open-commit snapshot)]
- (try
- (source.p/replace-all! commit)
- (doseq [spec file-specs]
- (source.p/stage-upsert! commit spec))
- (source.p/finish-commit! commit message)
- (catch Throwable e
- (source.p/abort-commit! commit)
- (throw e)))))
-
-(defn merge-and-store!
- "Like [[store!]], but reconciles the freshly serialized local state against a remote branch that has
- advanced beyond the last sync. Performs an entity-identity 3-way merge of:
- - the merge base (`base-snapshot`, the last successfully synced state),
- - the local state (serialized from `stream`),
- - the current remote tip (`snapshot`).
-
- On a clean merge, writes the merged file set (which fast-forwards onto the remote tip) and returns
- `{:status :success :version <sha> :summary {:added :updated :removed}}`. When the same entity changed
- on both sides, returns `{:status :conflict :conflicts [..] :summary {..}}` without writing anything."
- [stream snapshot base-snapshot task-id message]
- (let [{:keys [merged conflicts summary]} (compute-merge stream snapshot base-snapshot task-id)]
- (if (seq conflicts)
- {:status :conflict :conflicts conflicts :summary summary}
- {:status :success
- :version (replace-all-files! snapshot message merged)
- :summary summary})))
-
(defn source-from-settings
"Creates a git source from the current remote sync settings.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index 01a8103e9fd..74adf9afd0f 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -18,7 +18,7 @@
DirCacheEditor$DeleteTree DirCacheEditor$PathEdit DirCacheEntry)
(org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent Ref Repository)
(org.eclipse.jgit.lib ObjectInserter ObjectReader)
- (org.eclipse.jgit.revwalk RevCommit RevWalk)
+ (org.eclipse.jgit.revwalk RevCommit RevTree RevWalk)
(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdate
RemoteRefUpdate$Status UsernamePasswordCredentialsProvider)
(org.eclipse.jgit.treewalk TreeWalk)
@@ -295,11 +295,19 @@
(.close ^ObjectReader reader)
(.close ^RevWalk rev-walk))
+(defn- written-tree-id
+ "Finalize the editor and write the staged tree, memoizing it in `tree-id` so repeated calls (e.g.
+ `empty-commit?` then `finish-commit!`) finalize and write the tree only once."
+ ^ObjectId [{:keys [^DirCacheEditor editor ^DirCache index ^ObjectInserter inserter tree-id]}]
+ (or @tree-id
+ (do (.finish editor)
+ (reset! tree-id (.writeTree index inserter)))))
+
;; A commit being built incrementally against a GitSnapshot. Holds the open JGit resources (inserter, reader,
;; rev-walk) and the in-core index/editor; blobs are inserted as files are staged and the tree is written and
;; pushed at finish. Edits the branch tip's tree in place — unchanged entries/subtrees carry forward by object
;; id — so writeTree's work is proportional to the number of changes, not the repo size.
-(defrecord GitCommit [snapshot inserter reader rev-walk index editor parent-id]
+(defrecord GitCommit [snapshot inserter reader rev-walk index editor parent-id parent-tree-id tree-id]
source.p/CommitBuilder
(stage-upsert! [_ {:keys [^String path content]}]
(let [blob-id (.insert ^ObjectInserter inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]
@@ -319,12 +327,15 @@
(.add ^DirCacheEditor editor (DirCacheEditor$DeleteTree. dir)))
nil)
- (finish-commit! [_ message]
- (.finish ^DirCacheEditor editor)
+ (empty-commit? [this]
+ (boolean (when parent-tree-id
+ (.equals (written-tree-id this) ^ObjectId parent-tree-id))))
+
+ (finish-commit! [this message]
(let [^Git git (:git snapshot)
repo (.getRepository git)
branch-ref (qualify-branch (:branch snapshot))
- tree-id (.writeTree ^DirCache index ^ObjectInserter inserter)
+ tree-id (written-tree-id this)
commit-builder (doto (CommitBuilder.)
(.setTreeId tree-id)
(.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))
@@ -348,18 +359,19 @@
(defn- open-commit*
"Begin a GitCommit against `snapshot`, seeding the in-core index from the parent tree."
[{:keys [^Git git ^String version] :as snapshot}]
- (let [repo (.getRepository git)
- parent-id (.resolve repo version)
- inserter (.newObjectInserter repo)
- reader (.newObjectReader repo)
- rev-walk (RevWalk. repo)
- index (DirCache/newInCore)]
+ (let [repo (.getRepository git)
+ parent-id (.resolve repo version)
+ inserter (.newObjectInserter repo)
+ reader (.newObjectReader repo)
+ rev-walk (RevWalk. repo)
+ index (DirCache/newInCore)
+ parent-tree (when parent-id (.getTree (.parseCommit rev-walk parent-id)))]
(let [^DirCacheBuilder builder (.builder index)]
- (when parent-id
- (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader
- (.getTree (.parseCommit rev-walk parent-id))))
+ (when parent-tree
+ (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader ^RevTree parent-tree))
(.finish builder))
- (->GitCommit snapshot inserter reader rev-walk index (.editor index) parent-id)))
+ (->GitCommit snapshot inserter reader rev-walk index (.editor index) parent-id
+ (when parent-tree (.copy ^RevTree parent-tree)) (atom nil))))
(defn branches
"Retrieves all branch names from the remote repository.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index dac3be98fb1..ad04ba0a9f9 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -81,6 +81,9 @@
"Stage removal of `path` from the commit. Returns nil.")
(replace-all! [commit]
"Clear the snapshot's managed dirs, so the staged files replace them wholesale. Returns nil.")
+ (empty-commit? [commit]
+ "True if the staged tree is identical to the parent's, i.e. finishing would introduce no changes. Always
+ false for a root commit (no parent).")
(finish-commit! [commit message]
"Write the staged tree, commit with `message`, push, and release resources. Returns the new version.")
(abort-commit! [commit]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 330fa079b52..26366fee8d9 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -1666,19 +1666,22 @@ serdes/meta:
;;; ------------------------------------- export! divergence / merge tests -------------------------------------
(defn- export-test-snapshot
- "A minimal SourceSnapshot reporting a fixed `version`."
- [version]
- (reify source.p/SourceSnapshot
- (version [_] version)
- (list-files [_] [])
- (read-file [_ _] nil)
- (open-commit [_]
- (reify source.p/CommitBuilder
- (stage-upsert! [_ _] nil)
- (stage-delete! [_ _] nil)
- (replace-all! [_] nil)
- (finish-commit! [_ _] "written-version")
- (abort-commit! [_] nil)))))
+ "A minimal SourceSnapshot reporting a fixed `version`. `empty?` controls what its commit's `empty-commit?`
+ reports, to model a staged tree that already matches the remote tip."
+ ([version] (export-test-snapshot version false))
+ ([version empty?]
+ (reify source.p/SourceSnapshot
+ (version [_] version)
+ (list-files [_] [])
+ (read-file [_ _] nil)
+ (open-commit [_]
+ (reify source.p/CommitBuilder
+ (stage-upsert! [_ _] nil)
+ (stage-delete! [_ _] nil)
+ (replace-all! [_] nil)
+ (empty-commit? [_] empty?)
+ (finish-commit! [_ _] "written-version")
+ (abort-commit! [_] nil))))))
(defn- export-test-source
"A minimal Source whose snapshot-at returns a snapshot at the requested version."
@@ -1696,8 +1699,9 @@ serdes/meta:
(let [reconciled (atom nil)]
(with-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/merge-and-store! (fn [_ _ _ _ _]
- {:status :success :version "merged-M"
+ source/compute-merge (fn [_ _ _ _]
+ {:merged [{:path "collections/x.yaml" :content "x"}]
+ :conflicts []
:summary {:added 2 :updated 1 :removed 0}})
impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}]
(reset! reconciled (source.p/version snap))
@@ -1708,9 +1712,36 @@ serdes/meta:
:base-snapshot (export-test-snapshot "base-B"))]
(is (= :success (:status result)))
(is (= {:added 2 :updated 1 :removed 0} (:merge-summary result)))
- (is (= "merged-M" @reconciled)
+ (is (= "written-version" @reconciled)
"the merged result is loaded back into the local app DB (the pull half)")
- (is (= "merged-M" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))
+ (is (= "written-version" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))
+
+(deftest export!-empty-merge-reports-pull-test
+ (testing "when the merge matches the remote tip (nothing to push), export! folds in remote changes, reports a pull, and advances to the tip"
+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
+ (let [reconciled (atom nil)]
+ (with-redefs [remote-sync.task/last-version (constantly "base-B")
+ spec/extract-entities-for-export (constantly [{:dummy true}])
+ source/compute-merge (fn [_ _ _ _]
+ {:merged [{:path "collections/x.yaml" :content "x"}]
+ :conflicts []
+ :summary {:added 1 :updated 0 :removed 0}})
+ impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}]
+ (reset! reconciled (source.p/version snap))
+ (when finalize! (finalize!)))]
+ ;; the remote-tip snapshot reports the staged tree as identical (empty-commit? true), so the merge
+ ;; pushes no commit
+ (let [result (impl/export! (export-test-snapshot "remote-R" true) task-id "msg"
+ :merge? true
+ :source (export-test-source)
+ :base-snapshot (export-test-snapshot "base-B"))]
+ (is (= :success (:status result)))
+ (is (= "pulled" (get-in result [:outcome :kind]))
+ "an empty merge pushed nothing, so it is reported as a pull rather than a merge")
+ (is (= 1 (get-in result [:outcome :count])) "the pulled count comes from the merge summary")
+ (is (= "remote-R" @reconciled) "the fold-in pull loads from the remote tip")
+ (is (= "remote-R" (:version (t2/select-one :model/RemoteSyncTask :id task-id)))
+ "the version advances to the remote tip")))))))
(deftest export!-blocks-on-genuine-conflict-test
(testing "when the same entity changed on both sides, export! returns :conflict without advancing the version or reconciling"
@@ -1718,8 +1749,8 @@ serdes/meta:
(let [reconciled? (atom false)]
(with-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/merge-and-store! (fn [_ _ _ _ _]
- {:status :conflict
+ source/compute-merge (fn [_ _ _ _]
+ {:merged []
:conflicts [{:key [["Card" "A"]]
:ours {:path "collections/a.yaml" :content "x"}
:theirs {:path "collections/a.yaml" :content "y"}}]
@@ -1747,8 +1778,9 @@ serdes/meta:
(snapshot-at [_ _] nil))]
(with-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/merge-and-store! (fn [_ _ _ _ _]
- {:status :success :version "merged-M"
+ source/compute-merge (fn [_ _ _ _]
+ {:merged [{:path "collections/x.yaml" :content "x"}]
+ :conflicts []
:summary {:added 1 :updated 0 :removed 0}})
impl/load-snapshot! (fn [_ _ _ & _] (reset! reconciled? true))]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
@@ -1778,7 +1810,7 @@ serdes/meta:
(let [merged? (atom false)]
(with-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/merge-and-store! (fn [_ _ _ _ _] (reset! merged? true) {:status :success})]
+ source/compute-merge (fn [& _] (reset! merged? true) {:merged [] :conflicts [] :summary {}})]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
:source (export-test-source)
:base-snapshot (export-test-snapshot "base-B"))]
@@ -1793,7 +1825,7 @@ serdes/meta:
(let [merged? (atom false)]
(with-redefs [remote-sync.task/last-version (constantly "base-B")
spec/exportable-entities (constantly {"Card" [999999999]}) ; absent id -> empty extraction; routing is what's under test
- source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})]
+ source/compute-merge (fn [& _] (reset! merged? true) {:merged [] :conflicts [] :summary {}})]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
:force? true
:source (export-test-source)
@@ -1812,7 +1844,7 @@ serdes/meta:
;; never reaches the merge path.
(with-redefs [remote-sync.task/last-version (constantly "remote-R")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})]
+ source/compute-merge (fn [& _] (reset! merged? true) {:merged [] :conflicts [] :summary {}})]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
:source (export-test-source)
:base-snapshot (export-test-snapshot "remote-R"))]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
index 7e2538b03cd..07a82920380 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
@@ -203,6 +203,9 @@
(deftest delete-without-stored-path-falls-back-test
(with-exported-collection!
(fn [{:keys [mock card-b]}]
+ ;; A real delete (archive) with no stored file_path can't resolve its path incrementally, so it
+ ;; falls back to full — which drops the archived entity's file, producing a real commit.
+ (t2/update! :model/Card card-b {:archived true})
(t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})
(set-status! "Card" card-b "delete")
(let [task (new-task!)
@@ -320,6 +323,9 @@
;; fresh "create" whose target path already holds THIS same entity is a safe overwrite, so it must
;; stay incremental rather than fall back (the path is occupied, but by us).
(let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]
+ ;; A real content change keeps the staged commit non-empty, isolating the path-selection behavior
+ ;; under test from the empty-commit skip.
+ (t2/update! :model/Card card-a {:description "re-created A"})
(set-status! "Card" card-a "create")
(let [task (new-task!)]
(impl/export! (source.p/snapshot mock) task "re-create A")
@@ -327,6 +333,43 @@
"a create whose path already holds the same entity stays incremental")
(is (entity-exported? mock a-eid) "card A is still exported"))))))
+;;; ----------------------------------- Empty-commit suppression (GHY-3934) -----------------------------------
+;;; A dirty row whose serialized content already matches the repo stages no net tree change. The export must
+;;; succeed without pushing an empty commit, and still clear the stale dirty flag.
+
+(deftest incremental-no-op-skips-empty-commit-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ ;; Re-"create" an already-exported card without touching its content: the staged upsert is identical
+ ;; to the repo, so the incremental commit would be empty.
+ (let [files-before (files mock)]
+ (set-status! "Card" card-a "create")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "re-create A unchanged")]
+ (is (= :success (:status result)))
+ (is (= {:kind "push-skipped"} (:outcome result)) "no empty commit is pushed")
+ (is (nil? (written-version task)) "the task version is not advanced")
+ (is (= files-before (files mock)) "the repo is left untouched")
+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))
+ "the false-dirty flag is cleared"))))))
+
+(deftest full-export-no-op-skips-empty-commit-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-b]}]
+ ;; A delete with no stored file_path forces a full export, but the entity still exists and
+ ;; re-serializes identically — the full re-export matches the repo, so the empty commit is skipped.
+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})
+ (set-status! "Card" card-b "delete")
+ (let [files-before (files mock)
+ task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "no-op full")]
+ (is (= :success (:status result)))
+ (is (= {:kind "push-skipped"} (:outcome result)) "no empty commit is pushed")
+ (is (nil? (written-version task)) "the task version is not advanced")
+ (is (= files-before (files mock)) "the repo is left untouched")
+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))
+ "the false-dirty flag is cleared")))))
+
;;; ------------------------------------ Disabled content cleanup (GHY-3725) ------------------------------------
;;; When a content type is disabled (transforms is off in `with-exported-collection!`), stale files left
;;; behind in its repo dir must be removed — but that shouldn't drop us off the incremental fast-path.
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index 156c801ee34..d346abb3cb2 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -295,6 +295,42 @@
(is (= ["Incremental" "Initial commit"]
(map :message (git/log (assoc remote :branch "master")))))))))))
+(deftest empty-commit?-detects-no-op-tree-test
+ (testing "empty-commit? is true exactly when the staged tree matches the parent commit's tree"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"collections/a.yaml" "content A"
+ "collections/b.yaml" "content B"})]
+ (testing "re-staging identical content is empty"
+ (let [c (source.p/open-commit (source.p/snapshot master))]
+ (source.p/stage-upsert! c {:path "collections/a.yaml" :content "content A"})
+ (is (true? (source.p/empty-commit? c)))
+ (source.p/abort-commit! c)))
+ (testing "staging a real content change is not empty"
+ (let [c (source.p/open-commit (source.p/snapshot master))]
+ (source.p/stage-upsert! c {:path "collections/a.yaml" :content "changed A"})
+ (is (false? (source.p/empty-commit? c)))
+ (source.p/abort-commit! c)))
+ (testing "adding a new file is not empty"
+ (let [c (source.p/open-commit (source.p/snapshot master))]
+ (source.p/stage-upsert! c {:path "collections/c.yaml" :content "content C"})
+ (is (false? (source.p/empty-commit? c)))
+ (source.p/abort-commit! c)))
+ (testing "deleting a path that isn't in the tree is empty"
+ (let [c (source.p/open-commit (source.p/snapshot master))]
+ (source.p/stage-delete! c "collections/does-not-exist.yaml")
+ (is (true? (source.p/empty-commit? c)))
+ (source.p/abort-commit! c)))
+ (testing "none of the aborted no-op checks moved the branch"
+ (is (= ["Initial commit"] (map :message (git/log master)))))
+ (testing "empty-commit? then finish-commit! still commits a real change (tree written once, memoized)"
+ (let [c (source.p/open-commit (source.p/snapshot master))]
+ (source.p/stage-upsert! c {:path "collections/a.yaml" :content "changed A"})
+ (is (false? (source.p/empty-commit? c)))
+ (source.p/finish-commit! c "Edit A"))
+ (is (= ["Edit A" "Initial commit"] (map :message (git/log master))))
+ (is (= "changed A" (source.p/read-file (source.p/snapshot master) "collections/a.yaml"))))))))
+
(deftest apply-changes-preserves-deep-unchanged-subtree-test
(testing "an incremental upsert leaves deeply-nested, unrelated subtrees untouched (carried forward by id)"
(mt/with-temp-dir [remote-dir nil]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index e3d19543dcf..c8124969751 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -26,7 +26,7 @@
(defn- entities->snapshot
"Serializes `entities` to file specs and returns a snapshot whose list-files/read-file expose them. The
snapshot also records writes into `written` (an atom) and reports a fixed version, so it can stand in for
- the remote tip in [[source/merge-and-store!]]."
+ the remote tip in the merge tests."
[entities task-id written]
(let [by-path (into {} (map (juxt :path :content)) (source/serialize-specs entities task-id))]
(reify source.p/SourceSnapshot
@@ -38,50 +38,13 @@
(stage-upsert! [_ spec] (swap! staged conj spec) nil)
(stage-delete! [_ _path] nil)
(replace-all! [_] nil)
+ (empty-commit? [_] false)
(finish-commit! [_ message]
(reset! written {:message message :files @staged})
"merged-version")
(abort-commit! [_] nil))))
(version [_] "remote-tip"))))
-(deftest merge-and-store!-clean-merge-test
- (testing "when local and remote changed different entities, merge-and-store! writes the union with no conflict"
- (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
- (let [written (atom nil)
- base [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")]
- ;; local kept A and B, added C
- ours [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")
- (create-test-entity "C" "c" "Card")]
- ;; remote kept A and B, added D
- theirs [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")
- (create-test-entity "D" "d" "Card")]
- base-snap (entities->snapshot base task-id (atom nil))
- remote-snap (entities->snapshot theirs task-id written)
- result (source/merge-and-store! ours remote-snap base-snap task-id "merge commit")]
- (is (= :success (:status result)))
- (is (= "merged-version" (:version result)))
- (is (= {:added 1 :updated 0 :removed 0} (:summary result))
- "only D is folded in from the remote")
- (testing "the union of all four entities is written"
- (let [ids (->> @written :files
- (keep #(second (re-find #"entity_id: (\w+)" (:content %))))
- sort)]
- (is (= ["A" "B" "C" "D"] ids))))))))
-
-(deftest merge-and-store!-conflict-test
- (testing "when the same entity changed on both sides, merge-and-store! returns :conflict and writes nothing"
- (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
- (let [written (atom nil)
- base [(create-test-entity "A" "a" "Card")]
- ours [(create-test-entity* "A" "a" "Card" "ours")]
- theirs [(create-test-entity* "A" "a" "Card" "theirs")]
- base-snap (entities->snapshot base task-id (atom nil))
- remote-snap (entities->snapshot theirs task-id written)
- result (source/merge-and-store! ours remote-snap base-snap task-id "merge commit")]
- (is (= :conflict (:status result)))
- (is (= 1 (count (:conflicts result))))
- (is (nil? @written) "nothing should be written when there is a conflict")))))
-
(deftest preview-merge-clean-test
(testing "preview-merge reports a clean merge and summary without writing"
(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index ddfd4b25037..2398c60a5ca 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -196,26 +196,29 @@ width: fixed
(open-commit [_this]
(let [upserts (atom [])
deletes (atom #{})
- replace-all? (atom false)]
+ replace-all? (atom false)
+ apply-staged (fn [current]
+ (let [managed (if @replace-all? (set managed-dirs) #{})]
+ (as-> (or current {}) files
+ (into {} (remove (fn [[p _]] (or (managed (top-level-dir p))
+ (contains? @deletes p))))
+ files)
+ (into files (comp (remove #(str/blank? (:path %))) (map (juxt :path :content)))
+ @upserts))))]
(reify source.p/CommitBuilder
(stage-upsert! [_ file-spec] (swap! upserts conj file-spec) nil)
(stage-delete! [_ path] (swap! deletes conj path) nil)
(replace-all! [_] (reset! replace-all? true) nil)
+ (empty-commit? [_]
+ (let [current (get @files-atom branch)]
+ (= current (apply-staged current))))
(finish-commit! [_ _message]
(case fail-mode
:write-files-error (throw (Exception. "Failed to write files"))
:store-error (throw (Exception. "Store failed"))
:apply-changes-error (throw (Exception. "Failed to apply changes"))
:network-error (throw (java.net.UnknownHostException. "Remote host not found"))
- (let [managed (if @replace-all? (set managed-dirs) #{})]
- (swap! files-atom update branch
- (fn [current]
- (as-> (or current {}) files
- (into {} (remove (fn [[p _]] (or (managed (top-level-dir p))
- (contains? @deletes p))))
- files)
- (into files (comp (remove #(str/blank? (:path %))) (map (juxt :path :content)))
- @upserts))))))
+ (swap! files-atom update branch apply-staged))
;; version string discriminates a wholesale replace (replace-all!) from an incremental patch
(if @replace-all? "write-files-version" "apply-changes-version"))
(abort-commit! [_] nil))))
@@ -307,21 +310,25 @@ width: fixed
(open-commit [_]
(let [staged-upserts (atom [])
staged-deletes (atom #{})
- replace-all? (atom false)]
+ replace-all? (atom false)
+ staged-tree (fn []
+ (let [replace-dirs (if @replace-all? managed #{})] ; `managed` = (set managed-dirs)
+ (as-> (get-in @state [:trees version] {}) t
+ (into {} (remove (fn [[p _]] (or (replace-dirs (top-level-dir p))
+ (contains? @staged-deletes p)))) t)
+ (into t (comp (remove #(str/blank? (:path %)))
+ (map (juxt :path :content)))
+ @staged-upserts))))]
(reify source.p/CommitBuilder
(stage-upsert! [_ file-spec] (swap! staged-upserts conj file-spec) nil)
(stage-delete! [_ path] (swap! staged-deletes conj path) nil)
(replace-all! [_] (reset! replace-all? true) nil)
+ (empty-commit? [_]
+ (= (get-in @state [:trees version] {}) (staged-tree)))
(finish-commit! [_ _message]
- (let [replace-dirs (if @replace-all? managed #{}) ; `managed` = (set managed-dirs)
- n (:counter (swap! state update :counter inc))
+ (let [n (:counter (swap! state update :counter inc))
new-version (str "written-" n)
- tree (as-> (get-in @state [:trees version] {}) t
- (into {} (remove (fn [[p _]] (or (replace-dirs (top-level-dir p))
- (contains? @staged-deletes p)))) t)
- (into t (comp (remove #(str/blank? (:path %)))
- (map (juxt :path :content)))
- @staged-upserts))]
+ tree (staged-tree)]
(swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))
new-version))
(abort-commit! [_] nil))))