The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Skip empty remote-sync commits after a no-op export (GHY-3934) (#76753)
Eric Normand6cc57802d45e
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 07b27426cc7..df246daa051 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -629,55 +629,86 @@(finally(analytics/observe! :metabase-remote-sync/import-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))))+(defn- commit-staged!+ "Open a commit on `snapshot`, stage into it via `stage-fn`, then finish it.++ `stage-fn` will be passed the open commit. It should return the synced write-rows.++ Will abort the commit if it is empty.++ Returns `[version, write-rows]` where version is the new commit SHA or `:remote-sync/empty-commit` if empty.++ Will abort the commit and throw on any Throwable."+ [snapshot message stage-fn]+ (let [commit (source.p/open-commit snapshot)]+ (try+ (let [synced (stage-fn commit)+ version (if (source.p/empty-commit? commit)+ (do (source.p/abort-commit! commit) :remote-sync/empty-commit)+ (source.p/finish-commit! commit message))]+ [synced version])+ (catch Throwable e+ (source.p/abort-commit! commit)+ (throw e)))))+(defn- export-merged!- "Export path taken when the remote branch has advanced beyond the last synced version (`base-snapshot`- is the merge base). Performs an entity-identity 3-way merge of local state against the remote tip:- - on a genuine conflict (same entity changed on both sides) returns a `:conflict` result;+ "Export when the remote branch has advanced beyond the last synced version. Runs an entity-identity 3-way merge of+ local state (`models`) against the remote tip:++ - on a genuine conflict (same entity changed on both sides) returns a `:conflict` result without writing- on a clean merge, writes the merged set (fast-forwarding onto the remote tip), then reconciles thelocal app DB by loading the merged result (the 'pull' half), so local now contains the remote's- changes. Returns a `:success` result with a `:merge-summary`."+ changes.++ Returns a `:success` result with a `:merge-summary`."[source snapshot base-snapshot task-id message sync-timestamp models](let [pushed-count (count (remote-sync.object/dirty-rows))- {:keys [status version conflicts summary]}- (source/merge-and-store! models snapshot base-snapshot task-id message)]- (case status- :conflict- (let [conflicts (mapv remote-sync.merge/conflict-label conflicts)]- (log/infof "Export merge conflict on %d entit(ies): %s"- (count conflicts) (str/join ", " conflicts))+ {:keys [merged conflicts summary]} (source/compute-merge models snapshot base-snapshot task-id)]+ (if (seq conflicts)+ (let [labels (mapv remote-sync.merge/conflict-label conflicts)]+ (log/infof "Export merge conflict on %d entit(ies): %s" (count labels) (str/join ", " labels)){:status :conflict:version (source.p/version snapshot)- :conflicts conflicts+ :conflicts labels:merge-summary summary:message "Export blocked: the same content was changed both locally and on the remote branch."})-- :success- ;; Fold-in pull: the merge brought remote changes that aren't in the local app DB yet. Load the- ;; merged result so local state matches what we just pushed, marking everything synced and advancing- ;; the version atomically with the reconcile (set-version! runs only after the load, so a crash leaves- ;; the version at the old base and a retry re-merges — the push is idempotent — rather than advancing- ;; the pointer past un-reconciled local state).- (if-let [merged-snapshot (source.p/snapshot-at source version)]- (do- (load-snapshot! merged-snapshot task-id sync-timestamp- :finalize! (fn []- (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})- (remote-sync.task/set-version! task-id version)))- (log/infof "Exported with merge: folded in %d remote change(s) (added %d, updated %d, removed %d)"- (apply + (vals summary)) (:added summary) (:updated summary) (:removed summary))- {:status :success :version version :merge-summary summary- :outcome {:kind "merged"- :pulled (apply + (vals summary))- :pushed pushed-count- :branch (settings/remote-sync-branch)}})- ;; The merge was pushed to `version`, but its commit can't be resolved locally (should not happen —- ;; finish-commit! updates the local ref before returning). Fail loudly rather than silently advancing- ;; the version while skipping the reconcile, which would leave local missing the folded-in remote- ;; changes. The push already landed, so a retry sees the divergence and re-merges + reconciles.- (throw (ex-info (format (str "Merge pushed to %s but its commit could not be resolved locally to "- "reconcile the app DB; re-run the export to pull the merged changes.")- version)- {:version version}))))))+ (let [[_ version] (commit-staged! snapshot message+ (fn [commit]+ (source.p/replace-all! commit) ; merged set replaces the managed dirs wholesale+ (run! #(source.p/stage-upsert! commit %) merged)))+ ;; An empty merge means the merged set already matched the remote tip: nothing was pushed, so+ ;; reconcile (and advance) against the tip rather than a non-existent merge commit.+ empty? (= version :remote-sync/empty-commit)+ version (if empty? (source.p/version snapshot) version)]+ ;; Fold-in pull: the merge brought remote changes that aren't in the local app DB yet. Load the+ ;; merged result so local state matches what we just pushed, marking everything synced and advancing+ ;; the version atomically with the reconcile (set-version! runs only after the load, so a crash leaves+ ;; the version at the old base and a retry re-merges — the push is idempotent — rather than advancing+ ;; the pointer past un-reconciled local state).+ (if-let [merged-snapshot (source.p/snapshot-at source version)]+ (let [pulled (apply + (vals summary))]+ (load-snapshot! merged-snapshot task-id sync-timestamp+ :finalize! (fn []+ (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})+ (remote-sync.task/set-version! task-id version)))+ (log/infof "Exported with merge: folded in %d remote change(s) (added %d, updated %d, removed %d); pushed %d"+ pulled (:added summary) (:updated summary) (:removed summary) (if empty? 0 pushed-count))+ {:status :success :version version :merge-summary summary+ ;; An empty merge pushed nothing: it's a pull when remote changes were folded in, or a no-op+ ;; when nothing changed on either side.+ :outcome (cond+ (not empty?) {:kind "merged" :pulled pulled :pushed pushed-count+ :branch (settings/remote-sync-branch)}+ (pos? pulled) {:kind "pulled" :count pulled :branch (settings/remote-sync-branch)}+ :else {:kind "push-skipped"})})+ ;; The merge was pushed to `version`, but its commit can't be resolved locally (should not happen —+ ;; finish-commit! updates the local ref before returning). Fail loudly rather than silently advancing+ ;; the version while skipping the reconcile, which would leave local missing the folded-in remote+ ;; changes. The push already landed, so a retry sees the divergence and re-merges + reconciles.+ (throw (ex-info (format (str "Merge pushed to %s but its commit could not be resolved locally to "+ "reconcile the app DB; re-run the export to pull the merged changes.")+ version)+ {:version version})))))));;; ------------------------------------------- Incremental Export Fast-Path -------------------------------------------@@ -699,10 +730,10 @@(its transitive `serdes/descendants` + `serdes/required`, including the entity itself)."[model-type model-id];; serdes works in [model-type id] tuples; map the closure keys to {:model_type :model_id} on the way out- (map (fn [[mt id]] {:model_type mt :model_id id})- (keys (merge-with into- (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))- (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %)))))))+ (-> #{}+ (into (keys (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))))+ (into (keys (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %)))))+ (->> (map (fn [[mt id]] {:model_type mt :model_id id})))))(defn- untracked-content-deps"The `{:model_type :model_id}` entities in `[model-type model-id]`'s export closure that are remote-sync@@ -996,48 +1027,51 @@(when (empty? export-rows)(throw (ex-info "No remote-syncable content available." {})))(remote-sync.task/update-progress! task-id 0.3)- (let [opts (serdes/storage-base-context)- commit (source.p/open-commit snapshot)- [synced version] (try- (source.p/replace-all! commit) ; replace the managed dirs wholesale- [(stage-writes commit opts export-rows)- (source.p/finish-commit! commit message)]- (catch Throwable e- (source.p/abort-commit! commit)- (throw e)))]+ (let [opts (serdes/storage-base-context)+ [synced version] (commit-staged! snapshot message+ (fn [commit]+ (source.p/replace-all! commit) ; replace the managed dirs wholesale+ (stage-writes commit opts export-rows)))](t2/with-transaction [_]- (remote-sync.task/set-version! task-id version)+ (when-not (= version :remote-sync/empty-commit)+ (remote-sync.task/set-version! task-id version))(doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))](t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))(mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp))- {:status :success- :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}})))+ (if (= version :remote-sync/empty-commit)+ (do+ (log/info "Remote sync full export: re-serialized content matches remote; skipped empty commit")+ {:status :success :outcome {:kind "push-skipped"}})+ {:status :success+ :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}}))))(defn- incremental-export disabled-files)](remote-sync.task/update-progress! task-id 0.3)- (let [opts (serdes/storage-base-context)- commit (source.p/open-commit snapshot)- [synced version] (try- (let [synced (stage-writes commit opts writes)]- (stage-deletes commit delete-paths)- [synced (source.p/finish-commit! commit message)])- (catch Throwable e- (source.p/abort-commit! commit)- (throw e)))]+ (let [opts (serdes/storage-base-context)+ [synced version] (commit-staged! snapshot message+ (fn [commit]+ (let [synced (stage-writes commit opts writes)]+ (stage-deletes commit delete-paths)+ synced)))](t2/with-transaction [_]- (remote-sync.task/set-version! task-id version)+ (when-not (= version :remote-sync/empty-commit)+ (remote-sync.task/set-version! task-id version));; delete departed rows first, then update RSO metadata — same order as full-export!(doseq [removed-ids (partition-all 500 removed-ids)](t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))- (mark-rows-synced! (map :id synced) synced sync-timestamp)))- (log/infof "Remote sync incremental export: wrote %d, deleted %d" (count writes) (count delete-paths))- {:status :success- :outcome {:kind "pushed"- :count (+ (count writes) (count delete-paths))- :branch (settings/remote-sync-branch)}}))+ (mark-rows-synced! (map :id synced) synced sync-timestamp))+ (if (= version :remote-sync/empty-commit)+ (do (log/info "Remote sync incremental export: nothing changed; skipped empty commit")+ {:status :success :outcome {:kind "push-skipped"}})+ (do+ (log/infof "Remote sync incremental export: wrote %d, deleted %d" (count writes) (count delete-paths))+ {:status :success+ :outcome {:kind "pushed"+ :count (+ (count writes) (count delete-paths))+ :branch (settings/remote-sync-branch)}})))))(defn export!"Exports remote-synced collections to a remote source repository.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex 7e1be88f6b5..93652085417 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -143,7 +143,7 @@(version [_] nil))))(defn preview-merge- "Dry-run of [[merge-and-store!]]: computes the 3-way merge without writing anything. Returns+ "Dry-run of the export merge: computes the 3-way merge without writing anything. Returns`{:clean? bool :conflicts [labels] :summary {:added :updated :removed}:force-push-casualties {:deleted [labels] :overwritten [labels]}}`. The casualties are the remotecontent a force push (rather than a merge) would discard. Pass nil for `task-id` to skip progress@@ -166,38 +166,6 @@[stream snapshot](remote-sync.merge/force-push-casualties [] (serialize-specs stream nil) (snapshot->specs snapshot)))-(defn- replace-all-files!- "Wholesale-commit `file-specs` ({:path :content}) to `snapshot`: clear the managed dirs, stage every spec,- and push. Returns the new version; aborts the commit on any error."- [snapshot message file-specs]- (let [commit (source.p/open-commit snapshot)]- (try- (source.p/replace-all! commit)- (doseq [spec file-specs]- (source.p/stage-upsert! commit spec))- (source.p/finish-commit! commit message)- (catch Throwable e- (source.p/abort-commit! commit)- (throw e)))))--(defn merge-and-store!- "Like [[store!]], but reconciles the freshly serialized local state against a remote branch that has- advanced beyond the last sync. Performs an entity-identity 3-way merge of:- - the merge base (`base-snapshot`, the last successfully synced state),- - the local state (serialized from `stream`),- - the current remote tip (`snapshot`).-- On a clean merge, writes the merged file set (which fast-forwards onto the remote tip) and returns- `{:status :success :version <sha> :summary {:added :updated :removed}}`. When the same entity changed- on both sides, returns `{:status :conflict :conflicts [..] :summary {..}}` without writing anything."- [stream snapshot base-snapshot task-id message]- (let [{:keys [merged conflicts summary]} (compute-merge stream snapshot base-snapshot task-id)]- (if (seq conflicts)- {:status :conflict :conflicts conflicts :summary summary}- {:status :success- :version (replace-all-files! snapshot message merged)- :summary summary})))-(defn source-from-settings"Creates a git source from the current remote sync settings.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex 01a8103e9fd..74adf9afd0f 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -18,7 +18,7 @@DirCacheEditor$DeleteTree DirCacheEditor$PathEdit DirCacheEntry)(org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent Ref Repository)(org.eclipse.jgit.lib ObjectInserter ObjectReader)- (org.eclipse.jgit.revwalk RevCommit RevWalk)+ (org.eclipse.jgit.revwalk RevCommit RevTree RevWalk)(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdateRemoteRefUpdate$Status UsernamePasswordCredentialsProvider)(org.eclipse.jgit.treewalk TreeWalk)@@ -295,11 +295,19 @@(.close ^ObjectReader reader)(.close ^RevWalk rev-walk))+(defn- written-tree-id+ "Finalize the editor and write the staged tree, memoizing it in `tree-id` so repeated calls (e.g.+ `empty-commit?` then `finish-commit!`) finalize and write the tree only once."+ ^ObjectId [{:keys [^DirCacheEditor editor ^DirCache index ^ObjectInserter inserter tree-id]}]+ (or @tree-id+ (do (.finish editor)+ (reset! tree-id (.writeTree index inserter)))))+;; A commit being built incrementally against a GitSnapshot. Holds the open JGit resources (inserter, reader,;; rev-walk) and the in-core index/editor; blobs are inserted as files are staged and the tree is written and;; pushed at finish. Edits the branch tip's tree in place — unchanged entries/subtrees carry forward by object;; id — so writeTree's work is proportional to the number of changes, not the repo size.-(defrecord GitCommit [snapshot inserter reader rev-walk index editor parent-id]+(defrecord GitCommit [snapshot inserter reader rev-walk index editor parent-id parent-tree-id tree-id]source.p/CommitBuilder(stage-upsert! [_ {:keys [^String path content]}](let [blob-id (.insert ^ObjectInserter inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]@@ -319,12 +327,15 @@(.add ^DirCacheEditor editor (DirCacheEditor$DeleteTree. dir)))nil)- (finish-commit! [_ message]- (.finish ^DirCacheEditor editor)+ (empty-commit? [this]+ (boolean (when parent-tree-id+ (.equals (written-tree-id this) ^ObjectId parent-tree-id))))++ (finish-commit! [this message](let [^Git git (:git snapshot)repo (.getRepository git)branch-ref (qualify-branch (:branch snapshot))- tree-id (.writeTree ^DirCache index ^ObjectInserter inserter)+ tree-id (written-tree-id this)commit-builder (doto (CommitBuilder.)(.setTreeId tree-id)(.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))@@ -348,18 +359,19 @@(defn- open-commit*"Begin a GitCommit against `snapshot`, seeding the in-core index from the parent tree."[{:keys [^Git git ^String version] :as snapshot}]- (let [repo (.getRepository git)- parent-id (.resolve repo version)- inserter (.newObjectInserter repo)- reader (.newObjectReader repo)- rev-walk (RevWalk. repo)- index (DirCache/newInCore)]+ (let [repo (.getRepository git)+ parent-id (.resolve repo version)+ inserter (.newObjectInserter repo)+ reader (.newObjectReader repo)+ rev-walk (RevWalk. repo)+ index (DirCache/newInCore)+ parent-tree (when parent-id (.getTree (.parseCommit rev-walk parent-id)))](let [^DirCacheBuilder builder (.builder index)]- (when parent-id- (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader- (.getTree (.parseCommit rev-walk parent-id))))+ (when parent-tree+ (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader ^RevTree parent-tree))(.finish builder))- (->GitCommit snapshot inserter reader rev-walk index (.editor index) parent-id)))+ (->GitCommit snapshot inserter reader rev-walk index (.editor index) parent-id+ (when parent-tree (.copy ^RevTree parent-tree)) (atom nil))))(defn branches"Retrieves all branch names from the remote repository.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.cljindex dac3be98fb1..ad04ba0a9f9 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj@@ -81,6 +81,9 @@"Stage removal of `path` from the commit. Returns nil.")(replace-all! [commit]"Clear the snapshot's managed dirs, so the staged files replace them wholesale. Returns nil.")+ (empty-commit? [commit]+ "True if the staged tree is identical to the parent's, i.e. finishing would introduce no changes. Always+ false for a root commit (no parent).")(finish-commit! [commit message]"Write the staged tree, commit with `message`, push, and release resources. Returns the new version.")(abort-commit! [commit]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 330fa079b52..26366fee8d9 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -1666,19 +1666,22 @@ serdes/meta:;;; ------------------------------------- export! divergence / merge tests -------------------------------------(defn- export-test-snapshot- "A minimal SourceSnapshot reporting a fixed `version`."- [version]- (reify source.p/SourceSnapshot- (version [_] version)- (list-files [_] [])- (read-file [_ _] nil)- (open-commit [_]- (reify source.p/CommitBuilder- (stage-upsert! [_ _] nil)- (stage-delete! [_ _] nil)- (replace-all! [_] nil)- (finish-commit! [_ _] "written-version")- (abort-commit! [_] nil)))))+ "A minimal SourceSnapshot reporting a fixed `version`. `empty?` controls what its commit's `empty-commit?`+ reports, to model a staged tree that already matches the remote tip."+ ([version] (export-test-snapshot version false))+ ([version empty?]+ (reify source.p/SourceSnapshot+ (version [_] version)+ (list-files [_] [])+ (read-file [_ _] nil)+ (open-commit [_]+ (reify source.p/CommitBuilder+ (stage-upsert! [_ _] nil)+ (stage-delete! [_ _] nil)+ (replace-all! [_] nil)+ (empty-commit? [_] empty?)+ (finish-commit! [_ _] "written-version")+ (abort-commit! [_] nil))))))(defn- export-test-source"A minimal Source whose snapshot-at returns a snapshot at the requested version."@@ -1696,8 +1699,9 @@ serdes/meta:(let [reconciled (atom nil)](with-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/merge-and-store! (fn [_ _ _ _ _]- {:status :success :version "merged-M"+ source/compute-merge (fn [_ _ _ _]+ {:merged [{:path "collections/x.yaml" :content "x"}]+ :conflicts []:summary {:added 2 :updated 1 :removed 0}})impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}](reset! reconciled (source.p/version snap))@@ -1708,9 +1712,36 @@ serdes/meta::base-snapshot (export-test-snapshot "base-B"))](is (= :success (:status result)))(is (= {:added 2 :updated 1 :removed 0} (:merge-summary result)))- (is (= "merged-M" @reconciled)+ (is (= "written-version" @reconciled)"the merged result is loaded back into the local app DB (the pull half)")- (is (= "merged-M" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))+ (is (= "written-version" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))++(deftest export!-empty-merge-reports-pull-test+ (testing "when the merge matches the remote tip (nothing to push), export! folds in remote changes, reports a pull, and advances to the tip"+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]+ (let [reconciled (atom nil)]+ (with-redefs [remote-sync.task/last-version (constantly "base-B")+ spec/extract-entities-for-export (constantly [{:dummy true}])+ source/compute-merge (fn [_ _ _ _]+ {:merged [{:path "collections/x.yaml" :content "x"}]+ :conflicts []+ :summary {:added 1 :updated 0 :removed 0}})+ impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}]+ (reset! reconciled (source.p/version snap))+ (when finalize! (finalize!)))]+ ;; the remote-tip snapshot reports the staged tree as identical (empty-commit? true), so the merge+ ;; pushes no commit+ (let [result (impl/export! (export-test-snapshot "remote-R" true) task-id "msg"+ :merge? true+ :source (export-test-source)+ :base-snapshot (export-test-snapshot "base-B"))]+ (is (= :success (:status result)))+ (is (= "pulled" (get-in result [:outcome :kind]))+ "an empty merge pushed nothing, so it is reported as a pull rather than a merge")+ (is (= 1 (get-in result [:outcome :count])) "the pulled count comes from the merge summary")+ (is (= "remote-R" @reconciled) "the fold-in pull loads from the remote tip")+ (is (= "remote-R" (:version (t2/select-one :model/RemoteSyncTask :id task-id)))+ "the version advances to the remote tip")))))))(deftest export!-blocks-on-genuine-conflict-test(testing "when the same entity changed on both sides, export! returns :conflict without advancing the version or reconciling"@@ -1718,8 +1749,8 @@ serdes/meta:(let [reconciled? (atom false)](with-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/merge-and-store! (fn [_ _ _ _ _]- {:status :conflict+ source/compute-merge (fn [_ _ _ _]+ {:merged []:conflicts [{:key [["Card" "A"]]:ours {:path "collections/a.yaml" :content "x"}:theirs {:path "collections/a.yaml" :content "y"}}]@@ -1747,8 +1778,9 @@ serdes/meta:(snapshot-at [_ _] nil))](with-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/merge-and-store! (fn [_ _ _ _ _]- {:status :success :version "merged-M"+ source/compute-merge (fn [_ _ _ _]+ {:merged [{:path "collections/x.yaml" :content "x"}]+ :conflicts []:summary {:added 1 :updated 0 :removed 0}})impl/load-snapshot! (fn [_ _ _ & _] (reset! reconciled? true))](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"@@ -1778,7 +1810,7 @@ serdes/meta:(let [merged? (atom false)](with-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/merge-and-store! (fn [_ _ _ _ _] (reset! merged? true) {:status :success})]+ source/compute-merge (fn [& _] (reset! merged? true) {:merged [] :conflicts [] :summary {}})](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg":source (export-test-source):base-snapshot (export-test-snapshot "base-B"))]@@ -1793,7 +1825,7 @@ serdes/meta:(let [merged? (atom false)](with-redefs [remote-sync.task/last-version (constantly "base-B")spec/exportable-entities (constantly {"Card" [999999999]}) ; absent id -> empty extraction; routing is what's under test- source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})]+ source/compute-merge (fn [& _] (reset! merged? true) {:merged [] :conflicts [] :summary {}})](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg":force? true:source (export-test-source)@@ -1812,7 +1844,7 @@ serdes/meta:;; never reaches the merge path.(with-redefs [remote-sync.task/last-version (constantly "remote-R")spec/extract-entities-for-export (constantly [{:dummy true}])- source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})]+ source/compute-merge (fn [& _] (reset! merged? true) {:merged [] :conflicts [] :summary {}})](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg":source (export-test-source):base-snapshot (export-test-snapshot "remote-R"))]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.cljindex 7e2538b03cd..07a82920380 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj@@ -203,6 +203,9 @@(deftest delete-without-stored-path-falls-back-test(with-exported-collection!(fn [{:keys [mock card-b]}]+ ;; A real delete (archive) with no stored file_path can't resolve its path incrementally, so it+ ;; falls back to full — which drops the archived entity's file, producing a real commit.+ (t2/update! :model/Card card-b {:archived true})(t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})(set-status! "Card" card-b "delete")(let [task (new-task!)@@ -320,6 +323,9 @@;; fresh "create" whose target path already holds THIS same entity is a safe overwrite, so it must;; stay incremental rather than fall back (the path is occupied, but by us).(let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]+ ;; A real content change keeps the staged commit non-empty, isolating the path-selection behavior+ ;; under test from the empty-commit skip.+ (t2/update! :model/Card card-a {:description "re-created A"})(set-status! "Card" card-a "create")(let [task (new-task!)](impl/export! (source.p/snapshot mock) task "re-create A")@@ -327,6 +333,43 @@"a create whose path already holds the same entity stays incremental")(is (entity-exported? mock a-eid) "card A is still exported"))))))+;;; ----------------------------------- Empty-commit suppression (GHY-3934) -----------------------------------+;;; A dirty row whose serialized content already matches the repo stages no net tree change. The export must+;;; succeed without pushing an empty commit, and still clear the stale dirty flag.++(deftest incremental-no-op-skips-empty-commit-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ ;; Re-"create" an already-exported card without touching its content: the staged upsert is identical+ ;; to the repo, so the incremental commit would be empty.+ (let [files-before (files mock)]+ (set-status! "Card" card-a "create")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "re-create A unchanged")]+ (is (= :success (:status result)))+ (is (= {:kind "push-skipped"} (:outcome result)) "no empty commit is pushed")+ (is (nil? (written-version task)) "the task version is not advanced")+ (is (= files-before (files mock)) "the repo is left untouched")+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))+ "the false-dirty flag is cleared"))))))++(deftest full-export-no-op-skips-empty-commit-test+ (with-exported-collection!+ (fn [{:keys [mock card-b]}]+ ;; A delete with no stored file_path forces a full export, but the entity still exists and+ ;; re-serializes identically — the full re-export matches the repo, so the empty commit is skipped.+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})+ (set-status! "Card" card-b "delete")+ (let [files-before (files mock)+ task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "no-op full")]+ (is (= :success (:status result)))+ (is (= {:kind "push-skipped"} (:outcome result)) "no empty commit is pushed")+ (is (nil? (written-version task)) "the task version is not advanced")+ (is (= files-before (files mock)) "the repo is left untouched")+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))+ "the false-dirty flag is cleared")))))+;;; ------------------------------------ Disabled content cleanup (GHY-3725) ------------------------------------;;; When a content type is disabled (transforms is off in `with-exported-collection!`), stale files left;;; behind in its repo dir must be removed — but that shouldn't drop us off the incremental fast-path.diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 156c801ee34..d346abb3cb2 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -295,6 +295,42 @@(is (= ["Incremental" "Initial commit"](map :message (git/log (assoc remote :branch "master")))))))))))+(deftest empty-commit?-detects-no-op-tree-test+ (testing "empty-commit? is true exactly when the staged tree matches the parent commit's tree"+ (mt/with-temp-dir [remote-dir nil]+ (let [[master _remote] (init-source! "master" remote-dir+ :files {"collections/a.yaml" "content A"+ "collections/b.yaml" "content B"})]+ (testing "re-staging identical content is empty"+ (let [c (source.p/open-commit (source.p/snapshot master))]+ (source.p/stage-upsert! c {:path "collections/a.yaml" :content "content A"})+ (is (true? (source.p/empty-commit? c)))+ (source.p/abort-commit! c)))+ (testing "staging a real content change is not empty"+ (let [c (source.p/open-commit (source.p/snapshot master))]+ (source.p/stage-upsert! c {:path "collections/a.yaml" :content "changed A"})+ (is (false? (source.p/empty-commit? c)))+ (source.p/abort-commit! c)))+ (testing "adding a new file is not empty"+ (let [c (source.p/open-commit (source.p/snapshot master))]+ (source.p/stage-upsert! c {:path "collections/c.yaml" :content "content C"})+ (is (false? (source.p/empty-commit? c)))+ (source.p/abort-commit! c)))+ (testing "deleting a path that isn't in the tree is empty"+ (let [c (source.p/open-commit (source.p/snapshot master))]+ (source.p/stage-delete! c "collections/does-not-exist.yaml")+ (is (true? (source.p/empty-commit? c)))+ (source.p/abort-commit! c)))+ (testing "none of the aborted no-op checks moved the branch"+ (is (= ["Initial commit"] (map :message (git/log master)))))+ (testing "empty-commit? then finish-commit! still commits a real change (tree written once, memoized)"+ (let [c (source.p/open-commit (source.p/snapshot master))]+ (source.p/stage-upsert! c {:path "collections/a.yaml" :content "changed A"})+ (is (false? (source.p/empty-commit? c)))+ (source.p/finish-commit! c "Edit A"))+ (is (= ["Edit A" "Initial commit"] (map :message (git/log master))))+ (is (= "changed A" (source.p/read-file (source.p/snapshot master) "collections/a.yaml"))))))))+(deftest apply-changes-preserves-deep-unchanged-subtree-test(testing "an incremental upsert leaves deeply-nested, unrelated subtrees untouched (carried forward by id)"(mt/with-temp-dir [remote-dir nil]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex e3d19543dcf..c8124969751 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -26,7 +26,7 @@(defn- entities->snapshot"Serializes `entities` to file specs and returns a snapshot whose list-files/read-file expose them. Thesnapshot also records writes into `written` (an atom) and reports a fixed version, so it can stand in for- the remote tip in [[source/merge-and-store!]]."+ the remote tip in the merge tests."[entities task-id written](let [by-path (into {} (map (juxt :path :content)) (source/serialize-specs entities task-id))](reify source.p/SourceSnapshot@@ -38,50 +38,13 @@(stage-upsert! [_ spec] (swap! staged conj spec) nil)(stage-delete! [_ _path] nil)(replace-all! [_] nil)+ (empty-commit? [_] false)(finish-commit! [_ message](reset! written {:message message :files @staged})"merged-version")(abort-commit! [_] nil))))(version [_] "remote-tip"))))-(deftest merge-and-store!-clean-merge-test- (testing "when local and remote changed different entities, merge-and-store! writes the union with no conflict"- (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]- (let [written (atom nil)- base [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")]- ;; local kept A and B, added C- ours [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")- (create-test-entity "C" "c" "Card")]- ;; remote kept A and B, added D- theirs [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")- (create-test-entity "D" "d" "Card")]- base-snap (entities->snapshot base task-id (atom nil))- remote-snap (entities->snapshot theirs task-id written)- result (source/merge-and-store! ours remote-snap base-snap task-id "merge commit")]- (is (= :success (:status result)))- (is (= "merged-version" (:version result)))- (is (= {:added 1 :updated 0 :removed 0} (:summary result))- "only D is folded in from the remote")- (testing "the union of all four entities is written"- (let [ids (->> @written :files- (keep #(second (re-find #"entity_id: (\w+)" (:content %))))- sort)]- (is (= ["A" "B" "C" "D"] ids))))))))--(deftest merge-and-store!-conflict-test- (testing "when the same entity changed on both sides, merge-and-store! returns :conflict and writes nothing"- (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]- (let [written (atom nil)- base [(create-test-entity "A" "a" "Card")]- ours [(create-test-entity* "A" "a" "Card" "ours")]- theirs [(create-test-entity* "A" "a" "Card" "theirs")]- base-snap (entities->snapshot base task-id (atom nil))- remote-snap (entities->snapshot theirs task-id written)- result (source/merge-and-store! ours remote-snap base-snap task-id "merge commit")]- (is (= :conflict (:status result)))- (is (= 1 (count (:conflicts result))))- (is (nil? @written) "nothing should be written when there is a conflict")))))-(deftest preview-merge-clean-test(testing "preview-merge reports a clean merge and summary without writing"(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex ddfd4b25037..2398c60a5ca 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -196,26 +196,29 @@ width: fixed(open-commit [_this](let [upserts (atom [])deletes (atom #{})- replace-all? (atom false)]+ replace-all? (atom false)+ apply-staged (fn [current]+ (let [managed (if @replace-all? (set managed-dirs) #{})]+ (as-> (or current {}) files+ (into {} (remove (fn [[p _]] (or (managed (top-level-dir p))+ (contains? @deletes p))))+ files)+ (into files (comp (remove #(str/blank? (:path %))) (map (juxt :path :content)))+ @upserts))))](reify source.p/CommitBuilder(stage-upsert! [_ file-spec] (swap! upserts conj file-spec) nil)(stage-delete! [_ path] (swap! deletes conj path) nil)(replace-all! [_] (reset! replace-all? true) nil)+ (empty-commit? [_]+ (let [current (get @files-atom branch)]+ (= current (apply-staged current))))(finish-commit! [_ _message](case fail-mode:write-files-error (throw (Exception. "Failed to write files")):store-error (throw (Exception. "Store failed")):apply-changes-error (throw (Exception. "Failed to apply changes")):network-error (throw (java.net.UnknownHostException. "Remote host not found"))- (let [managed (if @replace-all? (set managed-dirs) #{})]- (swap! files-atom update branch- (fn [current]- (as-> (or current {}) files- (into {} (remove (fn [[p _]] (or (managed (top-level-dir p))- (contains? @deletes p))))- files)- (into files (comp (remove #(str/blank? (:path %))) (map (juxt :path :content)))- @upserts))))))+ (swap! files-atom update branch apply-staged));; version string discriminates a wholesale replace (replace-all!) from an incremental patch(if @replace-all? "write-files-version" "apply-changes-version"))(abort-commit! [_] nil))))@@ -307,21 +310,25 @@ width: fixed(open-commit [_](let [staged-upserts (atom [])staged-deletes (atom #{})- replace-all? (atom false)]+ replace-all? (atom false)+ staged-tree (fn []+ (let [replace-dirs (if @replace-all? managed #{})] ; `managed` = (set managed-dirs)+ (as-> (get-in @state [:trees version] {}) t+ (into {} (remove (fn [[p _]] (or (replace-dirs (top-level-dir p))+ (contains? @staged-deletes p)))) t)+ (into t (comp (remove #(str/blank? (:path %)))+ (map (juxt :path :content)))+ @staged-upserts))))](reify source.p/CommitBuilder(stage-upsert! [_ file-spec] (swap! staged-upserts conj file-spec) nil)(stage-delete! [_ path] (swap! staged-deletes conj path) nil)(replace-all! [_] (reset! replace-all? true) nil)+ (empty-commit? [_]+ (= (get-in @state [:trees version] {}) (staged-tree)))(finish-commit! [_ _message]- (let [replace-dirs (if @replace-all? managed #{}) ; `managed` = (set managed-dirs)- n (:counter (swap! state update :counter inc))+ (let [n (:counter (swap! state update :counter inc))new-version (str "written-" n)- tree (as-> (get-in @state [:trees version] {}) t- (into {} (remove (fn [[p _]] (or (replace-dirs (top-level-dir p))- (contains? @staged-deletes p)))) t)- (into t (comp (remove #(str/blank? (:path %)))- (map (juxt :path :content)))- @staged-upserts))]+ tree (staged-tree)](swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))new-version))(abort-commit! [_] nil))))