The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-4737: Remote sync: name the cause of a diverged-export conflict (#83682)
bryan1c0661fa03a1
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex f02405f45b8..ef2c83c5803 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -662,6 +662,7 @@{:status :conflict:version snapshot-version:conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]+ :outcome {:kind "history-rewritten"}:message "Cannot merge: the remote branch history was rewritten. Discard local changes and pull, or push to a new branch."};; Remote hasn't advanced past the merge base — nothing to fold in; keep local changes dirty.@@ -1277,6 +1278,7 @@{:status :conflict:version remote-version:conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]+ :outcome {:kind "history-rewritten"}:message "Cannot merge: the remote branch history was rewritten. Re-import then export, or force the export to overwrite."}:else@@ -1288,7 +1290,9 @@diverged? ;; and not merge? option{:status :conflict:version remote-version+ ;; Nothing collided: the divergence itself is why it stopped, so the cause rides in `:outcome`.:conflicts []+ :outcome {:kind "remote-changed"}:message "The remote branch has changed since your last sync. Choose how to proceed."};; There's nothing to export: no dirty rows and no stale files.@@ -1473,7 +1477,7 @@(remote-sync.task/complete-sync-task! task-id (:outcome result))):conflict (do(remote-sync.task/set-version! task-id (:version result))- (remote-sync.task/conflict-sync-task! task-id (:conflicts result)))+ (remote-sync.task/conflict-sync-task! task-id (:conflicts result) (:outcome result))):error (remote-sync.task/fail-sync-task! task-id (:message result))(remote-sync.task/fail-sync-task! task-id "Unexpected Error"))true))))]diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.cljindex d8093e3af71..76fd6574c65 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj@@ -295,13 +295,15 @@(defn conflict-sync-task!"Marks a sync task as having conflicts.- Takes the ID of the sync task and a collection of conflict category names (`import!` reports them as a- set). Conflicts are automatically serialized to JSON via the model transform.+ Takes the ID of the sync task, a collection of what collided (`import!` reports category names as a set), and the+ `:outcome` naming why the task stopped when that is not a collision (`nil` otherwise). Both are serialized to+ JSON via the model transform.Returns the number of rows updated (should be 1 if successful)."- [task-id conflicts]+ [task-id conflicts outcome](remote-sync.db/end-task! task-id- {:conflicts (vec conflicts)}))+ {:conflicts (vec conflicts)+ :outcome outcome}));;; ------------------------------------------- Hydration -------------------------------------------diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.cljindex d9c73e77200..4c51caef1b5 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj@@ -216,7 +216,12 @@[:kind [:= "merged"]][:pulled :int][:pushed :int]- [:branch [:maybe :string]]]]])+ [:branch [:maybe :string]]]]+ ;; Conflict outcomes: why a task stopped when the cause is not a collision (its `:conflicts` say what collided).+ ["remote-changed" [:map {:closed true}+ [:kind [:= "remote-changed"]]]]+ ["history-rewritten" [:map {:closed true}+ [:kind [:= "history-rewritten"]]]]])(mr/def ::remote-sync-task"A RemoteSyncTask as selected from the app DB: every column of `:remote_sync_task`."diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.cljindex 83c9d2dfb00..eb20b796f0f 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj@@ -25,9 +25,21 @@source (source/source-from-settings branch)snapshot (source.p/snapshot source)snapshot-version (source.p/version snapshot)- last-version (remote-sync.task/last-version)]- (if (= last-version snapshot-version)+ last-version (remote-sync.task/last-version)+ newest-task (remote-sync.task/most-recent-task)]+ (cond+ (= last-version snapshot-version)(log/infof "Skipping auto-import: source version %s matches last imported version" snapshot-version)++ ;; A conflict is not a sync base, so without this an unresolved conflict would be retried, and recorded as a+ ;; new conflict task, on every tick until the remote moves or someone resolves it.+ (and (remote-sync.task/conflict? newest-task)+ (= (:version newest-task) snapshot-version))+ (log/infof (str "Skipping auto-import: the last task conflicted at source version %s; "+ "waiting for a new commit or a manual import")+ snapshot-version)++ :else(let [{task-id :id existing? :existing?} (impl/create-task-with-lock! "import")](if existing?(log/info "Remote sync already in progress, not auto-importing")diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex bb08537f666..8dd222ca6a0 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -1945,6 +1945,7 @@ serdes/meta::source (export-test-source):base-snapshot nil)](is (= :conflict (:status result)))+ (is (= {:kind "history-rewritten"} (:outcome result)))(is (str/includes? (:message result) "rewritten")))))))(deftest export!-refuses-when-diverged-without-merge-flag-test@@ -1958,10 +1959,26 @@ serdes/meta::source (export-test-source):base-snapshot (export-test-snapshot "base-B"))](is (= :conflict (:status result)))+ (is (= {:conflicts [] :outcome {:kind "remote-changed"}}+ (select-keys result [:conflicts :outcome]))+ "nothing collided; the outcome names why it stopped")(is (false? @merged?) "no merge without the merge flag");; :conflict short-circuits before any write — the version is never advanced(is (nil? (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))+(deftest diverged-export-conflict-keeps-sync-base-test+ (testing "a diverged export that ends in conflict leaves the sync base alone, so a retry with merge? still merges"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import" :version "base-B" :ended_at (t/offset-date-time)}+ :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]+ (let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"+ :source (export-test-source)+ :base-snapshot (export-test-snapshot "base-B"))]+ (impl/handle-task-result! result task-id)+ (is (= {:version "remote-R" :conflicts [] :outcome {:kind "remote-changed"}}+ (t2/select-one [:model/RemoteSyncTask :version :conflicts :outcome] :id task-id))+ "the task row records the remote version it conflicted against and why it stopped")+ (is (= "base-B" (remote-sync.task/last-version)))))))+(deftest export!-force-overwrites-without-merging-test(testing "force? overwrites the remote wholesale (full export) even when it advanced — no merge"(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]@@ -2184,6 +2201,7 @@ serdes/meta::merge? true:base-snapshot nil)](is (= :conflict (:status result)))+ (is (= {:kind "history-rewritten"} (:outcome result)))(is (str/includes? (:message result) "rewritten"))))));;; ------------------------------- merging pull and push extract the library once -------------------------------diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.cljindex 2c9d21d6800..8c7eb4a1f8a 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj@@ -521,7 +521,7 @@(testing "Ignores a conflict task even though it records the version it conflicted against"(let [conflict-task (rst/create-sync-task! "import" (mt/user->id :rasta))](rst/set-version! (:id conflict-task) "version 1.5")- (rst/conflict-sync-task! (:id conflict-task) ["some conflict"])+ (rst/conflict-sync-task! (:id conflict-task) ["some conflict"] nil)(is (= "version 1" (rst/last-version)))))(testing "Returns a newer successful task's version"(let [new-task (rst/create-sync-task! "import" (mt/user->id :rasta))]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/task/import_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/task/import_test.cljindex a14666679de..b8f5ceb6577 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/task/import_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/task/import_test.clj@@ -1,7 +1,9 @@(ns metabase-enterprise.remote-sync.task.import-test(:require[clojure.test :refer :all]+ [metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.source :as source]+ [metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.task.import :as task.import][metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase.test :as mt]@@ -43,3 +45,23 @@(testing "a no-op run (source version unchanged) does not log another entry"(#'task.import/auto-import!)(is (= (inc before) (t2/count :model/AuditLog :topic "remote-sync-import")))))))))++(deftest auto-import-does-not-repeat-an-unresolved-conflict-test+ (testing "GHY-4737: an auto-import that conflicted is not retried, as a new conflict task, at the same source version"+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-type :read-only+ remote-sync-auto-import true]+ (mt/with-dynamic-fn-redefs [source/source-from-settings (fn [& _] (test-helpers/create-mock-source))+ impl/import! (fn [snapshot & _]+ {:status :conflict+ :version (source.p/version snapshot)+ :conflicts ["library"]})]+ (let [before (t2/count :model/RemoteSyncTask)]+ (#'task.import/auto-import!)+ (is (= (inc before) (t2/count :model/RemoteSyncTask))+ "the first tick records the conflict")+ (#'task.import/auto-import!)+ (is (= (inc before) (t2/count :model/RemoteSyncTask))+ "the next tick at the same source version skips instead of recording another conflict"))))))diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.tsindex 2fd456ab97a..048a23d068c 100644--- a/frontend/src/metabase-types/api/remote-sync.ts+++ b/frontend/src/metabase-types/api/remote-sync.ts@@ -216,7 +216,10 @@ export type RemoteSyncOutcome =| { kind: "pull-skipped" }| { kind: "pushed"; count: number; branch: string }| { kind: "push-skipped" }- | { kind: "merged"; pulled: number; pushed: number; branch: string };+ | { kind: "merged"; pulled: number; pushed: number; branch: string }+ // Conflict outcomes: why a task stopped when the cause is not a collision.+ | { kind: "remote-changed" }+ | { kind: "history-rewritten" };export type RemoteSyncTaskUser = {id: UserId;