Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix Remote Sync push hang: honest stalled UI + meaningful export progress (GHY-4132) (#77904)

Eric Normandf0ce91f74c89
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 57402f35c97..6d39394071e 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -668,24 +668,27 @@
(defn- commit-staged!
"Open a commit on `snapshot`, stage into it via `stage-fn`, then finish it.
- `stage-fn` will be passed the open commit. It should return the synced write-rows.
+ `stage-fn` will be passed the open commit. It should return the synced write-rows. `report-progress`, when
+ non-nil, is a reporter fn forwarded unchanged to `finish-commit!` (fired forced at the commit checkpoint,
+ before the push, then throttled during the push itself).
Will abort the commit if it is empty.
Returns `[version, write-rows]` where version is the new commit SHA or `:remote-sync/empty-commit` if empty.
Will abort the commit and throw on any Throwable."
- [snapshot message stage-fn]
- (let [commit (source.p/open-commit snapshot)]
- (try
- (let [synced (stage-fn commit)
- version (if (source.p/empty-commit? commit)
- (do (source.p/abort-commit! commit) :remote-sync/empty-commit)
- (source.p/finish-commit! commit message))]
- [synced version])
- (catch Throwable e
- (source.p/abort-commit! commit)
- (throw e)))))
+ ([snapshot message stage-fn] (commit-staged! snapshot message stage-fn nil))
+ ([snapshot message stage-fn report-progress]
+ (let [commit (source.p/open-commit snapshot)]
+ (try
+ (let [synced (stage-fn commit)
+ version (if (source.p/empty-commit? commit)
+ (do (source.p/abort-commit! commit) :remote-sync/empty-commit)
+ (source.p/finish-commit! commit message report-progress))]
+ [synced version])
+ (catch Throwable e
+ (source.p/abort-commit! commit)
+ (throw e))))))
(defn- export-merged!
"Export when the remote branch has advanced beyond the last synced version. Runs an entity-identity 3-way merge of
@@ -996,13 +999,22 @@
"Stage WriteRows ({:model_type :model_id, optional :id/:file_path}) to `commit`, chunking internally per
model type so each chunk's entities load in one query with bounded memory. Callers pass a flat seq.
+ `on-chunk`, when non-nil, is called once per staged chunk with the cumulative number of rows staged so
+ far — used to drive serialize-phase progress. It must not realize `rows`.
+
Returns:
- [{:id :file_path :content_hash}]"
- [commit opts rows]
- (->> rows
- (->sized-chunks)
- (mapcat #(chunk-stage-writes commit opts %))
- (doall)))
+ ([commit opts rows] (stage-writes commit opts rows nil))
+ ([commit opts rows on-chunk]
+ (let [staged (volatile! 0)]
+ (->> rows
+ (->sized-chunks)
+ (mapcat (fn [chunk]
+ (let [res (chunk-stage-writes commit opts chunk)]
+ (vswap! staged + (count (:rows chunk)))
+ (when on-chunk (on-chunk @staged))
+ res)))
+ (doall)))))
(defn- stage-deletes [commit delete-paths]
(doseq [delete-path delete-paths]
@@ -1052,6 +1064,9 @@
hits)
[:else :content_hash]))))))))
+(def ^:private export-progress-plan-done 0.33) ; phase 1 (plan) complete / serialize start
+(def ^:private export-progress-serialize 0.66) ; phase 2 (serialize) complete
+
(defn- full-export!
"Re-serialize and commit the entire remote-synced set, then reconcile every RemoteSyncObject.
@@ -1059,39 +1074,56 @@
- {:status :success}
or throws"
[snapshot task-id message sync-timestamp]
- (let [export-rows (exportable-write-rows)]
+ (let [export-rows (vec (exportable-write-rows))]
(when (empty? export-rows)
(throw (ex-info "No remote-syncable content available." {})))
- (remote-sync.task/update-progress! task-id 0.3)
- (let [opts (serdes/storage-base-context)
- [synced version] (commit-staged! snapshot message
- (fn [commit]
- (source.p/replace-all! commit) ; replace the managed dirs wholesale
- (stage-writes commit opts export-rows)))]
- (t2/with-transaction [_]
- (when-not (= version :remote-sync/empty-commit)
- (remote-sync.task/set-version! task-id version))
- (doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))]
- (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
- (mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp))
- (if (= version :remote-sync/empty-commit)
- (do
- (log/info "Remote sync full export: re-serialized content matches remote; skipped empty commit")
- {:status :success :outcome {:kind "push-skipped"}})
- {:status :success
- :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}}))))
+ (let [report (remote-sync.task/make-progress-reporter task-id)
+ total (count export-rows)
+ span (- export-progress-serialize export-progress-plan-done)]
+ (report export-progress-plan-done {:force? true})
+ (let [opts (serdes/storage-base-context)
+ [synced version] (commit-staged! snapshot message
+ (fn [commit]
+ (source.p/replace-all! commit) ; replace the managed dirs wholesale
+ (let [synced (stage-writes commit opts export-rows
+ (fn [staged]
+ (report (+ export-progress-plan-done
+ (* span (/ staged total))))))]
+ (report export-progress-serialize {:force? true})
+ synced))
+ report)]
+ (t2/with-transaction [_]
+ (when-not (= version :remote-sync/empty-commit)
+ (remote-sync.task/set-version! task-id version))
+ (doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))]
+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
+ (mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp))
+ (if (= version :remote-sync/empty-commit)
+ (do
+ (log/info "Remote sync full export: re-serialized content matches remote; skipped empty commit")
+ {:status :success :outcome {:kind "push-skipped"}})
+ {:status :success
+ :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}})))))
(defn- incremental-export!
[plan disabled-files task-id snapshot message sync-timestamp]
(let [{:keys [writes delete-paths removed-ids]} plan
- delete-paths (into (vec delete-paths) disabled-files)]
- (remote-sync.task/update-progress! task-id 0.3)
+ delete-paths (into (vec delete-paths) disabled-files)
+ report (remote-sync.task/make-progress-reporter task-id)
+ total (max 1 (count writes))
+ span (- export-progress-serialize export-progress-plan-done)]
+ (report export-progress-plan-done {:force? true})
(let [opts (serdes/storage-base-context)
[synced version] (commit-staged! snapshot message
(fn [commit]
- (let [synced (stage-writes commit opts writes)]
+ (let [synced (stage-writes commit opts writes
+ (fn [staged]
+ (report (+ export-progress-plan-done
+ (* span (/ staged total))))))]
(stage-deletes commit delete-paths)
- synced)))]
+ (report export-progress-serialize {:force? true})
+ synced))
+ report)]
(t2/with-transaction [_]
(when-not (= version :remote-sync/empty-commit)
(remote-sync.task/set-version! task-id version))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
index 56bda997ac0..c26a1b3deba 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
@@ -84,6 +84,39 @@
{:progress progress
:last_progress_report_at (mi/now)}))
+(def ^:private default-progress-throttle-ms
+ "Minimum ms between throttled (non-boundary) progress writes."
+ 10000)
+
+(defn make-progress-reporter
+ "Returns a stateful progress reporter for `task-id`.
+
+ Call the returned fn with a fraction in [0.0, 1.0]. It writes progress (and bumps
+ `last_progress_report_at`) at most once per throttle window and never moves the fraction backward.
+ Pass `{:force? true}` to write immediately regardless of the throttle — use at phase boundaries.
+
+ Options:
+ - :throttle-ms minimum ms between throttled writes (default 10000)
+ - :now-fn 0-arg fn returning current millis (default `System/currentTimeMillis`)
+ - :write-fn 1-arg fn taking the clamped fraction (default `update-progress!` for `task-id`)"
+ ([task-id] (make-progress-reporter task-id nil))
+ ([task-id {:keys [throttle-ms now-fn write-fn]
+ :or {throttle-ms default-progress-throttle-ms}}]
+ (let [now-fn (or now-fn #(System/currentTimeMillis))
+ write-fn (or write-fn (fn [f] (update-progress! task-id f)))
+ last-ms (volatile! nil)
+ last-fraction (volatile! -1.0)]
+ (fn report!
+ ([fraction] (report! fraction nil))
+ ([fraction {:keys [force?]}]
+ (let [now (now-fn)
+ f (-> (double fraction) (max 0.0) (min 1.0))]
+ (when (and (>= f @last-fraction)
+ (or force? (nil? @last-ms) (>= (- now @last-ms) throttle-ms)))
+ (vreset! last-ms now)
+ (vreset! last-fraction f)
+ (write-fn f))))))))
+
(defn set-version!
"Sets the version value for a sync task.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index 74adf9afd0f..05b151d0351 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -16,7 +16,7 @@
(org.eclipse.jgit.api Git GitCommand TransportCommand)
(org.eclipse.jgit.dircache DirCache DirCacheBuilder DirCacheEditor DirCacheEditor$DeletePath
DirCacheEditor$DeleteTree DirCacheEditor$PathEdit DirCacheEntry)
- (org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent Ref Repository)
+ (org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent ProgressMonitor Ref Repository)
(org.eclipse.jgit.lib ObjectInserter ObjectReader)
(org.eclipse.jgit.revwalk RevCommit RevTree RevWalk)
(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdate
@@ -251,27 +251,75 @@
(recur (update acc bucket conj (.getPathString tw))))
acc)))))))
+(def ^:private commit-progress-checkpoint
+ "Export progress fraction reported once the local commit is durable, just before the network push begins."
+ 0.8)
+
+(def ^:private push-progress-start
+ "Progress fraction at which the network push begins."
+ 0.8)
+
+(def ^:private push-progress-end
+ "Progress fraction the network push approaches as it completes; the final 1.0 is reported elsewhere."
+ 0.99)
+
+(defn- ->push-progress-monitor
+ "A JGit ProgressMonitor that maps the client-side \"Writing objects\" phase onto
+ [push-progress-start, push-progress-end] and calls `report-progress` (a 1-arg fraction fn) on every
+ `update` tick — even outside the writing phase, falling back to `push-progress-start` — so the push
+ always heartbeats regardless of JVM locale (which can rename or suppress the \"Writing objects\" title)
+ or an unknown (zero) total. Upstream throttling/monotonicity is handled by the reporter, so the repeated
+ fallback values are cheap and safe."
+ ^ProgressMonitor [report-progress]
+ (let [writing? (volatile! false)
+ total (volatile! 0)
+ done (volatile! 0)
+ report! (fn []
+ (report-progress
+ (if (and @writing? (pos? @total))
+ (+ push-progress-start
+ (* (- push-progress-end push-progress-start)
+ (min 1.0 (/ (double @done) @total))))
+ push-progress-start)))]
+ (reify ProgressMonitor
+ (start [_ _total-tasks])
+ (beginTask [_ title tot]
+ (vreset! writing? (= title "Writing objects"))
+ (vreset! total (max 0 tot))
+ (vreset! done 0)
+ (report!))
+ (update [_ completed]
+ (vswap! done + completed)
+ (report!))
+ (endTask [_]
+ (vreset! writing? false))
+ (isCancelled [_] false)
+ (showDuration [_ _]))))
+
(defn push-branch!
- "Pushes a local branch to the remote repository.
+ "Pushes a local branch to the remote repository. Optional `progress-monitor` (a JGit ProgressMonitor)
+ reports push progress.
Takes a git-source map containing a :git Git instance, :branch, and optional :token for
authentication. Uses the 'origin' remote which is configured by ensure-origin-configured!.
Returns the push response from JGit. Throws ExceptionInfo if the push operation fails or returns a
non-OK/UP_TO_DATE status."
- [{:keys [^Git git ^String branch] :as git-source}]
- (let [branch-name (qualify-branch branch)
- push-response (call-remote-command
- (-> (.push git)
- (.setRefSpecs (doto (java.util.ArrayList.)
- (.add (RefSpec. (str branch-name ":" branch-name))))))
- git-source)
- push-results (->> push-response
- (map #(into [] (.getRemoteUpdates ^PushResult %)))
- flatten)]
- (when-let [failures (seq (remove #(#{RemoteRefUpdate$Status/OK RemoteRefUpdate$Status/UP_TO_DATE} %) (map #(.getStatus ^RemoteRefUpdate %) push-results)))]
- (throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))
- push-response))
+ ([git-source] (push-branch! git-source nil))
+ ([{:keys [^Git git ^String branch] :as git-source} ^ProgressMonitor progress-monitor]
+ (let [branch-name (qualify-branch branch)
+ push-cmd (cond-> (-> (.push git)
+ (.setRefSpecs (doto (java.util.ArrayList.)
+ (.add (RefSpec. (str branch-name ":" branch-name))))))
+ progress-monitor (.setProgressMonitor progress-monitor))
+ push-response (call-remote-command push-cmd git-source)
+ push-results (->> push-response
+ (map #(into [] (.getRemoteUpdates ^PushResult %)))
+ flatten)]
+ (when-let [failures (seq (remove #(#{RemoteRefUpdate$Status/OK RemoteRefUpdate$Status/UP_TO_DATE} %)
+ (map #(.getStatus ^RemoteRefUpdate %) push-results)))]
+ (throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))
+ push-response)))
(defn default-branch
"Retrieves the default branch name of the git repository.
@@ -332,6 +380,9 @@
(.equals (written-tree-id this) ^ObjectId parent-tree-id))))
(finish-commit! [this message]
+ (source.p/finish-commit! this message nil))
+
+ (finish-commit! [this message report-progress]
(let [^Git git (:git snapshot)
repo (.getRepository git)
branch-ref (qualify-branch (:branch snapshot))
@@ -348,7 +399,9 @@
(doto (.updateRef repo branch-ref)
(.setNewObjectId commit-id)
(.update))
- (push-branch! snapshot)
+ ;; local commit durable; push about to start — force this one-shot checkpoint past the throttle
+ (when report-progress (report-progress commit-progress-checkpoint {:force? true}))
+ (push-branch! snapshot (when report-progress (->push-progress-monitor report-progress)))
(close-commit-resources! inserter reader rev-walk) ; close only after a successful push
(.name commit-id))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index ad04ba0a9f9..342e3de2b43 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -84,8 +84,15 @@
(empty-commit? [commit]
"True if the staged tree is identical to the parent's, i.e. finishing would introduce no changes. Always
false for a root commit (no parent).")
- (finish-commit! [commit message]
- "Write the staged tree, commit with `message`, push, and release resources. Returns the new version.")
+ (finish-commit!
+ [commit message]
+ [commit message report-progress]
+ "Write the staged tree, commit with `message`, push, and release resources. Returns the new version.
+
+ `report-progress`, when given, is an optional reporter fn accepting `(fraction)` or `(fraction opts)`;
+ nil to skip. It is called once, forced, at the commit checkpoint just after the local commit is durable
+ and before the network push begins, and then repeatedly (throttled) by the push's ProgressMonitor as
+ the push proceeds.")
(abort-commit! [commit]
"Release the commit's resources without committing or pushing. Returns nil."))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj
new file mode 100644
index 00000000000..8ae8cd4e496
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj
@@ -0,0 +1,51 @@
+(ns metabase-enterprise.remote-sync.impl-progress-test
+ "Serialize-phase progress hook for remote-sync export (GHY-4132)."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.source.protocol :as source.p]))
+
+(set! *warn-on-reflection* true)
+
+;; A CommitBuilder that records staged paths and does nothing else.
+(defrecord RecordingCommit [staged]
+ source.p/CommitBuilder
+ (stage-upsert! [_ {:keys [path]}] (swap! staged conj path) nil)
+ (stage-delete! [_ _] nil)
+ (replace-all! [_] nil)
+ (empty-commit? [_] false)
+ (finish-commit! [_ _message] "sha")
+ (abort-commit! [_] nil))
+
+(deftest stage-writes-invokes-on-chunk-with-cumulative-count-test
+ (testing "stage-writes calls on-chunk once per chunk with the running staged count, and stages everything"
+ (with-redefs [;; make each row its own chunk (matching the real {:model_type :rows} chunk shape) and a
+ ;; trivial entity, so we exercise the loop deterministically without hitting serdes/the app DB
+ impl/->sized-chunks (fn [rows] (map (fn [row] {:model_type (:model_type row) :rows [row]}) rows))
+ impl/extract-chunk (fn [{:keys [rows]}] (map (fn [row] [row {:e (:model_id row)}]) rows))]
+ (let [staged (atom [])
+ commit (->RecordingCommit staged)
+ rows [{:model_type "card" :model_id 1 :file_path "a"}
+ {:model_type "card" :model_id 2 :file_path "b"}
+ {:model_type "card" :model_id 3 :file_path "c"}]
+ seen (atom [])]
+ (#'impl/stage-writes commit {} rows (fn [n] (swap! seen conj n)))
+ (is (= [1 2 3] @seen) "cumulative staged count reported once per chunk")
+ (is (= ["a" "b" "c"] @staged) "all rows staged")))))
+
+(deftest finish-commit-reports-commit-checkpoint-test
+ (testing "commit-staged! passes the progress callback to finish-commit!, invoked at the commit checkpoint"
+ (let [reported (atom [])
+ commit (reify source.p/CommitBuilder
+ (stage-upsert! [_ _] nil)
+ (stage-delete! [_ _] nil)
+ (replace-all! [_] nil)
+ (empty-commit? [_] false)
+ (finish-commit! [_ _message] "sha")
+ (finish-commit! [_ _message report] (when report (report 0.8 {:force? true})) "sha")
+ (abort-commit! [_] nil))]
+ (with-redefs [source.p/open-commit (fn [_] commit)]
+ (#'impl/commit-staged! {:managed-dirs []} "msg"
+ (fn [_c] [{:id 1}])
+ (fn [f & _] (swap! reported conj f))))
+ (is (= [0.8] @reported)))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 8fee1791581..d5cf43bbfed 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -483,7 +483,7 @@
(is (= :success (:status result)))
(is (pos? (count @progress-calls)))
(is (= task-id (:task-id (first @progress-calls))))
- (is (= 0.3 (:progress (first @progress-calls)))))))))))))
+ (is (= 0.33 (:progress (first @progress-calls)))))))))))))
(deftest import!-resets-remote-sync-object-table-test
(testing "import! deletes and recreates RemoteSyncObject table with synced status"
@@ -1794,6 +1794,7 @@ serdes/meta:
(replace-all! [_] nil)
(empty-commit? [_] empty?)
(finish-commit! [_ _] "written-version")
+ (finish-commit! [_ _ report-progress] (when report-progress (report-progress 0.8)) "written-version")
(abort-commit! [_] nil))))))
(defn- export-test-source
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj
index 98754266ca5..cf4245c307e 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj
@@ -472,3 +472,42 @@
(is (false? (:cancelled after)))
(is (= (:ended_at before) (:ended_at after))
"ended_at must not be overwritten")))))
+
+;;; ------------------------------------------------------------------------------------------------
+;;; Tests for make-progress-reporter
+;;; ------------------------------------------------------------------------------------------------
+
+(deftest make-progress-reporter-throttles-and-forces-test
+ (testing "intra-phase writes are throttled; :force? bypasses the throttle; boundaries always land"
+ (let [now (atom 0)
+ writes (atom [])
+ report (rst/make-progress-reporter 1 {:throttle-ms 10000
+ :now-fn (fn [] @now)
+ :write-fn (fn [f] (swap! writes conj f))})]
+ (report 0.33 {:force? true}) ; boundary — always writes
+ (report 0.40) ; 0ms since last throttled anchor set by forced write -> throttled out
+ (reset! now 5000) (report 0.50) ; 5s < 10s -> throttled out
+ (reset! now 10000) (report 0.60) ; 10s -> writes
+ (report 0.66 {:force? true}) ; boundary -> writes
+ (is (= [0.33 0.60 0.66] @writes)))))
+
+(deftest make-progress-reporter-is-monotonic-test
+ (testing "the reporter never writes a fraction lower than the last written"
+ (let [now (atom 0)
+ writes (atom [])
+ report (rst/make-progress-reporter 1 {:throttle-ms 0
+ :now-fn (fn [] @now)
+ :write-fn (fn [f] (swap! writes conj f))})]
+ (report 0.5)
+ (report 0.4) ; lower -> dropped
+ (report 0.6)
+ (is (= [0.5 0.6] @writes)))))
+
+(deftest make-progress-reporter-clamps-test
+ (testing "fractions are clamped into [0.0, 1.0]"
+ (let [writes (atom [])
+ report (rst/make-progress-reporter 1 {:throttle-ms 0
+ :now-fn (constantly 0)
+ :write-fn (fn [f] (swap! writes conj f))})]
+ (report 1.5)
+ (is (= [1.0] @writes)))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_progress_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_progress_test.clj
new file mode 100644
index 00000000000..3c3660c8715
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_progress_test.clj
@@ -0,0 +1,30 @@
+(ns metabase-enterprise.remote-sync.source.git-progress-test
+ "Push ProgressMonitor mapping for remote-sync export (GHY-4132)."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.remote-sync.source.git :as git]))
+
+(set! *warn-on-reflection* true)
+
+(deftest push-progress-monitor-maps-writing-objects-into-tail-test
+ (testing "the push monitor reports [0.8, ~0.99] over the Writing objects phase and heartbeats 0.8 during
+ other phases (so a locale where JGit never titles a phase \"Writing objects\", or reports an
+ unknown/zero total, still gets liveness pings)"
+ (let [reported (atom [])
+ ^org.eclipse.jgit.lib.ProgressMonitor mon
+ (#'git/->push-progress-monitor (fn [f] (swap! reported conj f)))]
+ ;; a non-writing phase still heartbeats at push-progress-start, not the writing-phase ramp
+ (.beginTask mon "Finding sources" 10)
+ (.update mon 5)
+ (.endTask mon)
+ ;; the writing phase drives progress
+ (.beginTask mon "Writing objects" 100)
+ (.update mon 50) ; halfway -> ~0.8 + 0.19*0.5
+ (.update mon 50) ; done -> ~0.99
+ (.endTask mon)
+ (let [xs @reported]
+ (is (seq xs) "reported at least once")
+ (is (every? #(<= 0.8 % 0.99) xs) "all reports within the push tail band")
+ (is (apply <= xs) "monotonic non-decreasing")
+ (is (= 0.8 (first xs)) "non-writing phase heartbeats at push-progress-start")
+ (is (> (last xs) 0.98) "writing phase ramp reaches close to the push tail's end")))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index c8124969751..587ba275e5d 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -42,6 +42,10 @@
(finish-commit! [_ message]
(reset! written {:message message :files @staged})
"merged-version")
+ (finish-commit! [_ message report-progress]
+ (reset! written {:message message :files @staged})
+ (when report-progress (report-progress 0.8))
+ "merged-version")
(abort-commit! [_] nil))))
(version [_] "remote-tip"))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 2398c60a5ca..1b1e3325c39 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -212,13 +212,16 @@ width: fixed
(empty-commit? [_]
(let [current (get @files-atom branch)]
(= current (apply-staged current))))
- (finish-commit! [_ _message]
+ (finish-commit! [this _message]
+ (source.p/finish-commit! this _message nil))
+ (finish-commit! [_ _message report-progress]
(case fail-mode
:write-files-error (throw (Exception. "Failed to write files"))
:store-error (throw (Exception. "Store failed"))
:apply-changes-error (throw (Exception. "Failed to apply changes"))
:network-error (throw (java.net.UnknownHostException. "Remote host not found"))
(swap! files-atom update branch apply-staged))
+ (when report-progress (report-progress 0.8))
;; version string discriminates a wholesale replace (replace-all!) from an incremental patch
(if @replace-all? "write-files-version" "apply-changes-version"))
(abort-commit! [_] nil))))
@@ -325,11 +328,14 @@ width: fixed
(replace-all! [_] (reset! replace-all? true) nil)
(empty-commit? [_]
(= (get-in @state [:trees version] {}) (staged-tree)))
- (finish-commit! [_ _message]
+ (finish-commit! [this _message]
+ (source.p/finish-commit! this _message nil))
+ (finish-commit! [_ _message report-progress]
(let [n (:counter (swap! state update :counter inc))
new-version (str "written-" n)
tree (staged-tree)]
(swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))
+ (when report-progress (report-progress 0.8))
new-version))
(abort-commit! [_] nil))))
(version [_] version)))]
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx
new file mode 100644
index 00000000000..27067ccf117
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx
@@ -0,0 +1,46 @@
+import { ReduxProvider } from "__support__/storybook";
+import { createMockState } from "metabase/redux/store/mocks";
+import { createMockUser } from "metabase-types/api/mocks";
+
+import { SyncProgressModal } from "./SyncProgressModal";
+
+export default {
+ title: "Enterprise/RemoteSync/SyncProgressModal",
+ component: SyncProgressModal,
+};
+
+const initialState = createMockState({
+ currentUser: createMockUser({ is_superuser: true }),
+});
+
+const noop = () => {};
+
+export const Stalled = () => (
+ <ReduxProvider storeInitialState={initialState}>
+ <SyncProgressModal
+ taskType="export"
+ progress={0.3}
+ isStalled
+ minutesSinceLastUpdate={6}
+ isError={false}
+ errorMessage=""
+ isSuccess={false}
+ outcome={null}
+ onDismiss={noop}
+ />
+ </ReduxProvider>
+);
+
+export const InProgress = () => (
+ <ReduxProvider storeInitialState={initialState}>
+ <SyncProgressModal
+ taskType="export"
+ progress={0.3}
+ isError={false}
+ errorMessage=""
+ isSuccess={false}
+ outcome={null}
+ onDismiss={noop}
+ />
+ </ReduxProvider>
+);
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx
index 7e1018f9a67..4f17c716b35 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx
@@ -1,4 +1,4 @@
-import { t } from "ttag";
+import { msgid, ngettext, t } from "ttag";
import { ActionButton } from "metabase/common/components/ActionButton";
import { useToast } from "metabase/common/hooks";
@@ -11,6 +11,8 @@ import type { RemoteSyncOutcome, RemoteSyncTaskType } from "metabase-types/api";
interface SyncProgressModalProps {
taskType: RemoteSyncTaskType;
progress: number;
+ isStalled?: boolean;
+ minutesSinceLastUpdate?: number | null;
isError: boolean;
errorMessage: string;
isSuccess: boolean;
@@ -21,6 +23,8 @@ interface SyncProgressModalProps {
export function SyncProgressModal({
progress,
taskType,
+ isStalled = false,
+ minutesSinceLastUpdate = null,
isError,
errorMessage,
isSuccess,
@@ -106,8 +110,18 @@ export function SyncProgressModal({
withCloseButton={false}
>
<Stack mt="md" gap="md">
- <Text ta="center">{progressLabel}</Text>
- <Progress value={progress * 100} transitionDuration={300} animated />
+ {isStalled ? (
+ <Text ta="center">{getStalledMessage(minutesSinceLastUpdate)}</Text>
+ ) : (
+ <>
+ <Text ta="center">{progressLabel}</Text>
+ <Progress
+ value={progress * 100}
+ transitionDuration={300}
+ animated
+ />
+ </>
+ )}
<Text size="sm">
{t`Please wait until this finishes before editing content.`}
</Text>
@@ -126,6 +140,19 @@ export function SyncProgressModal({
);
}
+function getStalledMessage(minutesSinceLastUpdate: number | null): string {
+ if (minutesSinceLastUpdate == null) {
+ return t`Still working…`;
+ }
+
+ const minutes = minutesSinceLastUpdate;
+ return ngettext(
+ msgid`Still working. No update for ${minutes} minute.`,
+ `Still working. No update for ${minutes} minutes.`,
+ minutes,
+ );
+}
+
const getModalContent = (
taskType: RemoteSyncTaskType,
): {
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx
index 1b31a203c57..bde046a7561 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx
@@ -14,6 +14,8 @@ import { SyncProgressModal } from "./SyncProgressModal";
const setup = ({
taskType = "import" as const,
progress = 0.5,
+ isStalled = false,
+ minutesSinceLastUpdate = null,
isError = false,
errorMessage = "",
isSuccess = false,
@@ -24,6 +26,8 @@ const setup = ({
}: {
taskType?: "import" | "export";
progress?: number;
+ isStalled?: boolean;
+ minutesSinceLastUpdate?: number | null;
isError?: boolean;
errorMessage?: string;
isSuccess?: boolean;
@@ -42,6 +46,8 @@ const setup = ({
<SyncProgressModal
taskType={taskType}
progress={progress}
+ isStalled={isStalled}
+ minutesSinceLastUpdate={minutesSinceLastUpdate}
isError={isError}
errorMessage={errorMessage}
isSuccess={isSuccess}
@@ -101,6 +107,29 @@ describe("SyncProgressModal", () => {
});
});
+ describe("stalled progress state (GHY-4132)", () => {
+ it("replaces the frozen progress bar with a 'still working' message when the task has stalled", () => {
+ // Reproduces GHY-4132: a large push stops reporting progress, the backend marks the still-running
+ // task :timed-out, and the UI used to sit on a frozen 'Exporting content…' bar. Instead we report
+ // honestly that it is still running but hasn't reported progress recently.
+ setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 6 });
+
+ expect(
+ screen.getByText("Still working. No update for 6 minutes."),
+ ).toBeInTheDocument();
+ expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();
+ expect(screen.queryByText("Exporting content…")).not.toBeInTheDocument();
+ });
+
+ it("uses the singular 'minute' when only one minute has passed", () => {
+ setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 1 });
+
+ expect(
+ screen.getByText("Still working. No update for 1 minute."),
+ ).toBeInTheDocument();
+ });
+ });
+
describe("error state", () => {
it("should show error modal with message", () => {
setup({ isError: true, errorMessage: "Connection timeout" });
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx
index 146719cb08f..e1821a58766 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx
@@ -1,3 +1,5 @@
+import dayjs from "dayjs";
+
import { useSetting } from "metabase/common/hooks";
import { useDispatch, useSelector } from "metabase/redux";
import { useGetRemoteSyncCurrentTaskQuery } from "metabase-enterprise/api";
@@ -9,7 +11,9 @@ import {
getHasPendingMutation,
getIsError,
getIsRunning,
+ getIsStalled,
getIsSuccess,
+ getLastProgressReportAt,
getProgress,
getShowModal,
getTaskOutcome,
@@ -28,11 +32,17 @@ export const useSyncStatus = () => {
const taskType = useSelector(getTaskType);
const progress = useSelector(getProgress);
const isError = useSelector(getIsError);
+ const isStalled = useSelector(getIsStalled);
+ const lastProgressReportAt = useSelector(getLastProgressReportAt);
const errorMessage = useSelector(getErrorMessage);
const isSuccess = useSelector(getIsSuccess);
const outcome = useSelector(getTaskOutcome);
const hasPendingMutation = useSelector(getHasPendingMutation);
+ const minutesSinceLastUpdate = lastProgressReportAt
+ ? dayjs().diff(dayjs(lastProgressReportAt), "minute")
+ : null;
+
const shouldPoll = isRunning && showModal && !hasPendingMutation;
useGetRemoteSyncCurrentTaskQuery(undefined, {
@@ -46,6 +56,8 @@ export const useSyncStatus = () => {
<SyncProgressModal
taskType={taskType}
progress={progress}
+ isStalled={isStalled}
+ minutesSinceLastUpdate={minutesSinceLastUpdate}
isError={isError}
errorMessage={errorMessage}
isSuccess={isSuccess}
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts
index 952aa2d28f1..64ef25927b7 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts
@@ -40,6 +40,16 @@ export const getIsError = createSelector(
(currentTask) => currentTask?.status === "errored",
);
+export const getIsStalled = createSelector(
+ getCurrentTask,
+ (currentTask) => currentTask?.status === "timed-out",
+);
+
+export const getLastProgressReportAt = createSelector(
+ getCurrentTask,
+ (currentTask) => currentTask?.last_progress_report_at ?? null,
+);
+
export const getIsSuccess = createSelector(
getCurrentTask,
(currentTask) => currentTask?.status === "successful",