The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix Remote Sync push hang: honest stalled UI + meaningful export progress (GHY-4132) (#77904)
Eric Normandf0ce91f74c89
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 57402f35c97..6d39394071e 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -668,24 +668,27 @@(defn- commit-staged!"Open a commit on `snapshot`, stage into it via `stage-fn`, then finish it.- `stage-fn` will be passed the open commit. It should return the synced write-rows.+ `stage-fn` will be passed the open commit. It should return the synced write-rows. `report-progress`, when+ non-nil, is a reporter fn forwarded unchanged to `finish-commit!` (fired forced at the commit checkpoint,+ before the push, then throttled during the push itself).Will abort the commit if it is empty.Returns `[version, write-rows]` where version is the new commit SHA or `:remote-sync/empty-commit` if empty.Will abort the commit and throw on any Throwable."- [snapshot message stage-fn]- (let [commit (source.p/open-commit snapshot)]- (try- (let [synced (stage-fn commit)- version (if (source.p/empty-commit? commit)- (do (source.p/abort-commit! commit) :remote-sync/empty-commit)- (source.p/finish-commit! commit message))]- [synced version])- (catch Throwable e- (source.p/abort-commit! commit)- (throw e)))))+ ([snapshot message stage-fn] (commit-staged! snapshot message stage-fn nil))+ ([snapshot message stage-fn report-progress]+ (let [commit (source.p/open-commit snapshot)]+ (try+ (let [synced (stage-fn commit)+ version (if (source.p/empty-commit? commit)+ (do (source.p/abort-commit! commit) :remote-sync/empty-commit)+ (source.p/finish-commit! commit message report-progress))]+ [synced version])+ (catch Throwable e+ (source.p/abort-commit! commit)+ (throw e))))))(defn- export-merged!"Export when the remote branch has advanced beyond the last synced version. Runs an entity-identity 3-way merge of@@ -996,13 +999,22 @@"Stage WriteRows ({:model_type :model_id, optional :id/:file_path}) to `commit`, chunking internally permodel type so each chunk's entities load in one query with bounded memory. Callers pass a flat seq.+ `on-chunk`, when non-nil, is called once per staged chunk with the cumulative number of rows staged so+ far — used to drive serialize-phase progress. It must not realize `rows`.+Returns:- [{:id :file_path :content_hash}]"- [commit opts rows]- (->> rows- (->sized-chunks)- (mapcat #(chunk-stage-writes commit opts %))- (doall)))+ ([commit opts rows] (stage-writes commit opts rows nil))+ ([commit opts rows on-chunk]+ (let [staged (volatile! 0)]+ (->> rows+ (->sized-chunks)+ (mapcat (fn [chunk]+ (let [res (chunk-stage-writes commit opts chunk)]+ (vswap! staged + (count (:rows chunk)))+ (when on-chunk (on-chunk @staged))+ res)))+ (doall)))))(defn- stage-deletes [commit delete-paths](doseq [delete-path delete-paths]@@ -1052,6 +1064,9 @@hits)[:else :content_hash]))))))))+(def ^:private export-progress-plan-done 0.33) ; phase 1 (plan) complete / serialize start+(def ^:private export-progress-serialize 0.66) ; phase 2 (serialize) complete+(defn- full-export!"Re-serialize and commit the entire remote-synced set, then reconcile every RemoteSyncObject.@@ -1059,39 +1074,56 @@- {:status :success}or throws"[snapshot task-id message sync-timestamp]- (let [export-rows (exportable-write-rows)]+ (let [export-rows (vec (exportable-write-rows))](when (empty? export-rows)(throw (ex-info "No remote-syncable content available." {})))- (remote-sync.task/update-progress! task-id 0.3)- (let [opts (serdes/storage-base-context)- [synced version] (commit-staged! snapshot message- (fn [commit]- (source.p/replace-all! commit) ; replace the managed dirs wholesale- (stage-writes commit opts export-rows)))]- (t2/with-transaction [_]- (when-not (= version :remote-sync/empty-commit)- (remote-sync.task/set-version! task-id version))- (doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))]- (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))- (mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp))- (if (= version :remote-sync/empty-commit)- (do- (log/info "Remote sync full export: re-serialized content matches remote; skipped empty commit")- {:status :success :outcome {:kind "push-skipped"}})- {:status :success- :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}}))))+ (let [report (remote-sync.task/make-progress-reporter task-id)+ total (count export-rows)+ span (- export-progress-serialize export-progress-plan-done)]+ (report export-progress-plan-done {:force? true})+ (let [opts (serdes/storage-base-context)+ [synced version] (commit-staged! snapshot message+ (fn [commit]+ (source.p/replace-all! commit) ; replace the managed dirs wholesale+ (let [synced (stage-writes commit opts export-rows+ (fn [staged]+ (report (+ export-progress-plan-done+ (* span (/ staged total))))))]+ (report export-progress-serialize {:force? true})+ synced))+ report)]+ (t2/with-transaction [_]+ (when-not (= version :remote-sync/empty-commit)+ (remote-sync.task/set-version! task-id version))+ (doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))]+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))+ (mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp))+ (if (= version :remote-sync/empty-commit)+ (do+ (log/info "Remote sync full export: re-serialized content matches remote; skipped empty commit")+ {:status :success :outcome {:kind "push-skipped"}})+ {:status :success+ :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}})))))(defn- incremental-export disabled-files)]- (remote-sync.task/update-progress! task-id 0.3)+ delete-paths (into (vec delete-paths) disabled-files)+ report (remote-sync.task/make-progress-reporter task-id)+ total (max 1 (count writes))+ span (- export-progress-serialize export-progress-plan-done)]+ (report export-progress-plan-done {:force? true})(let [opts (serdes/storage-base-context)[synced version] (commit-staged! snapshot message(fn [commit]- (let [synced (stage-writes commit opts writes)]+ (let [synced (stage-writes commit opts writes+ (fn [staged]+ (report (+ export-progress-plan-done+ (* span (/ staged total))))))](stage-deletes commit delete-paths)- synced)))]+ (report export-progress-serialize {:force? true})+ synced))+ report)](t2/with-transaction [_](when-not (= version :remote-sync/empty-commit)(remote-sync.task/set-version! task-id version))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.cljindex 56bda997ac0..c26a1b3deba 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj@@ -84,6 +84,39 @@{:progress progress:last_progress_report_at (mi/now)}))+(def ^:private default-progress-throttle-ms+ "Minimum ms between throttled (non-boundary) progress writes."+ 10000)++(defn make-progress-reporter+ "Returns a stateful progress reporter for `task-id`.++ Call the returned fn with a fraction in [0.0, 1.0]. It writes progress (and bumps+ `last_progress_report_at`) at most once per throttle window and never moves the fraction backward.+ Pass `{:force? true}` to write immediately regardless of the throttle — use at phase boundaries.++ Options:+ - :throttle-ms minimum ms between throttled writes (default 10000)+ - :now-fn 0-arg fn returning current millis (default `System/currentTimeMillis`)+ - :write-fn 1-arg fn taking the clamped fraction (default `update-progress!` for `task-id`)"+ ([task-id] (make-progress-reporter task-id nil))+ ([task-id {:keys [throttle-ms now-fn write-fn]+ :or {throttle-ms default-progress-throttle-ms}}]+ (let [now-fn (or now-fn #(System/currentTimeMillis))+ write-fn (or write-fn (fn [f] (update-progress! task-id f)))+ last-ms (volatile! nil)+ last-fraction (volatile! -1.0)]+ (fn report!+ ([fraction] (report! fraction nil))+ ([fraction {:keys [force?]}]+ (let [now (now-fn)+ f (-> (double fraction) (max 0.0) (min 1.0))]+ (when (and (>= f @last-fraction)+ (or force? (nil? @last-ms) (>= (- now @last-ms) throttle-ms)))+ (vreset! last-ms now)+ (vreset! last-fraction f)+ (write-fn f))))))))+(defn set-version!"Sets the version value for a sync task.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex 74adf9afd0f..05b151d0351 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -16,7 +16,7 @@(org.eclipse.jgit.api Git GitCommand TransportCommand)(org.eclipse.jgit.dircache DirCache DirCacheBuilder DirCacheEditor DirCacheEditor$DeletePathDirCacheEditor$DeleteTree DirCacheEditor$PathEdit DirCacheEntry)- (org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent Ref Repository)+ (org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent ProgressMonitor Ref Repository)(org.eclipse.jgit.lib ObjectInserter ObjectReader)(org.eclipse.jgit.revwalk RevCommit RevTree RevWalk)(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdate@@ -251,27 +251,75 @@(recur (update acc bucket conj (.getPathString tw))))acc)))))))+(def ^:private commit-progress-checkpoint+ "Export progress fraction reported once the local commit is durable, just before the network push begins."+ 0.8)++(def ^:private push-progress-start+ "Progress fraction at which the network push begins."+ 0.8)++(def ^:private push-progress-end+ "Progress fraction the network push approaches as it completes; the final 1.0 is reported elsewhere."+ 0.99)++(defn- ->push-progress-monitor+ "A JGit ProgressMonitor that maps the client-side \"Writing objects\" phase onto+ [push-progress-start, push-progress-end] and calls `report-progress` (a 1-arg fraction fn) on every+ `update` tick — even outside the writing phase, falling back to `push-progress-start` — so the push+ always heartbeats regardless of JVM locale (which can rename or suppress the \"Writing objects\" title)+ or an unknown (zero) total. Upstream throttling/monotonicity is handled by the reporter, so the repeated+ fallback values are cheap and safe."+ ^ProgressMonitor [report-progress]+ (let [writing? (volatile! false)+ total (volatile! 0)+ done (volatile! 0)+ report! (fn []+ (report-progress+ (if (and @writing? (pos? @total))+ (+ push-progress-start+ (* (- push-progress-end push-progress-start)+ (min 1.0 (/ (double @done) @total))))+ push-progress-start)))]+ (reify ProgressMonitor+ (start [_ _total-tasks])+ (beginTask [_ title tot]+ (vreset! writing? (= title "Writing objects"))+ (vreset! total (max 0 tot))+ (vreset! done 0)+ (report!))+ (update [_ completed]+ (vswap! done + completed)+ (report!))+ (endTask [_]+ (vreset! writing? false))+ (isCancelled [_] false)+ (showDuration [_ _]))))+(defn push-branch!- "Pushes a local branch to the remote repository.+ "Pushes a local branch to the remote repository. Optional `progress-monitor` (a JGit ProgressMonitor)+ reports push progress.Takes a git-source map containing a :git Git instance, :branch, and optional :token forauthentication. Uses the 'origin' remote which is configured by ensure-origin-configured!.Returns the push response from JGit. Throws ExceptionInfo if the push operation fails or returns anon-OK/UP_TO_DATE status."- [{:keys [^Git git ^String branch] :as git-source}]- (let [branch-name (qualify-branch branch)- push-response (call-remote-command- (-> (.push git)- (.setRefSpecs (doto (java.util.ArrayList.)- (.add (RefSpec. (str branch-name ":" branch-name))))))- git-source)- push-results (->> push-response- (map #(into [] (.getRemoteUpdates ^PushResult %)))- flatten)]- (when-let [failures (seq (remove #(#{RemoteRefUpdate$Status/OK RemoteRefUpdate$Status/UP_TO_DATE} %) (map #(.getStatus ^RemoteRefUpdate %) push-results)))]- (throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))- push-response))+ ([git-source] (push-branch! git-source nil))+ ([{:keys [^Git git ^String branch] :as git-source} ^ProgressMonitor progress-monitor]+ (let [branch-name (qualify-branch branch)+ push-cmd (cond-> (-> (.push git)+ (.setRefSpecs (doto (java.util.ArrayList.)+ (.add (RefSpec. (str branch-name ":" branch-name))))))+ progress-monitor (.setProgressMonitor progress-monitor))+ push-response (call-remote-command push-cmd git-source)+ push-results (->> push-response+ (map #(into [] (.getRemoteUpdates ^PushResult %)))+ flatten)]+ (when-let [failures (seq (remove #(#{RemoteRefUpdate$Status/OK RemoteRefUpdate$Status/UP_TO_DATE} %)+ (map #(.getStatus ^RemoteRefUpdate %) push-results)))]+ (throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))+ push-response)))(defn default-branch"Retrieves the default branch name of the git repository.@@ -332,6 +380,9 @@(.equals (written-tree-id this) ^ObjectId parent-tree-id))))(finish-commit! [this message]+ (source.p/finish-commit! this message nil))++ (finish-commit! [this message report-progress](let [^Git git (:git snapshot)repo (.getRepository git)branch-ref (qualify-branch (:branch snapshot))@@ -348,7 +399,9 @@(doto (.updateRef repo branch-ref)(.setNewObjectId commit-id)(.update))- (push-branch! snapshot)+ ;; local commit durable; push about to start — force this one-shot checkpoint past the throttle+ (when report-progress (report-progress commit-progress-checkpoint {:force? true}))+ (push-branch! snapshot (when report-progress (->push-progress-monitor report-progress)))(close-commit-resources! inserter reader rev-walk) ; close only after a successful push(.name commit-id))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.cljindex ad04ba0a9f9..342e3de2b43 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj@@ -84,8 +84,15 @@(empty-commit? [commit]"True if the staged tree is identical to the parent's, i.e. finishing would introduce no changes. Alwaysfalse for a root commit (no parent).")- (finish-commit! [commit message]- "Write the staged tree, commit with `message`, push, and release resources. Returns the new version.")+ (finish-commit!+ [commit message]+ [commit message report-progress]+ "Write the staged tree, commit with `message`, push, and release resources. Returns the new version.++ `report-progress`, when given, is an optional reporter fn accepting `(fraction)` or `(fraction opts)`;+ nil to skip. It is called once, forced, at the commit checkpoint just after the local commit is durable+ and before the network push begins, and then repeatedly (throttled) by the push's ProgressMonitor as+ the push proceeds.")(abort-commit! [commit]"Release the commit's resources without committing or pushing. Returns nil."))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.cljnew file mode 100644index 00000000000..8ae8cd4e496--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj@@ -0,0 +1,51 @@+(ns metabase-enterprise.remote-sync.impl-progress-test+ "Serialize-phase progress hook for remote-sync export (GHY-4132)."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.source.protocol :as source.p]))++(set! *warn-on-reflection* true)++;; A CommitBuilder that records staged paths and does nothing else.+(defrecord RecordingCommit [staged]+ source.p/CommitBuilder+ (stage-upsert! [_ {:keys [path]}] (swap! staged conj path) nil)+ (stage-delete! [_ _] nil)+ (replace-all! [_] nil)+ (empty-commit? [_] false)+ (finish-commit! [_ _message] "sha")+ (abort-commit! [_] nil))++(deftest stage-writes-invokes-on-chunk-with-cumulative-count-test+ (testing "stage-writes calls on-chunk once per chunk with the running staged count, and stages everything"+ (with-redefs [;; make each row its own chunk (matching the real {:model_type :rows} chunk shape) and a+ ;; trivial entity, so we exercise the loop deterministically without hitting serdes/the app DB+ impl/->sized-chunks (fn [rows] (map (fn [row] {:model_type (:model_type row) :rows [row]}) rows))+ impl/extract-chunk (fn [{:keys [rows]}] (map (fn [row] [row {:e (:model_id row)}]) rows))]+ (let [staged (atom [])+ commit (->RecordingCommit staged)+ rows [{:model_type "card" :model_id 1 :file_path "a"}+ {:model_type "card" :model_id 2 :file_path "b"}+ {:model_type "card" :model_id 3 :file_path "c"}]+ seen (atom [])]+ (#'impl/stage-writes commit {} rows (fn [n] (swap! seen conj n)))+ (is (= [1 2 3] @seen) "cumulative staged count reported once per chunk")+ (is (= ["a" "b" "c"] @staged) "all rows staged")))))++(deftest finish-commit-reports-commit-checkpoint-test+ (testing "commit-staged! passes the progress callback to finish-commit!, invoked at the commit checkpoint"+ (let [reported (atom [])+ commit (reify source.p/CommitBuilder+ (stage-upsert! [_ _] nil)+ (stage-delete! [_ _] nil)+ (replace-all! [_] nil)+ (empty-commit? [_] false)+ (finish-commit! [_ _message] "sha")+ (finish-commit! [_ _message report] (when report (report 0.8 {:force? true})) "sha")+ (abort-commit! [_] nil))]+ (with-redefs [source.p/open-commit (fn [_] commit)]+ (#'impl/commit-staged! {:managed-dirs []} "msg"+ (fn [_c] [{:id 1}])+ (fn [f & _] (swap! reported conj f))))+ (is (= [0.8] @reported)))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 8fee1791581..d5cf43bbfed 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -483,7 +483,7 @@(is (= :success (:status result)))(is (pos? (count @progress-calls)))(is (= task-id (:task-id (first @progress-calls))))- (is (= 0.3 (:progress (first @progress-calls)))))))))))))+ (is (= 0.33 (:progress (first @progress-calls)))))))))))))(deftest import!-resets-remote-sync-object-table-test(testing "import! deletes and recreates RemoteSyncObject table with synced status"@@ -1794,6 +1794,7 @@ serdes/meta:(replace-all! [_] nil)(empty-commit? [_] empty?)(finish-commit! [_ _] "written-version")+ (finish-commit! [_ _ report-progress] (when report-progress (report-progress 0.8)) "written-version")(abort-commit! [_] nil))))))(defn- export-test-sourcediff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.cljindex 98754266ca5..cf4245c307e 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj@@ -472,3 +472,42 @@(is (false? (:cancelled after)))(is (= (:ended_at before) (:ended_at after))"ended_at must not be overwritten")))))++;;; ------------------------------------------------------------------------------------------------+;;; Tests for make-progress-reporter+;;; ------------------------------------------------------------------------------------------------++(deftest make-progress-reporter-throttles-and-forces-test+ (testing "intra-phase writes are throttled; :force? bypasses the throttle; boundaries always land"+ (let [now (atom 0)+ writes (atom [])+ report (rst/make-progress-reporter 1 {:throttle-ms 10000+ :now-fn (fn [] @now)+ :write-fn (fn [f] (swap! writes conj f))})]+ (report 0.33 {:force? true}) ; boundary — always writes+ (report 0.40) ; 0ms since last throttled anchor set by forced write -> throttled out+ (reset! now 5000) (report 0.50) ; 5s < 10s -> throttled out+ (reset! now 10000) (report 0.60) ; 10s -> writes+ (report 0.66 {:force? true}) ; boundary -> writes+ (is (= [0.33 0.60 0.66] @writes)))))++(deftest make-progress-reporter-is-monotonic-test+ (testing "the reporter never writes a fraction lower than the last written"+ (let [now (atom 0)+ writes (atom [])+ report (rst/make-progress-reporter 1 {:throttle-ms 0+ :now-fn (fn [] @now)+ :write-fn (fn [f] (swap! writes conj f))})]+ (report 0.5)+ (report 0.4) ; lower -> dropped+ (report 0.6)+ (is (= [0.5 0.6] @writes)))))++(deftest make-progress-reporter-clamps-test+ (testing "fractions are clamped into [0.0, 1.0]"+ (let [writes (atom [])+ report (rst/make-progress-reporter 1 {:throttle-ms 0+ :now-fn (constantly 0)+ :write-fn (fn [f] (swap! writes conj f))})]+ (report 1.5)+ (is (= [1.0] @writes)))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_progress_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_progress_test.cljnew file mode 100644index 00000000000..3c3660c8715--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_progress_test.clj@@ -0,0 +1,30 @@+(ns metabase-enterprise.remote-sync.source.git-progress-test+ "Push ProgressMonitor mapping for remote-sync export (GHY-4132)."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.remote-sync.source.git :as git]))++(set! *warn-on-reflection* true)++(deftest push-progress-monitor-maps-writing-objects-into-tail-test+ (testing "the push monitor reports [0.8, ~0.99] over the Writing objects phase and heartbeats 0.8 during+ other phases (so a locale where JGit never titles a phase \"Writing objects\", or reports an+ unknown/zero total, still gets liveness pings)"+ (let [reported (atom [])+ ^org.eclipse.jgit.lib.ProgressMonitor mon+ (#'git/->push-progress-monitor (fn [f] (swap! reported conj f)))]+ ;; a non-writing phase still heartbeats at push-progress-start, not the writing-phase ramp+ (.beginTask mon "Finding sources" 10)+ (.update mon 5)+ (.endTask mon)+ ;; the writing phase drives progress+ (.beginTask mon "Writing objects" 100)+ (.update mon 50) ; halfway -> ~0.8 + 0.19*0.5+ (.update mon 50) ; done -> ~0.99+ (.endTask mon)+ (let [xs @reported]+ (is (seq xs) "reported at least once")+ (is (every? #(<= 0.8 % 0.99) xs) "all reports within the push tail band")+ (is (apply <= xs) "monotonic non-decreasing")+ (is (= 0.8 (first xs)) "non-writing phase heartbeats at push-progress-start")+ (is (> (last xs) 0.98) "writing phase ramp reaches close to the push tail's end")))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex c8124969751..587ba275e5d 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -42,6 +42,10 @@(finish-commit! [_ message](reset! written {:message message :files @staged})"merged-version")+ (finish-commit! [_ message report-progress]+ (reset! written {:message message :files @staged})+ (when report-progress (report-progress 0.8))+ "merged-version")(abort-commit! [_] nil))))(version [_] "remote-tip"))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 2398c60a5ca..1b1e3325c39 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -212,13 +212,16 @@ width: fixed(empty-commit? [_](let [current (get @files-atom branch)](= current (apply-staged current))))- (finish-commit! [_ _message]+ (finish-commit! [this _message]+ (source.p/finish-commit! this _message nil))+ (finish-commit! [_ _message report-progress](case fail-mode:write-files-error (throw (Exception. "Failed to write files")):store-error (throw (Exception. "Store failed")):apply-changes-error (throw (Exception. "Failed to apply changes")):network-error (throw (java.net.UnknownHostException. "Remote host not found"))(swap! files-atom update branch apply-staged))+ (when report-progress (report-progress 0.8));; version string discriminates a wholesale replace (replace-all!) from an incremental patch(if @replace-all? "write-files-version" "apply-changes-version"))(abort-commit! [_] nil))))@@ -325,11 +328,14 @@ width: fixed(replace-all! [_] (reset! replace-all? true) nil)(empty-commit? [_](= (get-in @state [:trees version] {}) (staged-tree)))- (finish-commit! [_ _message]+ (finish-commit! [this _message]+ (source.p/finish-commit! this _message nil))+ (finish-commit! [_ _message report-progress](let [n (:counter (swap! state update :counter inc))new-version (str "written-" n)tree (staged-tree)](swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))+ (when report-progress (report-progress 0.8))new-version))(abort-commit! [_] nil))))(version [_] version)))]diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsxnew file mode 100644index 00000000000..27067ccf117--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx@@ -0,0 +1,46 @@+import { ReduxProvider } from "__support__/storybook";+import { createMockState } from "metabase/redux/store/mocks";+import { createMockUser } from "metabase-types/api/mocks";++import { SyncProgressModal } from "./SyncProgressModal";++export default {+ title: "Enterprise/RemoteSync/SyncProgressModal",+ component: SyncProgressModal,+};++const initialState = createMockState({+ currentUser: createMockUser({ is_superuser: true }),+});++const noop = () => {};++export const Stalled = () => (+ <ReduxProvider storeInitialState={initialState}>+ <SyncProgressModal+ taskType="export"+ progress={0.3}+ isStalled+ minutesSinceLastUpdate={6}+ isError={false}+ errorMessage=""+ isSuccess={false}+ outcome={null}+ onDismiss={noop}+ />+ </ReduxProvider>+);++export const InProgress = () => (+ <ReduxProvider storeInitialState={initialState}>+ <SyncProgressModal+ taskType="export"+ progress={0.3}+ isError={false}+ errorMessage=""+ isSuccess={false}+ outcome={null}+ onDismiss={noop}+ />+ </ReduxProvider>+);diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsxindex 7e1018f9a67..4f17c716b35 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx@@ -1,4 +1,4 @@-import { t } from "ttag";+import { msgid, ngettext, t } from "ttag";import { ActionButton } from "metabase/common/components/ActionButton";import { useToast } from "metabase/common/hooks";@@ -11,6 +11,8 @@ import type { RemoteSyncOutcome, RemoteSyncTaskType } from "metabase-types/api";interface SyncProgressModalProps {taskType: RemoteSyncTaskType;progress: number;+ isStalled?: boolean;+ minutesSinceLastUpdate?: number | null;isError: boolean;errorMessage: string;isSuccess: boolean;@@ -21,6 +23,8 @@ interface SyncProgressModalProps {export function SyncProgressModal({progress,taskType,+ isStalled = false,+ minutesSinceLastUpdate = null,isError,errorMessage,isSuccess,@@ -106,8 +110,18 @@ export function SyncProgressModal({withCloseButton={false}><Stack mt="md" gap="md">- <Text ta="center">{progressLabel}</Text>- <Progress value={progress * 100} transitionDuration={300} animated />+ {isStalled ? (+ <Text ta="center">{getStalledMessage(minutesSinceLastUpdate)}</Text>+ ) : (+ <>+ <Text ta="center">{progressLabel}</Text>+ <Progress+ value={progress * 100}+ transitionDuration={300}+ animated+ />+ </>+ )}<Text size="sm">{t`Please wait until this finishes before editing content.`}</Text>@@ -126,6 +140,19 @@ export function SyncProgressModal({);}+function getStalledMessage(minutesSinceLastUpdate: number | null): string {+ if (minutesSinceLastUpdate == null) {+ return t`Still working…`;+ }++ const minutes = minutesSinceLastUpdate;+ return ngettext(+ msgid`Still working. No update for ${minutes} minute.`,+ `Still working. No update for ${minutes} minutes.`,+ minutes,+ );+}+const getModalContent = (taskType: RemoteSyncTaskType,): {diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsxindex 1b31a203c57..bde046a7561 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx@@ -14,6 +14,8 @@ import { SyncProgressModal } from "./SyncProgressModal";const setup = ({taskType = "import" as const,progress = 0.5,+ isStalled = false,+ minutesSinceLastUpdate = null,isError = false,errorMessage = "",isSuccess = false,@@ -24,6 +26,8 @@ const setup = ({}: {taskType?: "import" | "export";progress?: number;+ isStalled?: boolean;+ minutesSinceLastUpdate?: number | null;isError?: boolean;errorMessage?: string;isSuccess?: boolean;@@ -42,6 +46,8 @@ const setup = ({<SyncProgressModaltaskType={taskType}progress={progress}+ isStalled={isStalled}+ minutesSinceLastUpdate={minutesSinceLastUpdate}isError={isError}errorMessage={errorMessage}isSuccess={isSuccess}@@ -101,6 +107,29 @@ describe("SyncProgressModal", () => {});});+ describe("stalled progress state (GHY-4132)", () => {+ it("replaces the frozen progress bar with a 'still working' message when the task has stalled", () => {+ // Reproduces GHY-4132: a large push stops reporting progress, the backend marks the still-running+ // task :timed-out, and the UI used to sit on a frozen 'Exporting content…' bar. Instead we report+ // honestly that it is still running but hasn't reported progress recently.+ setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 6 });++ expect(+ screen.getByText("Still working. No update for 6 minutes."),+ ).toBeInTheDocument();+ expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();+ expect(screen.queryByText("Exporting content…")).not.toBeInTheDocument();+ });++ it("uses the singular 'minute' when only one minute has passed", () => {+ setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 1 });++ expect(+ screen.getByText("Still working. No update for 1 minute."),+ ).toBeInTheDocument();+ });+ });+describe("error state", () => {it("should show error modal with message", () => {setup({ isError: true, errorMessage: "Connection timeout" });diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsxindex 146719cb08f..e1821a58766 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx@@ -1,3 +1,5 @@+import dayjs from "dayjs";+import { useSetting } from "metabase/common/hooks";import { useDispatch, useSelector } from "metabase/redux";import { useGetRemoteSyncCurrentTaskQuery } from "metabase-enterprise/api";@@ -9,7 +11,9 @@ import {getHasPendingMutation,getIsError,getIsRunning,+ getIsStalled,getIsSuccess,+ getLastProgressReportAt,getProgress,getShowModal,getTaskOutcome,@@ -28,11 +32,17 @@ export const useSyncStatus = () => {const taskType = useSelector(getTaskType);const progress = useSelector(getProgress);const isError = useSelector(getIsError);+ const isStalled = useSelector(getIsStalled);+ const lastProgressReportAt = useSelector(getLastProgressReportAt);const errorMessage = useSelector(getErrorMessage);const isSuccess = useSelector(getIsSuccess);const outcome = useSelector(getTaskOutcome);const hasPendingMutation = useSelector(getHasPendingMutation);+ const minutesSinceLastUpdate = lastProgressReportAt+ ? dayjs().diff(dayjs(lastProgressReportAt), "minute")+ : null;+const shouldPoll = isRunning && showModal && !hasPendingMutation;useGetRemoteSyncCurrentTaskQuery(undefined, {@@ -46,6 +56,8 @@ export const useSyncStatus = () => {<SyncProgressModaltaskType={taskType}progress={progress}+ isStalled={isStalled}+ minutesSinceLastUpdate={minutesSinceLastUpdate}isError={isError}errorMessage={errorMessage}isSuccess={isSuccess}diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.tsindex 952aa2d28f1..64ef25927b7 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts@@ -40,6 +40,16 @@ export const getIsError = createSelector((currentTask) => currentTask?.status === "errored",);+export const getIsStalled = createSelector(+ getCurrentTask,+ (currentTask) => currentTask?.status === "timed-out",+);++export const getLastProgressReportAt = createSelector(+ getCurrentTask,+ (currentTask) => currentTask?.last_progress_report_at ?? null,+);+export const getIsSuccess = createSelector(getCurrentTask,(currentTask) => currentTask?.status === "successful",