The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GHY-4593] Make remote sync robust against restarts (#82553)
Tyler Pludeb08a2f0e6503
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.cljindex 01aa0db5cc1..dec09ed73b6 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj@@ -177,12 +177,24 @@:force_push_casualties force-push-casualties:reason (some-> reason name)}))+(defn- present-task+ "Hydrate `task` for the API: its status and the initiating user trimmed to what the UI shows."+ [task]+ (-> task+ (t2/hydrate :status :initiated_by_user)+ (update :initiated_by_user #(some-> % (select-keys [:id :first_name :last_name :email])))))+(api.macros/defendpoint :get "/current-task" :- [:maybe remote-sync.schema/SyncTask]"Get the current sync task"[](api/check-superuser)- (when-let [task (remote-sync.task/most-recent-task)]- (t2/hydrate task :status)))+ (when-let [task (some-> (remote-sync.task/most-recent-task) present-task)]+ (if (= :timed-out (:status task))+ ;; The owning worker is gone (its heartbeat stopped), so close the row now rather than leaving it to the+ ;; next task creation. Idempotent and safe to race across nodes and polling tabs.+ (do (remote-sync.task/supersede-stale-tasks!)+ (present-task (remote-sync.task/most-recent-task)))+ task)))(api.macros/defendpoint :post "/current-task/cancel" :- remote-sync.schema/SyncTask"Cancels the current task if one is running"@@ -191,7 +203,7 @@(let [task (remote-sync.task/most-recent-task)](api/check-400 (and (some? task) (remote-sync.task/running? task)) "No active task to cancel")(remote-sync.task/cancel-sync-task! (:id task))- (t2/hydrate (remote-sync.task/most-recent-task) :status)))+ (present-task (remote-sync.task/most-recent-task))))(api.macros/defendpoint :post "/test-connection" :- remote-sync.schema/TestConnectionResponse"Test whether the Remote Sync credentials can reach the git repository.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.cljindex cbe6924d7eb..b50621aeb5d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj@@ -649,14 +649,19 @@[task-id :- ms/PositiveInt](t2/select-one-fn :cancelled :model/RemoteSyncTask :id task-id))+(def ^:private last-alive-at+ "The time the task's owning thread last proved it was alive: its heartbeat, or its last progress write for+ rows that predate the heartbeat column or whose worker died before its first beat."+ [:coalesce :last_heartbeat_at :last_progress_report_at])+(mu/defn current-task- "The newest started, unfinished RemoteSyncTask that reported progress after `progress-cutoff`, or nil."- [progress-cutoff :- ms/TemporalInstant]+ "The newest started, unfinished RemoteSyncTask whose owner was alive after `liveness-cutoff`, or nil."+ [liveness-cutoff :- ms/TemporalInstant](t2/select-one :model/RemoteSyncTask{:where [:and[:<> :started_at nil][:= :ended_at nil]- [:< progress-cutoff :last_progress_report_at]]+ [:< liveness-cutoff last-alive-at]]:limit 1:order-by [[:started_at :desc][:id :desc]]}))@@ -671,15 +676,16 @@:order-by [[:started_at :desc][:id :desc]]}))-(mu/defn last-successful-task- "The newest finished RemoteSyncTask that was neither cancelled nor failed and recorded a version, or nil."+(mu/defn last-synced-task+ "The newest RemoteSyncTask whose commit the local content matches, or nil. `version` is written inside the+ transaction that commits an import or a push, and by the conflict path (which also writes `conflicts`), so+ `version` set with `conflicts` null identifies a landed commit whatever `ended_at`, `cancelled`, or+ `error_message` say: a task cancelled or superseded after its transaction committed is still the sync base."[](t2/select-one :model/RemoteSyncTask{:where [:and- [:<> nil :ended_at]- [:= false :cancelled]- [:= nil :error_message]- [:<> nil :version]]+ [:<> nil :version]+ [:= nil :conflicts]]:limit 1:order-by [[:started_at :desc][:id :desc]]}))@@ -707,17 +713,29 @@progress :- number?](t2/update! :model/RemoteSyncTask task-id {:progress progress, :last_progress_report_at :%now}))-(mu/defn supersede-stale-tasks!- "Cancel and end now the started, unfinished RemoteSyncTasks that last reported progress before `cutoff`."- [cutoff :- ms/TemporalInstant]- (t2/query {:update (t2/table-name :model/RemoteSyncTask)- :set {:cancelled true- :ended_at :%now- :error_message "Superseded after staleness timeout"}- :where [:and- [:<> :started_at nil]- [:= :ended_at nil]- [:< :last_progress_report_at cutoff]]}))+(mu/defn touch-task!+ "Stamp the heartbeat time of the RemoteSyncTask with `task-id` if it has not ended, returning the number of rows+ updated. Never touches an ended row, so a cancel or supersede is not undone by a late beat."+ [task-id :- ms/PositiveInt]+ (t2/update! :model/RemoteSyncTask {:id task-id, :ended_at nil} {:last_heartbeat_at :%now}))++(mu/defn supersede-stale-tasks! :- [:sequential ms/PositiveInt]+ "Cancel and end now, with `message` as the error message, the started, unfinished RemoteSyncTasks whose owner+ was last alive before `cutoff`. Returns the ids of the rows ended, empty when none were stale."+ [cutoff :- ms/TemporalInstant+ message :- :string]+ (let [stale [:and+ [:<> :started_at nil]+ [:= :ended_at nil]+ [:< last-alive-at cutoff]]+ ids (vec (t2/select-pks-vec :model/RemoteSyncTask {:where stale}))]+ (when (seq ids)+ (t2/query {:update (t2/table-name :model/RemoteSyncTask)+ :set {:cancelled true+ :ended_at :%now+ :error_message message}+ :where [:and stale [:in :id ids]]}))+ ids))(mu/defn delete-tasks-started-before!"Delete the RemoteSyncTasks started before `cutoff`, returning the number deleted."diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex c8ab89eed2e..f2455b949ce 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -141,25 +141,26 @@Takes a throwable exception and returns a string message that categorizes the error (network, authentication,repository not found, branch, or generic) based on the exception type and message content."[e]- (let [missing-db (cause-with-error e :metabase.models.serialization.resolve.db/database-not-found)]+ (let [missing-db (cause-with-error e :metabase.models.serialization.resolve.db/database-not-found)+ message (or (ex-message e) "")](cond(or (instance? java.net.UnknownHostException e)(instance? java.net.UnknownHostException (ex-cause e)))"Network error: Unable to reach git repository host"- (str/includes? (ex-message e) "Authentication failed")+ (str/includes? message "Authentication failed")"Authentication failed: Please check your git credentials"- (str/includes? (ex-message e) "Repository not found")+ (str/includes? message "Repository not found")"Repository not found: Please check the repository URL"- (str/includes? (ex-message e) "branch")+ (str/includes? message "branch")"Branch error: Please check the specified branch exists"(some-> e ex-cause ex-message (str/includes? "Can't create a tenant collection without tenants enabled"))"This repository contains tenant collections, but the tenants feature is disabled on your instance."- (str/includes? (ex-message e) "Missing commit")+ (str/includes? message "Missing commit")"Repository cache is stale: the remote repository may have been force-pushed. Please retry the operation."(= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/not-found)@@ -188,7 +189,7 @@reason (str ": " reason)))))):else- (format "Failed to reload from git repository: %s" (ex-message e)))))+ (format "Failed to reload from git repository: %s" (or (ex-message e) (.getName (class e)))))))(defn- get-conflicts"Detects conflicts that would prevent or complicate import. Returns a map with two classes:@@ -310,10 +311,12 @@(not= pre-task-branch (settings/remote-sync-branch))))(defn- materialize-data-apps!- "After a successful content import, materialize data apps from the same- snapshot. Data apps live under `data_apps/` in the repo (outside the serdes- paths), so they ride this import rather than having their own sync. Adapts the- snapshot to plain reader fns; `data-apps.sync/sync-from-snapshot!` never throws."+ "Materialize data apps from the snapshot a content import is landing. Data apps live under `data_apps/` in+ the repo (outside the serdes paths), so they ride this import rather than having their own sync. Runs inside+ the import's commit transaction, before the version is written: a task's `version` is the sync base (see+ [[remote-sync.db/last-synced-task]]) and gates every later pull, so a version that landed without its data+ apps would never be re-materialized. Adapts the snapshot to plain reader fns;+ `data-apps.sync/sync-from-snapshot!` never throws."[^SourceSnapshot snapshot];; `read-file` returns file text (a string) or nil; data-apps.sync converts to;; bytes on its side, keeping all Java interop out of this namespace.@@ -322,6 +325,20 @@:list-dir (fn [path] (source.p/list-dir snapshot path)):sha (source.p/version snapshot)}))+(defn- import-progress-reporter+ "A throttled progress reporter (see `make-progress-reporter`) for the import task `task-id`."+ [task-id]+ (remote-sync.task/make-progress-reporter+ task-id+ {:write-fn (fn [fraction]+ ;; Inside a transaction the task row may be invisible to another connection (and on MySQL that+ ;; connection would block on it), so write on the current one; nobody outside can see it before+ ;; commit anyway. Otherwise a fresh pool connection keeps the write independent of the load's.+ (if (mdb/in-transaction?)+ (remote-sync.task/update-progress! task-id fraction)+ (t2/with-connection [_conn (mdb/app-db)]+ (remote-sync.task/update-progress! task-id fraction))))}))+(defn load-snapshot!"Loads a snapshot's serialized entities into the app DB and reconciles local state to match it:runs `load-metabase!`, toggles the `remote-sync-transforms` setting based on the snapshot's contents,@@ -335,20 +352,24 @@reconcile. That keeps the version pointer, RemoteSyncObject statuses, and the reconcile atomic: eitherthey all commit or all roll back, so a crash can never leave the version advanced past stale localstate or drop captured dirty markers (see [[import-merged!]])."- [snapshot task-id sync-timestamp & {:keys [finalize!]}]+ [snapshot report sync-timestamp & {:keys [finalize!]}]+ (report 0.05 {:force? true})(let [path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})has-transforms? (snapshot-has-transforms? base-ingestable)- ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)+ ingestable-snapshot (source.ingestable/wrap-progress-ingestable report [0.05 0.7] base-ingestable)load-result (serdes/with-cache(serialization/load-metabase! ingestable-snapshot :reindex? false))seen-paths (:seen load-result)imported-data (spec/extract-imported-entities seen-paths)]- (remote-sync.task/update-progress! task-id 0.8)+ (report 0.7 {:force? true})(when (and has-transforms?(not (settings/remote-sync-transforms)))(log/info "Detected transforms in remote source, enabling remote-sync-transforms setting")(settings/remote-sync-transforms! true))+ ;; Reported before the transaction, not inside it: a write inside would hold the task row's lock until+ ;; commit, blocking the heartbeat for the whole reconcile and hashing phase.+ (report 0.75 {:force? true})(t2/with-transaction [_conn](remove-unsynced! (spec/all-syncable-collection-ids) imported-data);; Replace the RemoteSyncObject table, folding each entity's repo file_path (so later renames/deletes@@ -358,6 +379,7 @@(insert-with-metadata! (spec/sync-all-entities! sync-timestamp imported-data)(source.ingestable/cached-file-paths base-ingestable))(when finalize! (finalize!)))+ (report 0.9 {:force? true})(when (and (not has-transforms?)(settings/remote-sync-transforms))(log/info "No transforms in remote source, disabling remote-sync-transforms setting")@@ -368,7 +390,7 @@(search/reindex! :async? (not= :h2 (mdb/db-type)))(catch Exception e(log/warn e "Search reindex after import failed")))- (remote-sync.task/update-progress! task-id 0.95)+ (report 0.95 {:force? true})imported-data));;; ------------------------------------------- Incremental Import Fast-Path -------------------------------------------@@ -451,11 +473,12 @@Renames are handled by entity identity, not path: a rename re-loads the same entity_id at the new path(an add), so the old path's delete is recognized as a rename and the entity is not removed."- [{:keys [ingestable deleted-rsos] :as _plan} snapshot-version task-id sync-timestamp & {:keys [finalize!]}]+ [{:keys [ingestable deleted-rsos] :as _plan} snapshot-version report sync-timestamp & {:keys [finalize!]}]+ (report 0.05 {:force? true})(let [load-result (when ingestable(serdes/with-cache(serialization/load-metabase!- (source.ingestable/wrap-progress-ingestable task-id 0.7 ingestable)+ (source.ingestable/wrap-progress-ingestable report [0.05 0.7] ingestable):reindex? false)))imported-data (spec/extract-imported-entities (:seen load-result))loaded-eid? (fn [model-type eid]@@ -472,7 +495,9 @@{:model_type model_type :model_id model_id})model-key-of (fn [{:keys [model_type]}] (:model-key (spec/spec-for-model-type model_type)))sync-rows (spec/sync-all-entities! sync-timestamp imported-data)]- (remote-sync.task/update-progress! task-id 0.8)+ (report 0.7 {:force? true})+ ;; Before the transaction for the same reason as in [[load-snapshot!]].+ (report 0.75 {:force? true})(t2/with-transaction [_conn](doseq [[model-key ds] (group-by model-key-of deletes)](remote-sync.db/delete-instances! model-key (mapv :model_id ds)))@@ -483,12 +508,13 @@(remote-sync.db/delete-rsos-of-keys! sync-rows)(insert-with-metadata! sync-rows (when ingestable (source.ingestable/cached-file-paths ingestable))))(when finalize! (finalize!)))+ (report 0.9 {:force? true});; We skip the whole-appdb reindex the full load runs. Added/modified entities are already;; re-indexed by the load itself — serdes' t2 insert!/update! fire the :hook/search-index;; after-insert/after-update hooks. Deletes have no such hook, so remove them explicitly.(doseq [[model-key ds] (group-by model-key-of deletes)](search/delete! model-key (mapv :model_id ds)))- (remote-sync.task/update-progress! task-id 0.95)+ (report 0.95 {:force? true})(log/info "Successfully reloaded entities from git repository"){:status :success:version snapshot-version@@ -521,7 +547,7 @@- Marks remote changes synced- Local changes stay dirty- Sets version to remote tip"- [snapshot base-snapshot task-id sync-timestamp]+ [snapshot base-snapshot task-id report sync-timestamp finalize!](let [{:keys [conflicts merged summary]} (serdes/with-cache(let [targets (spec/exportable-entities)](source/compute-merge (spec/extract-entities-for-export targets)@@ -536,12 +562,12 @@:message "Import blocked: the same content was changed both locally and on the remote branch."});; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are disjoint;; from the remote changes, so restoring them reproduces exactly the local diff vs remote. Restore +- ;; set-version run inside the load's transaction so a crash can't leave the dirty markers overwritten.+ ;; finalize! run inside the load's transaction so a crash can't leave the dirty markers overwritten.(let [dirty-objects (capture-dirty-objects)]- (load-snapshot! (source/specs->snapshot merged) task-id sync-timestamp+ (load-snapshot! (source/specs->snapshot merged) report sync-timestamp:finalize! (fn [](restore-dirty-objects! dirty-objects sync-timestamp)- (remote-sync.task/set-version! task-id (source.p/version snapshot))))+ (finalize!)))(log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"(apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)(count dirty-objects))@@ -577,15 +603,23 @@first-import? (nil? last-imported-version);; force-deletion? defaults to force? when a caller doesn't pass it.force-deletion? (if (nil? force-deletion?) force? force-deletion?)- finalize! #(remote-sync.task/set-version! task-id snapshot-version)+ da-result (volatile! nil)+ ;; Data apps before the version: the version write locks the task row until commit, which would+ ;; block the heartbeat for the whole materialization.+ finalize! (fn []+ (vreset! da-result (materialize-data-apps! snapshot))+ (remote-sync.task/set-version! task-id snapshot-version))+ report (import-progress-reporter task-id)path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths);; First-import conflicts only block the first import; deletion conflicts block every import (an;; already-configured instance must not silently delete unsynced transforms). The get-conflicts tree scan;; is itself deferred so it only runs when one of the gates is open (e.g. a forced import with;; force-deletion? true skips it entirely).blocking-conflicts (delay- (let [conflicts (delay (get-conflicts (source.p/->ingestable snapshot {:path-filters path-filters})- first-import?))]+ (let [conflicts (delay+ (report 0.02 {:force? true})+ (get-conflicts (source.p/->ingestable snapshot {:path-filters path-filters})+ first-import?))](cond-> [](and first-import? (not force?))(into (:first-import-conflicts @conflicts))@@ -616,7 +650,7 @@:outcome {:kind "pull-skipped"}}):else- (import-merged! snapshot base-snapshot task-id sync-timestamp))+ (import-merged! snapshot base-snapshot task-id report sync-timestamp finalize!));; --- Forced reload: bypasses the no-op/incremental guards. Deletion conflicts (when;; force-deletion? is false) still block; otherwise a full reload. ---@@ -633,7 +667,7 @@:else(let [_ (log/info "Remote sync full import: forced")- imported-data (load-snapshot! snapshot task-id sync-timestamp :finalize! finalize!)]+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)](log/info "Successfully reloaded entities from git repository"){:status :success:version snapshot-version@@ -654,7 +688,7 @@;; conflict. It touches only changed files, so it can't wholesale-delete unsynced transforms.(and (not @dirty?)(not= :remote-sync/incremental-not-possible @incremental-plan))- (incremental-load-snapshot! @incremental-plan snapshot-version task-id sync-timestamp :finalize! finalize!)+ (incremental-load-snapshot! @incremental-plan snapshot-version report sync-timestamp :finalize! finalize!)(seq @blocking-conflicts)(let [message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)]@@ -671,24 +705,22 @@first-import? "first import":else "changes not incrementally loadable")_ (log/infof "Remote sync full import: %s" reason)- imported-data (load-snapshot! snapshot task-id sync-timestamp :finalize! finalize!)]+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)](log/info "Successfully reloaded entities from git repository"){:status :success:version snapshot-version:outcome {:kind "pulled":count (pulled-change-count imported-data):branch (settings/remote-sync-branch)}}))]- ;; Data apps ride the pull: re-materialize from the real source snapshot- ;; (the repo file tree under `data_apps/`), not the synthetic merged- ;; snapshot `load-snapshot!` sees. They're counted outside serdes, so fold- ;; how many they upserted or removed into the outcome — otherwise a- ;; data-app-only pull would report `pull-skipped` / `count 0`.+ ;; Data apps rode the pull inside `finalize!`, materialized from the real source snapshot (the repo+ ;; file tree under `data_apps/`), not the synthetic merged snapshot `load-snapshot!` sees. They're+ ;; counted outside serdes, so fold how many they upserted or removed into the outcome — otherwise a+ ;; data-app-only pull would report `pull-skipped` / `count 0`. Paths that skipped `finalize!` left+ ;; no result: the version did not move, so neither did the apps.(if (= :success (:status result))- (let [da-result (materialize-data-apps! snapshot)- ;; Removals count too: a pull whose only change is deleting an app- ;; directory upserts nothing (`:changed` 0) but still changed what- ;; this instance serves, so it must report as a pull, not skipped.- da-changed (+ (:changed da-result 0) (:removed da-result 0))]+ ;; Removals count too: a pull whose only change is deleting an app directory upserts nothing+ ;; (`:changed` 0) but still changed what this instance serves, so it must report as a pull, not skipped.+ (let [da-changed (+ (:changed @da-result 0) (:removed @da-result 0))](cond-> result(pos? da-changed) (update :outcome fold-data-app-changes da-changed)))result))@@ -732,6 +764,9 @@(source.p/abort-commit! commit)(throw e))))))+(def ^:private export-progress-plan-done 0.33) ; phase 1 (plan) complete / serialize start+(def ^:private export-progress-serialize 0.66) ; phase 2 (serialize) complete+(defn- export-merged!"Export when the remote branch has advanced beyond the last synced version. Runs an entity-identity 3-way merge oflocal state (`models`) against the remote tip:@@ -767,8 +802,14 @@;; the version at the old base and a retry re-merges — the push is idempotent — rather than advancing;; the pointer past un-reconciled local state).(if-let [merged-snapshot (source.p/snapshot-at source version)]- (let [pulled (apply + (vals summary))]- (load-snapshot! merged-snapshot task-id sync-timestamp+ (let [pulled (apply + (vals summary))+ report (import-progress-reporter task-id)]+ ;; The fold-in load reports on the import scale; map it onto the tail of the export bar so the+ ;; bar never moves backward after the push checkpoints.+ (load-snapshot! merged-snapshot+ (fn [fraction & opts]+ (apply report (+ export-progress-serialize (* fraction (- 1.0 export-progress-serialize))) opts))+ sync-timestamp:finalize! (fn [](remote-sync.db/mark-all-rsos-synced! sync-timestamp)(remote-sync.task/set-version! task-id version)))@@ -1092,9 +1133,6 @@(doseq [id-chunk (partition-all app-db-batch-size ids)](remote-sync.db/mark-rsos-synced! id-chunk (select-keys by-id id-chunk) sync-timestamp))))-(def ^:private export-progress-plan-done 0.33) ; phase 1 (plan) complete / serialize start-(def ^:private export-progress-serialize 0.66) ; phase 2 (serialize) complete-(defn- full-export!"Re-serialize and commit the entire remote-synced set, then reconcile every RemoteSyncObject.@@ -1424,13 +1462,78 @@:details details:user-id user-id})))+(defonce ^:private running-tasks+ (atom #{}))++(defn running-task-ids+ "The IDs of the RemoteSyncTasks whose worker is running in this JVM. Multi-node safe by construction: another+ node's tasks are never in here."+ []+ @running-tasks)++(defn fail-running-tasks!+ "Fail every RemoteSyncTask in [[running-task-ids]] with `message`. For graceful shutdown: the workers die with+ the process and their rows would otherwise stay open until reaped as stale. Goes through `handle-task-result!`+ so a result that already landed, or a cancel, wins. Never throws."+ [message]+ (doseq [task-id (running-task-ids)]+ (try+ (log/infof "Failing remote sync task %d: %s" task-id message)+ (handle-task-result! {:status :error :message message} task-id)+ (catch Throwable t+ (log/errorf t "Failed to close remote sync task %d" task-id)))))++(defn- ensure-task-ended!+ "Close the RemoteSyncTask `task-id` as failed if it is still open after its worker exited. Goes through+ `handle-task-result!` so a concurrent cancel, or a result that did land, wins. Never throws: the reason the row+ is still open may be that the app DB is unreachable."+ [task-id]+ (try+ (when (some-> (remote-sync.db/task task-id) remote-sync.task/running?)+ (log/warnf "Remote sync task %d ended without recording a result; failing it" task-id)+ (handle-task-result! {:status :error :message "Task ended without recording a result"} task-id))+ (catch Throwable t+ (log/errorf t "Failed to close remote sync task %d" task-id))))++(defn run-task-body!+ "Run `sync-fn` (a fn of task-id returning a result map) for the already-created RemoteSyncTask `task-id` on the+ current thread, recording the outcome on the row. `branch` is written to the remote-sync-branch setting on+ success when non-nil; `:on-success` receives [task-id result] after a successful result is recorded.++ Guarantees, whatever `sync-fn` or the bookkeeping does: a heartbeat runs on the row for the duration, the task+ is registered in [[running-task-ids]] for the duration, and the row is ended on exit. Any `Throwable` from+ `sync-fn` becomes an `:error` result; an `Error` must not escape the worker thread, where nothing would log it+ and the row would stay open."+ [task-id branch sync-fn & {:keys [on-success]}]+ (let [stop-heartbeat! (remote-sync.task/start-heartbeat! task-id)]+ (swap! running-tasks conj task-id)+ (try+ (let [result (try+ (sync-fn task-id)+ (catch Throwable t+ (log/error t "Remote sync task failed")+ {:status :error+ :message (source-error-message t)}))]+ (handle-task-result! result task-id branch)+ (when (and on-success (= :success (:status result)))+ (try+ (on-success task-id result)+ (catch Exception e+ (log/errorf "Remote sync task :on-success function failed: %s" (ex-message e))))))+ (catch Throwable t+ (log/errorf t "Remote sync task %d bookkeeping failed" task-id))+ (finally+ (stop-heartbeat!)+ (swap! running-tasks disj task-id)+ (ensure-task-ended! task-id)))))+(defn- run-async!"Executes a remote sync task asynchronously in a virtual thread.Takes a task-type string ('import' or 'export'), a branch name to update in settings upon completion, async-fn function that takes a task-id and performs the sync operation, and an optional :on-success callbackthat receives [task-id result] after a successful sync. Creates a new task (or errors if one is already- running), then executes the sync function in a virtual thread with a timeout.+ running), then runs [[run-task-body!]] in a virtual thread with a timeout.Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 if a sync task is already in progress."[task-type branch sync-fn & {:keys [on-success]}]@@ -1439,18 +1542,7 @@(u.jvm/in-virtual-thread*(dh/with-timeout {:interrupt? true:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}- (let [result (try- (sync-fn task-id)- (catch Exception e- (log/errorf "Remote sync task failed: %s" (ex-message e))- {:status :error- :message (source-error-message e)}))]- (handle-task-result! result task-id branch)- (when (and on-success (= :success (:status result)))- (try- (on-success task-id result)- (catch Exception e- (log/errorf "Remote sync task :on-success function failed: %s" (ex-message e))))))))+ (run-task-body! task-id branch sync-fn :on-success on-success)))task))(defn async-import!diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/init.cljindex be26ebd3f12..e49ce66895a 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj@@ -5,6 +5,7 @@[metabase-enterprise.remote-sync.events :as events][metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]+ [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.task.import][metabase-enterprise.remote-sync.task.table-cleanup]@@ -39,6 +40,8 @@(save [] nil))))))(defn- remote-sync-init []+ ;; Only stale rows: a fresh open row may belong to a worker on another node sharing this app DB.+ (remote-sync.task/supersede-stale-tasks!)(if (settings/remote-sync-enabled)(do(when (= :read-only (settings/remote-sync-type))@@ -77,3 +80,7 @@(defmethod startup/def-startup-logic! ::remote-sync-setup[_](remote-sync-init))++(defmethod startup/def-shutdown-logic! ::remote-sync-shutdown+ [_]+ (impl/fail-running-tasks! "Interrupted by server shutdown"))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.cljindex 70f8e5dab7c..d8093e3af71 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj@@ -6,6 +6,8 @@[metabase-enterprise.remote-sync.schema :as remote-sync.schema][metabase.models.interface :as mi][metabase.settings.core :as setting]+ [metabase.util.jvm :as u.jvm]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][methodical.core :as methodical]@@ -114,6 +116,31 @@(vreset! last-fraction f)(write-fn f))))))))+(def default-heartbeat-interval-ms+ "Ms between heartbeat writes while a task runs. Deliberately not derived from `remote-sync-task-time-limit-ms`:+ an operator can set that below this interval, and a test would rather fail loudly than beat silently too slowly."+ 15000)++(defn start-heartbeat!+ "Start a virtual thread stamping `last_heartbeat_at` on the task with `task-id` every `interval-ms` (default+ [[default-heartbeat-interval-ms]]) for as long as it runs. Returns a zero-arg fn that stops it.++ A failed write is logged at debug and the loop continues: a DB hiccup must not kill the heartbeat, and a+ connection wait only makes the beat late. The update count is ignored because under a test transaction the+ row is invisible to the heartbeat's own connection, and because an ended row is meant to update nothing."+ ([task-id] (start-heartbeat! task-id default-heartbeat-interval-ms))+ ([task-id interval-ms]+ (let [stop (promise)]+ (u.jvm/in-virtual-thread*+ (loop []+ (when (= ::tick (deref stop interval-ms ::tick))+ (try+ (remote-sync.db/touch-task! task-id)+ (catch Throwable t+ (log/debugf t "Heartbeat write failed for remote sync task %d" task-id)))+ (recur))))+ (fn stop-heartbeat! [] (deliver stop ::stop) nil))))+(defn set-version!"Sets the version value for a sync task.@@ -147,32 +174,44 @@(remote-sync.db/end-task! task-id{:error_message error-msg}))+(defn- liveness-cutoff+ "The instant before which a running task's last sign of life makes it stale."+ []+ (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms))))+(defn current-task"Gets the current active sync task.- Returns the most recent RemoteSyncTask that is still running (started but not ended, and has reported progress- within the time limit), or nil if no active task exists."+ Returns the most recent RemoteSyncTask that is still running (started but not ended, and whose owning thread+ heartbeat or reported progress within the time limit), or nil if no active task exists."[]- (remote-sync.db/current-task (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms)))))+ (remote-sync.db/current-task (liveness-cutoff)))(defn supersede-stale-tasks!- "Marks any genuinely stale task rows as cancelled and terminated.+ "Marks any genuinely stale task rows as cancelled and terminated, and logs a warning naming each row it ended.+ Returns the ids of those rows, empty when none were stale.- A task is considered stale if it has `started_at` set, `ended_at` nil, and `last_progress_report_at`- is older than `remote-sync-task-time-limit-ms`. The DB schema requires `last_progress_report_at`- to be non-null with a default of `current_timestamp`, so a brand-new task always has a recent- value (set on insert) and is not considered stale.+ A task is considered stale if it has `started_at` set, `ended_at` nil, and its last sign of life+ (`last_heartbeat_at`, or `last_progress_report_at` when no beat was ever written) is older than+ `remote-sync-task-time-limit-ms`. The DB schema requires `last_progress_report_at` to be non-null with+ a default of `current_timestamp`, so a brand-new task always has a recent value (set on insert) and is+ not considered stale.- Called from `create-task-with-lock!` before creating a new task, to clean up rows whose owning- JVM/thread is gone or hung. Returns nothing meaningful.+ The error message written names the staleness window, so the UI can show it as the reason the sync stopped.Combined with `handle-task-result!`'s already-terminated check, this means a stale task's threadthat eventually wakes up and tries to complete will detect that its row is terminated and exitwithout writing the setting or overwriting bookkeeping."[]- (let [cutoff (t/minus (t/offset-date-time)- (t/millis (setting/get :remote-sync-task-time-limit-ms)))]- (remote-sync.db/supersede-stale-tasks! cutoff)))+ (let [cutoff (liveness-cutoff)+ minutes (max 1 (quot (setting/get :remote-sync-task-time-limit-ms) 60000))+ ids (remote-sync.db/supersede-stale-tasks!+ cutoff+ (format "Sync was interrupted: the server stopped responding for %d minute%s (it may have restarted)"+ minutes (if (= 1 minutes) "" "s")))]+ (when (seq ids)+ (log/warnf "Superseded stale remote sync tasks %s: their owners were last alive before %s" ids cutoff))+ ids))(defn most-recent-task"Gets the most recently run task, including currently running tasks.@@ -184,10 +223,10 @@(defn last-version"Gets the version that any changes are built off of.- Returns the version string from the most recent successful task (either export or import), or nil if no successful- tasks exist."+ Returns the version string of the newest task (import or export) whose commit landed, regardless of how the task+ row was closed afterwards (see [[remote-sync.db/last-synced-task]]), or nil if no task has ever synced."[]- (:version (remote-sync.db/last-successful-task)))+ (:version (remote-sync.db/last-synced-task)))(defn running?"Checks if a task is currently running.@@ -234,11 +273,12 @@Takes a RemoteSyncTask instance.- Returns true if the task is incomplete and has not reported progress within the time limit, false otherwise."+ Returns true if the task is incomplete and its owning thread has neither heartbeat nor reported progress within+ the time limit, false otherwise. A row with no heartbeat falls back to its progress stamp."[task](and (nil? (:ended_at task))- (t/< (:last_progress_report_at task)- (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms))))))+ (t/< (or (:last_heartbeat_at task) (:last_progress_report_at task))+ (liveness-cutoff))))(defn conflict?"Checks if a task ended with conflicts.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.cljindex 292d5de062d..d9c73e77200 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj@@ -21,10 +21,16 @@[:id pos-int?][:sync_task_type TaskType][:initiated_by {:optional true} [:maybe pos-int?]]+ [:initiated_by_user {:optional true} [:maybe [:map+ [:id pos-int?]+ [:first_name [:maybe :string]]+ [:last_name [:maybe :string]]+ [:email :string]]]][:progress [:maybe [:float {:min 0.0 :max 1.0}]]][:started_at :any][:ended_at {:optional true} [:maybe :any]][:last_progress_report_at {:optional true} [:maybe :any]]+ [:last_heartbeat_at {:optional true} [:maybe :any]][:version {:optional true} [:maybe :string]][:cancelled {:optional true} [:maybe :boolean]][:error_message {:optional true} [:maybe :string]]@@ -228,6 +234,7 @@[:started_at {:optional true} [:maybe ms/TemporalInstant]][:ended_at {:optional true} [:maybe ms/TemporalInstant]][:last_progress_report_at {:optional true} [:maybe ms/TemporalInstant]]+ [:last_heartbeat_at {:optional true} [:maybe ms/TemporalInstant]][:initiated_by {:optional true} [:maybe ::lib.schema.id/user]][:error_message {:optional true} [:maybe :string]][:version {:optional true} [:maybe :string]]diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex b7d9272009b..a50e9d8fe56 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -83,7 +83,7 @@:default 5)(defsetting remote-sync-task-time-limit-ms- (deferred-tru "The maximum amount of time a remote sync task will be given to complete")+ (deferred-tru "How long a remote sync task may go without proving its process is alive (via heartbeat, or progress on rows without one) before it is treated as dead and superseded. A slow but live task is never affected. The task itself is aborted after ten times this limit."):type :integer:visibility :authenticated:export? falsediff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.cljindex 30c8176c71d..c643071bf5c 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj@@ -1,15 +1,12 @@(ns metabase-enterprise.remote-sync.source.ingestable(:require[clojure.string :as str]- [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.serialization.core :as serialization]- [metabase.app-db.core :as app-db][metabase.models.serialization :as serdes][metabase.util :as u][metabase.util.log :as log]- [metabase.util.yaml :as yaml]- [toucan2.core :as t2])+ [metabase.util.yaml :as yaml])(:import(metabase_enterprise.remote_sync.source.protocol SourceSnapshot)(org.yaml.snakeyaml.error MarkedYAMLException)))@@ -78,35 +75,23 @@(serialization/ingest-errors ingestable)))(defn wrap-progress-ingestable- "Wraps an Ingestable to track and update progress during ingestion.+ "Wraps `ingestable` so that ingesting the n-th of its N entities reports the fraction `lo` + n/N * (`hi` - `lo`)+ through `report`, a fn of a fraction such as one from `make-progress-reporter`.- Takes a task-id (the integer ID of the RemoteSyncTask model to update with progress), a normalize value (the- maximum progress ratio value, with progress calculated as a fraction of this number), and an ingestable (an- Ingestable object to wrap with progress tracking).-- Returns a CallbackIngestable instance that updates task progress as items are ingested."- [task-id normalize ingestable]- (let [total (count (serialization/ingest-list ingestable))+ A failed report is logged and ignored so it can never abort the load it tracks; a cancellation raised by the+ report (see `update-progress!`) propagates and stops the load."+ [report [lo hi] ingestable]+ (let [total (max 1 (count (serialization/ingest-list ingestable)))calls (atom 0)](letfn [(progress-callback [item _](when item- (let [progress (* (/ (swap! calls inc) total) normalize)]- ;; Progress reporting must never abort the ingestion it tracks. The update runs on a- ;; separate connection, which on some app DBs can contend with the in-flight load (e.g.- ;; a MySQL lock-wait timeout), so swallow DB failures and keep going — but still honor a- ;; cancellation signal, which `update-progress!` raises to stop the task.- (try- (if (app-db/in-transaction?)- ;; The separate connection cannot see the task row if the ambient transaction created it, so it- ;; would block on that row until innodb_lock_wait_timeout before failing. Nobody outside the- ;; transaction can observe progress until it commits anyway, so stay on the current connection.- (remote-sync.task/update-progress! task-id progress)- (t2/with-connection [_conn (app-db/app-db)]- (remote-sync.task/update-progress! task-id progress)))- (catch Exception e- (if (:cancelled? (ex-data e))- (throw e)- (log/warn (u/strip-error e "Failed to report import progress; continuing"))))))))]+ (try+ ;; counted down from hi so the last entity lands on hi exactly, not a rounding neighbour+ (report (- hi (* (- hi lo) (/ (- total (swap! calls inc)) total))))+ (catch Exception e+ (if (:cancelled? (ex-data e))+ (throw e)+ (log/warn (u/strip-error e "Failed to report import progress; continuing")))))))](->CallbackIngestable ingestable progress-callback))));; Wraps another Ingestable and filters the `list-files` content to only content that has the specifieddiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.cljindex f51c7abc09e..83c9d2dfb00 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj@@ -36,16 +36,11 @@(dh/with-timeout {:interrupt? true:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}(log/info "Auto-importing remote-sync collections")- (let [result (impl/import! snapshot task-id)]- (impl/handle-task-result! result task-id)- (when (= :success (:status result))- ;; events/publish-event! rethrows handler exceptions; don't let an audit-log- ;; failure mark an already-successful import as failed- (try- (impl/publish-sync-event! :event/remote-sync-import task-id- {:branch branch :auto true} nil)- (catch Exception e- (log/errorf "Failed to publish remote-sync audit event: %s" (ex-message e))))))))))))))+ (impl/run-task-body! task-id nil+ (fn [task-id] (impl/import! snapshot task-id))+ :on-success (fn [task-id _result]+ (impl/publish-sync-event! :event/remote-sync-import task-id+ {:branch branch :auto true} nil)))))))))))(task/defjob ^{:doc "Auto-imports any remote collections."} AutoImport [_](auto-import!))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex bea13912a5e..01cd7bbf2d0 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -755,8 +755,67 @@:error_message "Task cancelled"}(mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task"))))))+(deftest current-task-closes-a-stale-open-task-test+ (testing "GET /api/ee/remote-sync/current-task closes an open task whose owner is gone and returns it cancelled"+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]+ (mt/with-temp [:model/RemoteSyncTask {id :id} {:sync_task_type "import"+ :started_at old-time+ :last_progress_report_at old-time}]+ (let [first-response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task")]+ (is (=? {:id id+ :status "cancelled"+ :cancelled true+ :ended_at some?+ :error_message #"^Sync was interrupted.*"}+ first-response))+ (testing "a second GET returns the same closed row"+ (is (=? (select-keys first-response [:id :status :cancelled :ended_at :error_message])+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task")))))))))++(deftest current-task-leaves-a-live-task-running-test+ (testing "GET /api/ee/remote-sync/current-task returns a task with a fresh heartbeat as running even when its progress is stale"+ (mt/with-temp [:model/RemoteSyncTask {id :id} {:sync_task_type "import"+ :started_at (t/minus (t/offset-date-time) (t/hours 1))+ :last_progress_report_at (t/minus (t/offset-date-time) (t/hours 1))+ :last_heartbeat_at :%now}]+ (is (=? {:id id :status "running" :ended_at nil}+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task"))))))++(deftest current-task-carries-the-trimmed-initiating-user-test+ (testing "GET /api/ee/remote-sync/current-task carries the initiating user as id, first_name, last_name, email only"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at :%now+ :last_progress_report_at :%now}]+ (let [user (:initiated_by_user (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task"))]+ (is (= {:id (mt/user->id :rasta)+ :first_name "Rasta"+ :last_name "Toucan"+ :email "[email protected]"}+ user))))))++(deftest current-task-initiating-user-is-nil-for-system-tasks-test+ (testing "GET /api/ee/remote-sync/current-task carries a nil initiating user for a task with no initiator (auto-import)"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by nil+ :started_at :%now+ :last_progress_report_at :%now}]+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task")]+ (is (contains? response :initiated_by_user))+ (is (nil? (:initiated_by_user response)))))))+;;; ------------------------------------------------- Cancel Task Endpoint -------------------------------------------------+(deftest cancel-task-carries-the-trimmed-initiating-user-test+ (testing "POST /api/ee/remote-sync/current-task/cancel returns the cancelled task with its trimmed initiating user"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "export"+ :initiated_by (mt/user->id :rasta)+ :started_at :%now+ :last_progress_report_at :%now}]+ (is (=? {:status "cancelled"+ :initiated_by_user {:id (mt/user->id :rasta) :email "[email protected]"}}+ (mt/user-http-request :crowberto :post 200 "ee/remote-sync/current-task/cancel"))))))+(deftest cancel-task-requires-superuser-test(testing "POST /api/ee/remote-sync/current-task/cancel requires superuser permissions (GHY-3804)"(mt/with-temp [:model/RemoteSyncTask {id :id} {:sync_task_type "export"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.cljindex 5bfd4dba648..6f68dcc8017 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj@@ -7,7 +7,9 @@mixed (serdes + data apps), and serdes-only."(:require[clojure.test :refer :all]+ [metabase-enterprise.data-apps.sync :as data-apps.sync][metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as rs.test][metabase.search.test-util :as search.tu]@@ -86,3 +88,39 @@(merge coll-file (app-tree! "ops" "BUNDLE-V2")))](is (= "pulled" (:kind outcome)))(is (= 2 (:count outcome)) "the collection (1) plus the changed data app (1)"))))))))++(deftest data-apps-land-with-the-version-test+ (testing "a pull records the data apps at the snapshot version it commits as the sync base"+ (search.tu/with-index-disabled+ (mt/with-premium-features #{:data-apps-preview}+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")+ "v1" (app-tree! "sales" "BUNDLE-V2")}+ :current "v0")]+ (is (= :success (:status (import-at! src "v0" :force? true))))+ (is (= :success (:status (import-at! src "v1"))))+ (is (= "v1" (remote-sync.task/last-version)))+ (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))++(deftest data-app-failure-rolls-back-the-version-test+ (testing "a crash while materializing data apps rolls the version back, so the next pull redoes the import+ instead of skipping a version whose data apps never landed"+ (search.tu/with-index-disabled+ (mt/with-premium-features #{:data-apps-preview}+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")+ "v1" (app-tree! "sales" "BUNDLE-V2")}+ :current "v0")]+ (is (= :success (:status (import-at! src "v0" :force? true))))+ (let [v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales")]+ (mt/with-dynamic-fn-redefs [data-apps.sync/sync-from-snapshot! (fn [_] (throw (ex-info "materialization died" {})))]+ (is (= :error (:status (import-at! src "v1")))))+ (is (= "v0" (remote-sync.task/last-version)) "the failed pull is not the sync base")+ (is (= v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales"))+ "the v0 app is untouched"))+ (is (=? {:status :success :outcome {:kind "pulled"}} (import-at! src "v1"))+ "the next pull redoes the import rather than skipping v1")+ (is (= "v1" (remote-sync.task/last-version)))+ (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.cljindex f807462927d..08013308a9a 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj@@ -1,13 +1,90 @@(ns metabase-enterprise.remote-sync.impl-progress-test- "Serialize-phase progress hook for remote-sync export (GHY-4132)."+ "Progress reporting through the export serialize phase (GHY-4132) and the import phases (GHY-4593)."(:require[clojure.test :refer :all][metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.models.remote-sync-task :as rst][metabase-enterprise.remote-sync.source.protocol :as source.p]- [metabase.test :as mt]))+ [metabase-enterprise.remote-sync.test-helpers :as rs.test]+ [metabase.search.test-util :as search.tu]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)+(use-fixtures :once (fixtures/initialize :db))+(use-fixtures :each rs.test/clean-remote-sync-state)++;;; ------------------------------------------------- Import phases --------------------------------------------------++(def ^:private collection-path+ "collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml")++(def ^:private card-path+ "collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/f1C68pznmrpN1F5xFDj6d_some_question.yaml")++(defn- tree [card-name]+ {collection-path (rs.test/generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")+ card-path (rs.test/generate-card-yaml "f1C68pznmrpN1F5xFDj6d" card-name "M-Q4pcV0qkiyJ0kiSWECl")})++(defn- import-capturing-progress!+ "Runs `import!` of `snapshot` with the reporter replaced by one that records every fraction it would write+ (`:throttle-ms` 0 unless `opts` say otherwise). Records the result on the task. Returns [result fractions]."+ [snapshot & {:keys [force? throttle-ms now-fn] :or {throttle-ms 0 now-fn (constantly 0)}}]+ ;; the card in [[tree]] references the test-data database, which must exist before the load+ (mt/db)+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ writes (atom [])]+ (mt/with-dynamic-fn-redefs [impl/import-progress-reporter+ (fn [_task-id]+ (rst/make-progress-reporter task-id {:throttle-ms throttle-ms+ :now-fn now-fn+ :write-fn (fn [f] (swap! writes conj f))}))]+ (let [result (impl/import! snapshot task-id :force? force?)]+ (impl/handle-task-result! result task-id)+ [result @writes]))))++(def ^:private checkpoints+ "Forced writes of a full import, in order: conflict scan, load start, load done, reconcile start, commit, reindex."+ [0.02 0.05 0.7 0.75 0.9 0.95])++(defn- approx= [expected actual]+ (and (= (count expected) (count actual))+ (every? true? (map #(< (abs (- %1 %2)) 1e-9) expected actual))))++(deftest full-import-progress-is-monotonic-and-checkpointed-test+ (testing "a first import reports 0.02 before the conflict scan, one fraction per loaded entity inside [0.05 0.7], then the fixed checkpoints"+ (search.tu/with-index-disabled+ (let [src (rs.test/versioned-source :trees {"v0" (tree "Some Question")} :current "v0")+ [result writes] (import-capturing-progress! (source.p/snapshot src))]+ (is (= :success (:status result)))+ (is (= writes (sort writes)) "never moves backward")+ ;; two entities: the collection at the halfway point of the load range, the card on its top+ (is (approx= [0.02 0.05 0.375 0.7 0.7 0.75 0.9 0.95] writes))))))++(deftest per-entity-progress-writes-are-throttled-test+ (testing "with the real throttle and a frozen clock, only the forced checkpoints are written"+ (search.tu/with-index-disabled+ (let [src (rs.test/versioned-source :trees {"v0" (tree "Some Question")} :current "v0")+ [result writes] (import-capturing-progress! (source.p/snapshot src) :throttle-ms 10000)]+ (is (= :success (:status result)))+ (is (= checkpoints writes))))))++(deftest incremental-import-progress-test+ (testing "an incremental pull skips the conflict-scan checkpoint and reports the changed entity inside the load range"+ (search.tu/with-index-disabled+ (let [src (rs.test/versioned-source :trees {"v0" (tree "Some Question") "v1" (tree "Renamed Question")} :current "v0")]+ (is (= :success (:status (first (import-capturing-progress! (source.p/snapshot-at src "v0") :force? true)))))+ (let [[result writes] (import-capturing-progress! (source.p/snapshot-at src "v1"))]+ (is (= :success (:status result)))+ (is (= {:kind "pulled" :count 1} (dissoc (:outcome result) :branch))+ "the incremental path ran: one changed entity pulled")+ ;; no conflict scan; the single changed entity lands on the top of the load range+ (is (= [0.05 0.7 0.7 0.75 0.9 0.95] writes)))))))++;;; ------------------------------------------------- Export phases --------------------------------------------------+;; A CommitBuilder that records staged paths and does nothing else.(defrecord RecordingCommit [staged]source.p/CommitBuilderdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex e0bc364d41e..30b2e2b1d4d 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -463,24 +463,24 @@(is (= :error (:status result)))(is (re-find #"Network error" (:message result))))))-(deftest import!-calls-update-progress-with-expected-values-test- (testing "import! calls update-progress! with expected progress values"+(deftest import!-reports-progress-checkpoints-test+ (testing "a first full import writes only the forced checkpoints when the per-entity writes fall inside the throttle window"(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]- (let [mock-source (test-helpers/create-mock-source)+ (let [mock-source (test-helpers/create-mock-source)progress-calls (atom [])](mt/with-dynamic-fn-redefs [remote-sync.task/update-progress!(fn [task-id progress]- (swap! progress-calls conj {:task-id task-id :progress progress}))]+ (swap! progress-calls conj {:task-id task-id :progress progress}))+ ;; a frozen clock keeps every per-entity write inside the throttle window+ remote-sync.task/make-progress-reporter+ (let [real (mt/original-fn #'remote-sync.task/make-progress-reporter)]+ (fn [task-id opts] (real task-id (assoc opts :now-fn (constantly 0)))))](let [result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- (is (= 5 (count @progress-calls)))- (is (= task-id (:task-id (first @progress-calls))))- (is (= task-id (:task-id (second @progress-calls))))- (is (= task-id (:task-id (nth @progress-calls 2))))- (is (= 0.7 (:progress (nth @progress-calls 2))))- (is (= 0.8 (:progress (nth @progress-calls 3))))- (is (= 0.95 (:progress (nth @progress-calls 4)))))))))))+ (is (every? #(= task-id (:task-id %)) @progress-calls))+ (is (= [0.02 0.05 0.7 0.75 0.9 0.95] (mapv :progress @progress-calls))+ "conflict scan, load start, load done, reconcile start, commit, reindex"))))))))(deftest import!-runs-a-single-reindex-inside-the-task-test(testing "a full import runs exactly one reindex, synchronous on H2 and asynchronous elsewhere"@@ -2320,3 +2320,87 @@ serdes/meta:"F1's edited description is present")(is (not (t2/exists? :model/FieldUserSettings :field_id f2-id))"F2's stale local row, absent from the imported file's :fields, is gone")))))))++;; ---------- run-task-body!: the row is always closed, whatever the worker does ----------------++(defn- new-task-id []+ (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)}))++(deftest run-task-body!-records-a-successful-result-test+ (testing "a :success result completes the row, runs :on-success, and unregisters the task"+ (let [task-id (new-task-id)+ on-success (atom nil)+ seen-running (atom nil)]+ (impl/run-task-body! task-id nil+ (fn [id]+ (reset! seen-running (contains? (impl/running-task-ids) id))+ {:status :success :outcome {:kind "pull-skipped"}})+ :on-success (fn [id result] (reset! on-success [id (:status result)])))+ (is (true? @seen-running) "the task is registered while its body runs")+ (is (not (contains? (impl/running-task-ids) task-id)))+ (is (= [task-id :success] @on-success))+ (is (=? {:ended_at some? :cancelled false :error_message nil :outcome {:kind "pull-skipped"}}+ (t2/select-one :model/RemoteSyncTask :id task-id))))))++(deftest run-task-body!-fails-the-row-when-sync-fn-throws-an-error-test+ (testing "an Error (not an Exception) from sync-fn still ends the row with a message"+ (let [task-id (new-task-id)]+ (impl/run-task-body! task-id nil (fn [_] (throw (Error. "boom"))))+ (is (=? {:ended_at some? :cancelled false :error_message #".*boom"}+ (t2/select-one :model/RemoteSyncTask :id task-id)))+ (is (not (contains? (impl/running-task-ids) task-id))))))++(deftest run-task-body!-fails-the-row-when-the-throwable-has-no-message-test+ (testing "a throwable with a nil message still produces a non-empty error_message"+ (let [task-id (new-task-id)]+ (impl/run-task-body! task-id nil (fn [_] (throw (StackOverflowError.))))+ (is (=? {:ended_at some? :error_message #".*StackOverflowError"}+ (t2/select-one :model/RemoteSyncTask :id task-id))))))++(deftest run-task-body!-closes-the-row-when-bookkeeping-throws-test+ (testing "when handle-task-result! itself throws, the row is still ended by the exit path"+ (let [task-id (new-task-id)+ calls (atom 0)]+ (mt/with-dynamic-fn-redefs [impl/handle-task-result! (let [orig (mt/original-fn #'impl/handle-task-result!)]+ (fn [& args]+ (if (= 1 (swap! calls inc))+ (throw (ex-info "pool exhausted" {}))+ (apply orig args))))]+ (impl/run-task-body! task-id nil (fn [_] {:status :success})))+ (is (= 2 @calls))+ (is (=? {:ended_at some? :cancelled false :error_message "Task ended without recording a result"}+ (t2/select-one :model/RemoteSyncTask :id task-id))))))++(deftest run-task-body!-does-not-clobber-a-concurrent-cancel-test+ (testing "a cancel that lands while sync-fn runs is preserved: neither the result nor the exit path overwrites it"+ (let [task-id (new-task-id)]+ (impl/run-task-body! task-id nil+ (fn [id]+ (remote-sync.task/cancel-sync-task! id)+ {:status :success}))+ (is (=? {:ended_at some? :cancelled true :error_message "Task cancelled"}+ (t2/select-one :model/RemoteSyncTask :id task-id))))))++(deftest run-task-body!-stops-the-heartbeat-on-exit-test+ (testing "the heartbeat started for the task is stopped when the body exits, including on failure"+ (let [task-id (new-task-id)+ stopped? (atom false)]+ (mt/with-dynamic-fn-redefs [remote-sync.task/start-heartbeat! (fn [_id] (fn [] (reset! stopped? true)))]+ (impl/run-task-body! task-id nil (fn [_] (throw (Error. "boom")))))+ (is (true? @stopped?)))))++(deftest run-task-body!-on-success-failure-does-not-fail-the-row-test+ (testing "an exception from :on-success is logged and the row stays successful"+ (let [task-id (new-task-id)]+ (impl/run-task-body! task-id nil (fn [_] {:status :success})+ :on-success (fn [_ _] (throw (ex-info "audit log down" {}))))+ (is (=? {:ended_at some? :cancelled false :error_message nil}+ (t2/select-one :model/RemoteSyncTask :id task-id))))))++(deftest run-task-body!-writes-the-branch-on-success-test+ (testing "a non-nil branch is written to the setting on success and left alone on error"+ (mt/with-temporary-setting-values [remote-sync-branch "main"]+ (impl/run-task-body! (new-task-id) "feature" (fn [_] {:status :error :message "nope"}))+ (is (= "main" (remote-sync.settings/remote-sync-branch)))+ (impl/run-task-body! (new-task-id) "feature" (fn [_] {:status :success}))+ (is (= "feature" (remote-sync.settings/remote-sync-branch))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.cljindex 42ccd5985ac..1ef132c482e 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj@@ -261,6 +261,24 @@;;; ------------------------------------------ First import: never incremental ------------------------------------------+(deftest cancelled-after-commit-keeps-the-sync-base-test+ (testing "a pull whose task is cancelled after its transaction committed still advances last-version, so the+ next pull of the same snapshot is skipped instead of re-importing everything"+ (do-with-bench!+ (fn [f0]+ (search.tu/with-index-disabled+ (let [f1 (update f0 (path-with f0 "card_b") str/replace "display: table" "display: line")+ src (rs.test/versioned-source :trees {"v0" f0 "v1" f1} :current "v0")]+ (is (= :success (:status (import-at! src "v0" :force? true))))+ (let [task (t2/insert-returning-pk! :model/RemoteSyncTask+ {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (is (= :success (:status (impl/import! (source.p/snapshot-at src "v1") task))))+ ;; The worker died (or an admin cancelled) between the commit and the result bookkeeping.+ (remote-sync.task/cancel-sync-task! task)+ (is (=? {:cancelled true :version "v1"} (t2/select-one :model/RemoteSyncTask :id task))))+ (is (= "v1" (remote-sync.task/last-version)))+ (is (=? {:status :success :outcome {:kind "pull-skipped"}} (import-at! src "v1")))))))))+(deftest first-import-no-force-uses-full-load-test(testing "GHY-3779: a first import (no prior version, so last-version is nil) with force? false must NOTattempt the incremental fast-path. incremental-load-snapshot! assumes local state equalsdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.cljindex e36e2699f02..b54f366192b 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj@@ -6,11 +6,19 @@[metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.init :as init][metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]+ [metabase-enterprise.remote-sync.test-helpers :as th][metabase.collections.models.collection :as collection][metabase.collections.test-utils :as collections.tu]+ [metabase.startup.core :as startup][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][toucan2.core :as t2]))+(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db))+(use-fixtures :each th/clean-remote-sync-state)+(defn- capture-async-import! [](let [calls (atom [])][calls (fn [branch force? args & _opts] (swap! calls conj [branch force? args]) nil)]))@@ -105,6 +113,55 @@(#'init/remote-sync-init)(is (empty? @calls))))))))+(defn- wait-until+ "True once `pred` returns truthy, polling every 10 ms for up to `timeout-ms`; false otherwise."+ [pred timeout-ms]+ (let [deadline (+ (System/currentTimeMillis) timeout-ms)]+ (loop []+ (cond+ (pred) true+ (> (System/currentTimeMillis) deadline) false+ :else (do (Thread/sleep 10) (recur))))))++(defn- new-task-id []+ (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)}))++(deftest remote-sync-shutdown-fails-this-jvms-running-tasks-test+ (testing "shutdown fails every task whose worker runs in this JVM and leaves a stale open row from elsewhere untouched"+ (let [other-id (new-task-id)+ _ (t2/update! :model/RemoteSyncTask other-id+ {:last_progress_report_at (t/minus (t/offset-date-time) (t/hours 2))})+ own-id (new-task-id)+ release (promise)+ worker (future (impl/run-task-body! own-id nil+ (fn [_] @release {:status :success :outcome {:kind "pull-skipped"}})))]+ (try+ (is (true? (wait-until #(contains? (impl/running-task-ids) own-id) 5000)))+ (startup/def-shutdown-logic! ::init/remote-sync-shutdown)+ (is (=? {:ended_at some? :cancelled false :error_message "Interrupted by server shutdown"}+ (t2/select-one :model/RemoteSyncTask :id own-id)))+ (is (=? {:ended_at nil :error_message nil}+ (t2/select-one :model/RemoteSyncTask :id other-id)))+ (finally+ (deliver release nil)+ (is (not= ::timeout (deref worker 10000 ::timeout)))))+ (testing "the worker's late result does not overwrite the shutdown bookkeeping"+ (is (=? {:error_message "Interrupted by server shutdown" :outcome nil}+ (t2/select-one :model/RemoteSyncTask :id own-id)))))))++(deftest remote-sync-init-supersedes-stale-tasks-test+ (testing "boot closes open task rows whose owner has been silent past the window and leaves fresh ones alone"+ (mt/with-temporary-setting-values [:remote-sync-url nil]+ (let [stale-id (new-task-id)+ _ (t2/update! :model/RemoteSyncTask stale-id+ {:last_progress_report_at (t/minus (t/offset-date-time) (t/hours 2))})+ fresh-id (new-task-id)]+ (#'init/remote-sync-init)+ (is (=? {:cancelled true :ended_at some? :error_message #"^Sync was interrupted.*"}+ (t2/select-one :model/RemoteSyncTask :id stale-id)))+ (is (=? {:cancelled false :ended_at nil :error_message nil}+ (t2/select-one :model/RemoteSyncTask :id fresh-id)))))))+;;; ------------------------------------------- Glossary ledger backfill -------------------------------------------(defn- glossary-rso-count []diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.cljindex cf4245c307e..2c9d21d6800 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj@@ -3,6 +3,7 @@(:require[clojure.test :refer :all][java-time.api :as t]+ [metabase-enterprise.remote-sync.db :as remote-sync.db][metabase-enterprise.remote-sync.models.remote-sync-task :as rst][metabase-enterprise.remote-sync.test-helpers :as th][metabase.models.interface :as mi]@@ -365,6 +366,129 @@(finally(rst/complete-sync-task! (:id task)))))))+(deftest timed-out?-uses-heartbeat-over-progress-test+ (testing "timed-out? is false when the heartbeat is fresh even though progress is stale"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))+ old-time (t/minus (t/offset-date-time) (t/hours 2))]+ (try+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time+ :last_heartbeat_at (t/offset-date-time)})+ (is (false? (rst/timed-out? (t2/select-one :model/RemoteSyncTask :id (:id task)))))+ (finally+ (rst/complete-sync-task! (:id task))))))+ (testing "timed-out? is true when the heartbeat is stale even though progress is fresh"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))+ old-time (t/minus (t/offset-date-time) (t/hours 2))]+ (try+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at (t/offset-date-time)+ :last_heartbeat_at old-time})+ (is (true? (rst/timed-out? (t2/select-one :model/RemoteSyncTask :id (:id task)))))+ (finally+ (rst/complete-sync-task! (:id task))))))+ (testing "timed-out? falls back to progress when no heartbeat was ever written"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))+ old-time (t/minus (t/offset-date-time) (t/hours 2))]+ (try+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time+ :last_heartbeat_at nil})+ (is (true? (rst/timed-out? (t2/select-one :model/RemoteSyncTask :id (:id task)))))+ (finally+ (rst/complete-sync-task! (:id task)))))))++(deftest current-task-uses-heartbeat-over-progress-test+ (testing "current-task returns a task whose heartbeat is fresh and progress stale"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))+ old-time (t/minus (t/offset-date-time) (t/hours 2))]+ (try+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time+ :last_heartbeat_at (t/offset-date-time)})+ (is (= (:id task) (:id (rst/current-task))))+ (finally+ (rst/complete-sync-task! (:id task))))))+ (testing "current-task ignores a task whose heartbeat is stale and progress fresh"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))+ old-time (t/minus (t/offset-date-time) (t/hours 2))]+ (try+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at (t/offset-date-time)+ :last_heartbeat_at old-time})+ (is (nil? (rst/current-task)))+ (finally+ (rst/complete-sync-task! (:id task)))))))++;;; ------------------------------------------------------------------------------------------------+;;; Tests for touch-task! and start-heartbeat!+;;; ------------------------------------------------------------------------------------------------++(defn- wait-until+ "True once `pred` returns truthy, polling every 10 ms for up to `timeout-ms`; false otherwise."+ [pred timeout-ms]+ (let [deadline (+ (System/currentTimeMillis) timeout-ms)]+ (loop []+ (cond+ (pred) true+ (> (System/currentTimeMillis) deadline) false+ :else (do (Thread/sleep 10) (recur))))))++(deftest touch-task!-stamps-heartbeat-only-test+ (testing "touch-task! sets last_heartbeat_at and leaves last_progress_report_at alone on a running task"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))+ old-time (t/minus (t/offset-date-time) (t/hours 2))]+ (try+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time})+ (let [before (t2/select-one :model/RemoteSyncTask :id (:id task))]+ (is (nil? (:last_heartbeat_at before)))+ (is (= 1 (remote-sync.db/touch-task! (:id task))))+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id task))]+ (is (some? (:last_heartbeat_at after)))+ (is (= (:last_progress_report_at before) (:last_progress_report_at after)))))+ (finally+ (rst/complete-sync-task! (:id task)))))))++(deftest touch-task!-ignores-ended-task-test+ (testing "touch-task! updates nothing on a cancelled task, so a late beat cannot revive it"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))]+ (rst/cancel-sync-task! (:id task))+ (let [before (t2/select-one :model/RemoteSyncTask :id (:id task))]+ (is (= 0 (remote-sync.db/touch-task! (:id task))))+ (is (= before (t2/select-one :model/RemoteSyncTask :id (:id task))))))))++(deftest start-heartbeat!-beats-until-stopped-test+ (testing "the heartbeat writes on every interval and stops writing once the stop fn is called"+ (let [writes (atom 0)]+ (mt/with-dynamic-fn-redefs [remote-sync.db/touch-task! (fn [_task-id] (swap! writes inc) 1)]+ (let [stop! (rst/start-heartbeat! 1 20)]+ (try+ (is (true? (wait-until #(>= @writes 3) 2000)))+ (finally+ (stop!)))+ (let [after-stop @writes]+ (Thread/sleep 100)+ (is (= after-stop @writes))))))))++(deftest start-heartbeat!-survives-write-failure-test+ (testing "a failing write does not end the heartbeat loop"+ (let [calls (atom 0)]+ (mt/with-dynamic-fn-redefs [remote-sync.db/touch-task! (fn [_task-id]+ (when (= 1 (swap! calls inc))+ (throw (ex-info "db down" {})))+ 1)]+ (let [stop! (rst/start-heartbeat! 1 20)]+ (try+ (is (true? (wait-until #(>= @calls 3) 2000)))+ (finally+ (stop!))))))))++(deftest start-heartbeat!-writes-the-row-test+ (testing "the heartbeat advances last_heartbeat_at on the real row"+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))+ stop! (rst/start-heartbeat! (:id task) 20)]+ (try+ (is (true? (wait-until #(some? (t2/select-one-fn :last_heartbeat_at :model/RemoteSyncTask :id (:id task)))+ 2000)))+ (finally+ (stop!)+ (rst/complete-sync-task! (:id task)))))))+(deftest update-progress-throws-on-cancelled-task-test(testing "update-progress! throws exception when updating a cancelled task"(let [task (rst/create-sync-task! "import" (mt/user->id :rasta))]@@ -375,35 +499,52 @@(deftest last-version-test(testing "When there are no tasks, last-version returns nil"(is (nil? (rst/last-version))))- (testing "When there are no successful tasks, last-version returns nil"+ (testing "When no task has recorded a version, last-version returns nil"(let [task (rst/create-sync-task! "import" (mt/user->id :rasta))](rst/fail-sync-task! (:id task) "Test failure")(is (nil? (rst/last-version)))))- (testing "Returns last successful version"+ (testing "Returns the version of the newest task whose commit landed"(let [successful-task (rst/create-sync-task! "import" (mt/user->id :rasta))](rst/complete-sync-task! (:id successful-task))(rst/set-version! (:id successful-task) "version 1")(is (= "version 1" (rst/last-version)))- (testing "Ignores cancelled tasks"+ (testing "Ignores a cancelled task with no version"(rst/cancel-sync-task! (:id (rst/create-sync-task! "import" (mt/user->id :rasta))))(is (= "version 1" (rst/last-version))))- (testing "Ignores failed tasks"+ (testing "Ignores a failed task with no version"(rst/fail-sync-task! (:id (rst/create-sync-task! "import" (mt/user->id :rasta))) "Error")(is (= "version 1" (rst/last-version))))- (testing "Ignores incomplete tasks"- (is (= "version 1" (rst/last-version))))- (testing "Returns newer successful tasks"+ (testing "Ignores an open task with no version"+ (let [open-task (rst/create-sync-task! "import" (mt/user->id :rasta))]+ (is (= "version 1" (rst/last-version)))+ (rst/complete-sync-task! (:id open-task))))+ (testing "Ignores a conflict task even though it records the version it conflicted against"+ (let [conflict-task (rst/create-sync-task! "import" (mt/user->id :rasta))]+ (rst/set-version! (:id conflict-task) "version 1.5")+ (rst/conflict-sync-task! (:id conflict-task) ["some conflict"])+ (is (= "version 1" (rst/last-version)))))+ (testing "Returns a newer successful task's version"(let [new-task (rst/create-sync-task! "import" (mt/user->id :rasta))](rst/complete-sync-task! (:id new-task))(rst/set-version! (:id new-task) "version 2")- (is (= "version 2" (rst/last-version))))))))+ (is (= "version 2" (rst/last-version)))))+ (testing "A task cancelled after its commit landed is still the sync base"+ (let [cancelled-task (rst/create-sync-task! "import" (mt/user->id :rasta))]+ (rst/set-version! (:id cancelled-task) "version 3")+ (rst/cancel-sync-task! (:id cancelled-task))+ (is (= "version 3" (rst/last-version)))))+ (testing "A task that failed after its commit landed is still the sync base"+ (let [failed-task (rst/create-sync-task! "export" (mt/user->id :rasta))]+ (rst/set-version! (:id failed-task) "version 4")+ (rst/fail-sync-task! (:id failed-task) "bookkeeping failed")+ (is (= "version 4" (rst/last-version))))))));;; ------------------------------------------------------------------------------------------------;;; Tests for supersede-stale-tasks!;;; ------------------------------------------------------------------------------------------------;;;-;;; Used by `create-task-with-lock!` (auto-import path) to clean up rows whose owning JVM/thread-;;; is gone or hung. Must NOT supersede brand-new tasks that haven't reported progress yet.+;;; Called at boot, by `GET /current-task`, and before a new task is created, to close rows whose owning+;;; JVM/thread is gone or hung. Must NOT supersede brand-new tasks that haven't reported progress yet.(defn- insert-task!"Helper: insert a RemoteSyncTask row with the given fields, bypassing the create-sync-task!@@ -422,11 +563,22 @@stale-task (insert-task! {:started_at old-time:last_progress_report_at old-time:progress 0.5})]- (rst/supersede-stale-tasks!)+ (mt/with-temporary-setting-values [:remote-sync-task-time-limit-ms 120000]+ (rst/supersede-stale-tasks!))(let [after (t2/select-one :model/RemoteSyncTask :id (:id stale-task))](is (true? (:cancelled after)))(is (some? (:ended_at after)))- (is (= "Superseded after staleness timeout" (:error_message after)))))))+ (is (= "Sync was interrupted: the server stopped responding for 2 minutes (it may have restarted)"+ (:error_message after)))))))++(deftest supersede-stale-tasks!-message-uses-singular-for-one-minute-test+ (testing "the interruption message reads '1 minute' when the window is one minute"+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))+ stale-task (insert-task! {:started_at old-time :last_progress_report_at old-time})]+ (mt/with-temporary-setting-values [:remote-sync-task-time-limit-ms 60000]+ (rst/supersede-stale-tasks!))+ (is (= "Sync was interrupted: the server stopped responding for 1 minute (it may have restarted)"+ (t2/select-one-fn :error_message :model/RemoteSyncTask :id (:id stale-task)))))))(deftest supersede-stale-tasks!-leaves-brand-new-tasks-alone-test(testing "supersede-stale-tasks! must NOT mark a brand-new task that just inserted —@@ -458,6 +610,29 @@"task with recent progress must not be terminated"))(rst/complete-sync-task! (:id active-task)))))+(deftest supersede-stale-tasks!-uses-heartbeat-over-progress-test+ (testing "supersede-stale-tasks! leaves a task with a fresh heartbeat alone even when its progress is stale"+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))+ task (insert-task! {:started_at old-time+ :last_progress_report_at old-time+ :last_heartbeat_at (t/offset-date-time)+ :progress 0.5})]+ (rst/supersede-stale-tasks!)+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id task))]+ (is (false? (:cancelled after)))+ (is (nil? (:ended_at after))))+ (rst/complete-sync-task! (:id task))))+ (testing "supersede-stale-tasks! reaps a task with a stale heartbeat even when its progress is fresh"+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))+ task (insert-task! {:started_at old-time+ :last_progress_report_at (t/offset-date-time)+ :last_heartbeat_at old-time+ :progress 0.5})]+ (rst/supersede-stale-tasks!)+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id task))]+ (is (true? (:cancelled after)))+ (is (some? (:ended_at after)))))))+(deftest supersede-stale-tasks!-leaves-terminated-tasks-alone-test(testing "supersede-stale-tasks! must NOT touch tasks that already have ended_at set"(let [old-time (t/minus (t/offset-date-time) (t/hours 1))@@ -473,6 +648,24 @@(is (= (:ended_at before) (:ended_at after))"ended_at must not be overwritten")))))+(deftest supersede-stale-tasks!-returns-and-logs-reaped-ids-test+ (testing "superseding returns the ids of the rows it ended and logs one warning naming them"+ (let [now (t/offset-date-time)+ old-time (t/minus now (t/hours 1))+ stale (insert-task! {:started_at old-time :last_progress_report_at old-time})+ fresh (insert-task! {:started_at now :last_progress_report_at now})]+ (mt/with-log-messages-for-level [messages [metabase-enterprise.remote-sync.models.remote-sync-task :warn]]+ (is (= [(:id stale)] (rst/supersede-stale-tasks!)))+ (is (=? [{:level :warn+ :message (re-pattern (str "Superseded stale remote sync tasks \\[" (:id stale) "\\]: .*"))}]+ (messages))))+ (is (nil? (:ended_at (t2/select-one :model/RemoteSyncTask :id (:id fresh)))))+ (testing "a second pass finds nothing and logs nothing"+ (mt/with-log-messages-for-level [messages [metabase-enterprise.remote-sync.models.remote-sync-task :warn]]+ (is (= [] (rst/supersede-stale-tasks!)))+ (is (= [] (messages)))))+ (rst/complete-sync-task! (:id fresh)))))+;;; ------------------------------------------------------------------------------------------------;;; Tests for make-progress-reporter;;; ------------------------------------------------------------------------------------------------diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.cljindex a07997f3cd8..01715759e0e 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj@@ -1,15 +1,14 @@(ns metabase-enterprise.remote-sync.source.ingestable-test(:require[clojure.test :refer :all]+ [metabase-enterprise.remote-sync.models.remote-sync-task :as rst][metabase-enterprise.remote-sync.source.ingestable :as ingestable][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase-enterprise.serialization.core :as serialization][metabase.audit-app.events.audit-log][metabase.test.fixtures :as fixtures]- [metabase.test.util :as mt]- [metabase.users.settings]- [toucan2.core :as t2]))+ [metabase.users.settings]))(comment metabase.audit-app.events.audit-log/keep-memetabase.users.settings/keep-me)@@ -66,50 +65,31 @@(is (= 3 (count @calls)) "Callback should be called three times"))))))(deftest wrap-progress-ingestable-test- (testing "wrap-progress-ingestable creates CallbackIngestable with progress tracking"- ;; This test cannot use with-temp because the callback uses a separate connection to make- ;; sure it is updating progress outside of the transaction serdes is using- (mt/with-model-cleanup [:model/User :model/RemoteSyncTask]- (let [user (first (t2/insert-returning-instances! :model/User {:first_name "Test"- :last_name "User"- :email "[email protected]"- :password "password123"}))- task (first (t2/insert-returning-instances! :model/RemoteSyncTask {:sync_task_type "import"- :initiated_by (:id user)}))- task-id (:id task)- mock-source (test-helpers/create-mock-source)- base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil) (atom []))- normalize 100]- (testing "creates a CallbackIngestable"- (let [wrapped (ingestable/wrap-progress-ingestable task-id normalize base-ingestable)]- (is (instance? metabase_enterprise.remote_sync.source.ingestable.CallbackIngestable wrapped)- "Should return CallbackIngestable instance")))- (testing "updates task progress in database as items are ingested"- (let [wrapped (ingestable/wrap-progress-ingestable task-id normalize base-ingestable)- paths (serialization/ingest-list wrapped)- total-paths (count paths)]- (is (seq paths) "Should have paths to ingest")- (let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)]- (is (nil? (:progress initial-task)) "Progress should be nil initially"))- (serialization/ingest-one wrapped (first paths))- (let [task-after-first (t2/select-one :model/RemoteSyncTask :id task-id)]- (is (some? (:progress task-after-first)) "Progress should be updated after first item")- (is (< (abs (- (:progress task-after-first) (double (* (/ 1 total-paths) normalize)))) 0.01)- "Progress should reflect one item ingested")- (is (some? (:last_progress_report_at task-after-first))- "last_progress_report_at should be set"))- (serialization/ingest-one wrapped (second paths))- (let [task-after-second (t2/select-one :model/RemoteSyncTask :id task-id)]- (is (< (abs (- (:progress task-after-second) (double (* (/ 2 total-paths) normalize)))) 0.01)- "Progress should reflect two items ingested"))))- (testing "progress reaches normalize value when all items ingested"- (let [wrapped (ingestable/wrap-progress-ingestable task-id normalize base-ingestable)- paths (serialization/ingest-list wrapped)]- (doseq [path paths]- (serialization/ingest-one wrapped path))- (let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)]- (is (< (abs (- (:progress final-task) (double normalize))) 0.01)- "Progress should equal normalize value when all items ingested"))))))))+ (let [mock-source (test-helpers/create-mock-source)+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil) (atom []))+ paths (serialization/ingest-list base-ingestable)+ total (count paths)+ approx= (fn [expected actual] (< (abs (- expected actual)) 1e-9))]+ (is (pos? total))+ (testing "the n-th of N ingested entities reports lo + n/N * (hi - lo) through the reporter"+ (let [writes (atom [])+ report (rst/make-progress-reporter 1 {:throttle-ms 0 :write-fn (fn [f] (swap! writes conj f))})+ wrapped (ingestable/wrap-progress-ingestable report [0.05 0.7] base-ingestable)]+ (is (instance? metabase_enterprise.remote_sync.source.ingestable.CallbackIngestable wrapped))+ (doseq [path paths]+ (serialization/ingest-one wrapped path))+ (is (= total (count @writes)) "one report per ingested entity")+ (is (every? true? (map approx=+ (map #(+ 0.05 (* (/ % total) 0.65)) (range 1 (inc total)))+ @writes)))+ (is (= 0.7 (last @writes)) "the last entity lands on hi exactly")))+ (testing "a failing report is logged and the entity is still ingested"+ (let [wrapped (ingestable/wrap-progress-ingestable (fn [_] (throw (RuntimeException. "db down"))) [0.0 1.0] base-ingestable)]+ (is (some? (serialization/ingest-one wrapped (first paths))))))+ (testing "a cancellation raised by the report propagates and stops the ingestion"+ (let [wrapped (ingestable/wrap-progress-ingestable (fn [_] (throw (ex-info "cancelled" {:cancelled? true}))) [0.0 1.0] base-ingestable)]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"cancelled"+ (serialization/ingest-one wrapped (first paths))))))))(deftest root-dependency-ingestable-test(testing "RootDependencyIngestable filters items based on root dependencies"diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsxindex b389347c0ba..47ac85db5a8 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx@@ -15,13 +15,22 @@ const initialState = createMockState({const noop = () => {};-export const Stalled = () => (+const startedAt = new Date(Date.now() - 10 * 60 * 1000).toISOString();++const initiatedByUser = {+ id: 1,+ first_name: "Cynthia",+ last_name: "Balusek",+ email: "[email protected]",+};++export const InProgress = () => (<ReduxProvider storeInitialState={initialState}><SyncProgressModaltaskType="export"progress={0.3}- isStalled- minutesSinceLastUpdate={6}+ startedAt={startedAt}+ initiatedByUser={initiatedByUser}isError={false}errorMessage=""isSuccess={false}@@ -31,11 +40,15 @@ export const Stalled = () => (</ReduxProvider>);-export const InProgress = () => (+export const Quiet = () => (<ReduxProvider storeInitialState={initialState}><SyncProgressModal- taskType="export"- progress={0.3}+ taskType="import"+ progress={0.8}+ isQuiet+ minutesSinceLastUpdate={3}+ startedAt={startedAt}+ initiatedByUser={initiatedByUser}isError={false}errorMessage=""isSuccess={false}@@ -44,3 +57,20 @@ export const InProgress = () => (/></ReduxProvider>);++export const Cancelled = () => (+ <ReduxProvider storeInitialState={initialState}>+ <SyncProgressModal+ taskType="import"+ progress={0.21}+ isCancelled+ startedAt={startedAt}+ initiatedByUser={null}+ isError={false}+ errorMessage="Sync was interrupted: the server stopped responding for 5 minutes (it may have restarted)"+ isSuccess={false}+ outcome={null}+ onDismiss={noop}+ />+ </ReduxProvider>+);diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsxindex 470268c52c8..5223fed876f 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx@@ -1,18 +1,29 @@-import { msgid, ngettext, t } from "ttag";+import { c, msgid, ngettext, t } from "ttag";import { ActionButton } from "metabase/common/components/ActionButton";import { useToast } from "metabase/common/hooks";import { getUserIsAdmin } from "metabase/current-user";+import { dayjs } from "metabase/dayjs";import { useSelector } from "metabase/redux";import { Button, Group, Modal, Progress, Stack, Text } from "metabase/ui";+import { getUserName } from "metabase/utils/user";import { useCancelRemoteSyncCurrentTaskMutation } from "metabase-enterprise/api";-import type { RemoteSyncOutcome, RemoteSyncTaskType } from "metabase-types/api";+import type {+ RemoteSyncOutcome,+ RemoteSyncTaskType,+ RemoteSyncTaskUser,+} from "metabase-types/api";++import { getProgressPhaseLabel } from "../utils";interface SyncProgressModalProps {taskType: RemoteSyncTaskType;progress: number;- isStalled?: boolean;+ isQuiet?: boolean;+ isCancelled?: boolean;minutesSinceLastUpdate?: number | null;+ startedAt?: string | null;+ initiatedByUser?: RemoteSyncTaskUser | null;isError: boolean;errorMessage: string;isSuccess: boolean;@@ -23,15 +34,18 @@ interface SyncProgressModalProps {export function SyncProgressModal({progress,taskType,- isStalled = false,+ isQuiet = false,+ isCancelled = false,minutesSinceLastUpdate = null,+ startedAt = null,+ initiatedByUser = null,isError,errorMessage,isSuccess,outcome,onDismiss,}: SyncProgressModalProps) {- const canCancel = useSelector(getUserIsAdmin);+ const isAdmin = useSelector(getUserIsAdmin);const [cancelRemoteSyncCurrentTask] =useCancelRemoteSyncCurrentTaskMutation();@@ -63,12 +77,17 @@ export function SyncProgressModal({}};+ const startedBy = (+ <StartedByLine startedAt={startedAt} initiatedByUser={initiatedByUser} />+ );+if (isError) {return (<Modal onClose={onDismiss} opened size="md" title={t`Sync failed`}><Stack mt="lg" gap="lg"><Text>{t`An error occurred during sync.`}</Text>{errorMessage && <Text>{errorMessage}</Text>}+ {startedBy}<Group justify="flex-end"><Buttondata-testid="sync-error-close-button"@@ -100,6 +119,25 @@ export function SyncProgressModal({);}+ if (isCancelled) {+ return (+ <Modal onClose={onDismiss} opened size="md" title={t`Sync stopped`}>+ <Stack mt="lg" gap="lg">+ <Text>{errorMessage || t`The sync was cancelled.`}</Text>+ <StoppedAtLine progress={progress} taskType={taskType} />+ {startedBy}+ <Group justify="flex-end">+ <Button+ data-testid="sync-cancelled-close-button"+ onClick={onDismiss}+ variant="filled"+ >{t`Close`}</Button>+ </Group>+ </Stack>+ </Modal>+ );+ }+const { title, progressLabel } = getModalContent(taskType);return (@@ -111,22 +149,16 @@ export function SyncProgressModal({withCloseButton={false}><Stack mt="lg" gap="lg">- {isStalled ? (- <Text ta="center">{getStalledMessage(minutesSinceLastUpdate)}</Text>- ) : (- <>- <Text ta="center">{progressLabel}</Text>- <Progress- value={progress * 100}- transitionDuration={300}- animated- />- </>+ <Text ta="center">{progressLabel}</Text>+ <Progress value={progress * 100} transitionDuration={300} animated />+ {isQuiet && (+ <Text ta="center">{getQuietMessage(minutesSinceLastUpdate)}</Text>)}+ {startedBy}<Text size="sm">{t`Please wait until this finishes before editing content.`}</Text>- {canCancel && (+ {isAdmin && (<Group justify="flex-end"><ActionButtonactionFn={onCancel}@@ -141,15 +173,57 @@ export function SyncProgressModal({);}-function getStalledMessage(minutesSinceLastUpdate: number | null): string {+function StartedByLine({+ startedAt,+ initiatedByUser,+}: {+ startedAt: string | null;+ initiatedByUser: RemoteSyncTaskUser | null;+}) {+ if (!startedAt) {+ return null;+ }++ const time = dayjs(startedAt).format("LT");+ const name = initiatedByUser ? getUserName(initiatedByUser) : null;++ return (+ <Text size="sm" c="text-secondary" data-testid="sync-started-by">+ {name+ ? c("{0} is a person's name, {1} is a time of day")+ .t`Started by ${name} at ${time}`+ : c("{0} is a time of day").t`Started at ${time}`}+ </Text>+ );+}++function StoppedAtLine({+ progress,+ taskType,+}: {+ progress: number;+ taskType: RemoteSyncTaskType;+}) {+ const percent = Math.round(progress * 100);+ const phase = getProgressPhaseLabel(progress, taskType);++ return (+ <Text size="sm" c="text-secondary" data-testid="sync-stopped-at">+ {c("{0} is a percentage, {1} is a sync phase such as 'importing content'")+ .t`Stopped at ${percent}% while ${phase}`}+ </Text>+ );+}++function getQuietMessage(minutesSinceLastUpdate: number | null): string {if (minutesSinceLastUpdate == null) {- return t`Still working…`;+ return t`No progress reported recently. The sync is still running.`;}const minutes = minutesSinceLastUpdate;return ngettext(- msgid`Still working. No update for ${minutes} minute.`,- `Still working. No update for ${minutes} minutes.`,+ msgid`No progress for ${minutes} minute. The sync is still running.`,+ `No progress for ${minutes} minutes. The sync is still running.`,minutes,);}diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsxindex e8553949e9a..bddd59ca3eb 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx@@ -6,16 +6,32 @@ import {} from "__support__/server-mocks";import { createMockState } from "__support__/state";import { renderWithProviders, screen, waitFor } from "__support__/ui";-import type { RemoteSyncOutcome } from "metabase-types/api";+import type { RemoteSyncOutcome, RemoteSyncTaskUser } from "metabase-types/api";import { createMockUser } from "metabase-types/api/mocks";import { SyncProgressModal } from "./SyncProgressModal";+const STARTED_AT = "2026-09-15T22:06:23Z";+const STARTED_AT_TIME = new Date(STARTED_AT).toLocaleTimeString([], {+ hour: "numeric",+ minute: "2-digit",+});++const INITIATED_BY_USER: RemoteSyncTaskUser = {+ id: 231,+ first_name: "Cynthia",+ last_name: "Balusek",+ email: "[email protected]",+};+const setup = ({taskType = "import" as const,progress = 0.5,- isStalled = false,+ isQuiet = false,+ isCancelled = false,minutesSinceLastUpdate = null,+ startedAt = null,+ initiatedByUser = null,isError = false,errorMessage = "",isSuccess = false,@@ -26,8 +42,11 @@ const setup = ({}: {taskType?: "import" | "export";progress?: number;- isStalled?: boolean;+ isQuiet?: boolean;+ isCancelled?: boolean;minutesSinceLastUpdate?: number | null;+ startedAt?: string | null;+ initiatedByUser?: RemoteSyncTaskUser | null;isError?: boolean;errorMessage?: string;isSuccess?: boolean;@@ -46,8 +65,11 @@ const setup = ({<SyncProgressModaltaskType={taskType}progress={progress}- isStalled={isStalled}+ isQuiet={isQuiet}+ isCancelled={isCancelled}minutesSinceLastUpdate={minutesSinceLastUpdate}+ startedAt={startedAt}+ initiatedByUser={initiatedByUser}isError={isError}errorMessage={errorMessage}isSuccess={isSuccess}@@ -107,26 +129,144 @@ describe("SyncProgressModal", () => {});});- describe("stalled progress state (GHY-4132)", () => {- it("replaces the frozen progress bar with a 'still working' message when the task has stalled", () => {- // Reproduces GHY-4132: a large push stops reporting progress, the backend marks the still-running- // task :timed-out, and the UI used to sit on a frozen 'Exporting content…' bar. Instead we report- // honestly that it is still running but hasn't reported progress recently.- setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 6 });+ describe("quiet running state", () => {+ it("keeps the progress bar and says the sync is still running when progress has not moved", () => {+ setup({+ taskType: "export",+ progress: 0.3,+ isQuiet: true,+ minutesSinceLastUpdate: 6,+ });expect(- screen.getByText("Still working. No update for 6 minutes."),+ screen.getByText(+ "No progress for 6 minutes. The sync is still running.",+ ),+ ).toBeInTheDocument();+ expect(screen.getByRole("progressbar")).toBeInTheDocument();+ expect(screen.getByText("Exporting content…")).toBeInTheDocument();+ expect(+ screen.getByRole("button", { name: "Cancel" }),).toBeInTheDocument();- expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();- expect(screen.queryByText("Exporting content…")).not.toBeInTheDocument();});it("uses the singular 'minute' when only one minute has passed", () => {- setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 1 });+ setup({ isQuiet: true, minutesSinceLastUpdate: 1 });++ expect(+ screen.getByText(+ "No progress for 1 minute. The sync is still running.",+ ),+ ).toBeInTheDocument();+ });++ it("shows no hint while progress is fresh", () => {+ setup({ isQuiet: false, minutesSinceLastUpdate: 0 });++ expect(screen.queryByText(/still running/)).not.toBeInTheDocument();+ });+ });++ describe("cancelled state", () => {+ it("shows the row's message with a Close button and no progress bar", async () => {+ const onDismiss = jest.fn();+ setup({+ taskType: "import",+ progress: 0.21,+ isCancelled: true,+ errorMessage:+ "Sync was interrupted: the server stopped responding for 5 minutes (it may have restarted)",+ onDismiss,+ });+ expect(screen.getByText("Sync stopped")).toBeInTheDocument();expect(- screen.getByText("Still working. No update for 1 minute."),+ screen.getByText(+ "Sync was interrupted: the server stopped responding for 5 minutes (it may have restarted)",+ ),).toBeInTheDocument();+ expect(+ screen.getByText("Stopped at 21% while importing content"),+ ).toBeInTheDocument();+ expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();+ expect(+ screen.queryByRole("button", { name: "Cancel" }),+ ).not.toBeInTheDocument();++ await userEvent.click(screen.getByTestId("sync-cancelled-close-button"));++ expect(onDismiss).toHaveBeenCalled();+ });++ it("falls back to generic copy when the row has no message", () => {+ setup({ isCancelled: true, errorMessage: "" });++ expect(screen.getByText("The sync was cancelled.")).toBeInTheDocument();+ });+ });++ describe("started-by line", () => {+ it("names the initiating user and the start time", () => {+ setup({+ isCancelled: true,+ startedAt: STARTED_AT,+ initiatedByUser: INITIATED_BY_USER,+ });++ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(+ `Started by Cynthia Balusek at ${STARTED_AT_TIME}`,+ );+ });++ it("falls back to the email when the user has no name", () => {+ setup({+ isCancelled: true,+ startedAt: STARTED_AT,+ initiatedByUser: {+ ...INITIATED_BY_USER,+ first_name: null,+ last_name: null,+ },+ });++ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(+ `Started by [email protected] at ${STARTED_AT_TIME}`,+ );+ });++ it("shows only the start time for a task with no initiating user", () => {+ setup({ isError: true, startedAt: STARTED_AT, initiatedByUser: null });++ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(+ `Started at ${STARTED_AT_TIME}`,+ );+ });++ it("is shown on the running view so a second admin can see whose task it is", () => {+ setup({+ startedAt: STARTED_AT,+ initiatedByUser: INITIATED_BY_USER,+ });++ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(+ "Started by Cynthia Balusek",+ );+ });++ it("is omitted when the start time is unknown", () => {+ setup({ isCancelled: true, startedAt: null });++ expect(screen.queryByTestId("sync-started-by")).not.toBeInTheDocument();+ });++ it("is omitted from the success view", () => {+ setup({+ isSuccess: true,+ startedAt: STARTED_AT,+ initiatedByUser: INITIATED_BY_USER,+ });++ expect(screen.queryByTestId("sync-started-by")).not.toBeInTheDocument();});});diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.tsindex ae72d53cb60..59028d16eb7 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts@@ -19,6 +19,8 @@ export const TRANSFORMS_KEY = "remote-sync-transforms";export const COLLECTIONS_KEY = "collections";// Used in modal variant when library doesn't exist yet but user wants to sync itexport const SYNC_LIBRARY_PENDING_KEY = "sync-library-pending";+// A running task whose progress has not moved for this long gets a "still running" hint in the modal+export const SYNC_QUIET_AFTER_MINUTES = 2;export const REMOTE_SYNC_SCHEMA = Yup.object({[REMOTE_SYNC_KEY]: Yup.boolean().nullable().default(true),diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsxindex 2c8dffc081f..d17ac4a3a48 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx@@ -4,21 +4,23 @@ import { useSetting } from "metabase/settings";import { useGetRemoteSyncCurrentTaskQuery } from "metabase-enterprise/api";import { SyncProgressModal } from "../components/SyncProgressModal";-import { REMOTE_SYNC_KEY } from "../constants";+import { REMOTE_SYNC_KEY, SYNC_QUIET_AFTER_MINUTES } from "../constants";import {getErrorMessage,getHasPendingMutation,+ getInitiatedByUser,+ getIsCancelled,getIsError,getIsRunning,- getIsStalled,getIsSuccess,getLastProgressReportAt,getProgress,getShowModal,+ getStartedAt,getTaskOutcome,getTaskType,} from "../selectors";-import { modalDismissed } from "../sync-task-slice";+import { modalDismissed, taskCleared } from "../sync-task-slice";const SYNC_STATUS_POLL_INTERVAL = 2000;@@ -31,8 +33,10 @@ export const useSyncStatus = () => {const taskType = useSelector(getTaskType);const progress = useSelector(getProgress);const isError = useSelector(getIsError);- const isStalled = useSelector(getIsStalled);+ const isCancelled = useSelector(getIsCancelled);const lastProgressReportAt = useSelector(getLastProgressReportAt);+ const startedAt = useSelector(getStartedAt);+ const initiatedByUser = useSelector(getInitiatedByUser);const errorMessage = useSelector(getErrorMessage);const isSuccess = useSelector(getIsSuccess);const outcome = useSelector(getTaskOutcome);@@ -42,26 +46,41 @@ export const useSyncStatus = () => {? dayjs().diff(dayjs(lastProgressReportAt), "minute"): null;+ const isQuiet =+ isRunning &&+ minutesSinceLastUpdate !== null &&+ minutesSinceLastUpdate >= SYNC_QUIET_AFTER_MINUTES;+const shouldPoll = isRunning && showModal && !hasPendingMutation;+ // Always subscribed, so a fresh tab discovers a sync started elsewhere (or a stale row the backend+ // repairs on read); polling only while a task is being watched.useGetRemoteSyncCurrentTaskQuery(undefined, {pollingInterval: shouldPoll ? SYNC_STATUS_POLL_INTERVAL : undefined,skipPollingIfUnfocused: true,- skip: !isRemoteSyncEnabled || !shouldPoll,+ skip: !isRemoteSyncEnabled,});+ // A stopped task has nothing left to poll, so drop it instead of hiding the modal; otherwise the next+ // taskUpdated for the same stale row would reopen it.+ const onDismiss = () =>+ dispatch(isCancelled ? taskCleared() : modalDismissed());+const progressModal =showModal && taskType ? (<SyncProgressModaltaskType={taskType}progress={progress}- isStalled={isStalled}+ isQuiet={isQuiet}+ isCancelled={isCancelled}minutesSinceLastUpdate={minutesSinceLastUpdate}+ startedAt={startedAt}+ initiatedByUser={initiatedByUser}isError={isError}errorMessage={errorMessage}isSuccess={isSuccess}outcome={outcome}- onDismiss={() => dispatch(modalDismissed())}+ onDismiss={onDismiss}/>) : null;diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.unit.spec.tsxnew file mode 100644index 00000000000..d937528bc1f--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.unit.spec.tsx@@ -0,0 +1,66 @@+import fetchMock from "fetch-mock";++import { setupEnterprisePlugins } from "__support__/enterprise";+import { mockSettings } from "__support__/settings";+import { createMockState } from "__support__/state";+import { renderHookWithProviders, waitFor } from "__support__/ui";+import type { RemoteSyncTask } from "metabase-types/api";+import {+ createMockTokenFeatures,+ createMockUser,+} from "metabase-types/api/mocks";++import { useSyncStatus } from "./use-sync-status";++const CURRENT_TASK = "path:/api/ee/remote-sync/current-task";++const setup = ({+ task,+ enabled = true,+}: {+ task: Partial<RemoteSyncTask> | null;+ enabled?: boolean;+}) => {+ // the plugin registers its reducer and middleware only once the premium gate passes+ const settings = mockSettings({+ "token-features": createMockTokenFeatures({ remote_sync: true }),+ "remote-sync-enabled": enabled,+ "remote-sync-type": "read-write",+ "remote-sync-branch": "main",+ });+ setupEnterprisePlugins();+ fetchMock.get(CURRENT_TASK, task ? { status: 200, body: task } : 204);++ return renderHookWithProviders(() => useSyncStatus(), {+ storeInitialState: createMockState({+ currentUser: createMockUser({ is_superuser: true }),+ settings,+ }),+ });+};++const settle = () => new Promise((resolve) => setTimeout(resolve, 100));++// The test store carries no plugin middleware, so these cover the subscription itself; what a fetched+// task does to the store is covered by the listener middleware spec.+describe("useSyncStatus", () => {+ it("fetches once and stays idle when there is no task", async () => {+ const { result } = setup({ task: null });++ await waitFor(() => {+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(1);+ });+ await settle();++ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(1);+ expect(result.current.isRunning).toBe(false);+ expect(result.current.progressModal).toBeNull();+ });++ it("does not fetch when remote sync is disabled", async () => {+ setup({ task: null, enabled: false });+ await settle();++ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(0);+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.tsindex 62959252763..64252324590 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts@@ -9,6 +9,7 @@ import { tag } from "metabase-enterprise/api/tags";import type { RemoteSyncTaskStatus } from "metabase-types/api";import { REMOTE_SYNC_INVALIDATION_TAGS } from "../constants";+import { getCurrentTask } from "../selectors";import {modalDismissed,syncConflictVariantUpdated,@@ -109,15 +110,29 @@ const terminalTaskStates: RemoteSyncTaskStatus[] = ["successful","errored","cancelled",- "timed-out",] as const;remoteSyncListenerMiddleware.startListening({matcher: remoteSyncApi.endpoints.getRemoteSyncCurrentTask.matchFulfilled,- effect: async (action, { dispatch }) => {+ effect: async (action, { dispatch, getOriginalState }) => {const task = action.payload;if (task) {+ // The query is subscribed whenever remote sync is enabled, so a fetch made while nothing is being+ // watched is discovery, not an event: a task still running elsewhere is picked up so this tab can+ // follow it, while a finished one is history and is left alone. Treating history as an event would+ // replay an old conflict on every page load and, since the terminal handling invalidates this+ // query's own tag, refetch forever.+ const previous = getCurrentTask(getOriginalState());+ const wasRunning = previous !== null && previous.ended_at === null;++ if (!wasRunning) {+ if (task.ended_at === null) {+ dispatch(taskUpdated(task));+ }+ return;+ }+dispatch(taskUpdated(task));if (task.status === "conflict") {diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.tsindex e53eb0faad1..887a10b456f 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.ts@@ -22,7 +22,11 @@ import {createMockSettings,} from "metabase-types/api/mocks";-import { initialState, remoteSyncReducer } from "../sync-task-slice";+import {+ initialState,+ remoteSyncReducer,+ taskStarted,+} from "../sync-task-slice";import { remoteSyncListenerMiddleware } from "./remote-sync-listener-middleware";@@ -31,12 +35,13 @@ const createTestStore = () => {const store = configureStore({reducer: combineReducers({- remoteSyncPlugin: remoteSyncReducer,+ // nested as the app registers it (PLUGIN_REDUCERS -> state.plugins), which the selectors read+ plugins: combineReducers({ remoteSyncPlugin: remoteSyncReducer }),// EnterpriseApi is an enhanced version of Api, so they share the same reducer[Api.reducerPath]: Api.reducer,}),preloadedState: {- remoteSyncPlugin: initialState,+ plugins: { remoteSyncPlugin: initialState },// Settings are served from the getSessionProperties RTK Query cache.[Api.reducerPath]: seedApiQueryCache(undefined, [{@@ -96,12 +101,12 @@ describe("remote-sync-listener-middleware", () => {// Wait for the request to complete and middleware to processawait waitForCondition(() => {const state = store.getState();- return state.remoteSyncPlugin?.showModal === true;+ return state.plugins.remoteSyncPlugin.showModal === true;});const state = store.getState();- expect(state.remoteSyncPlugin?.showModal).toBe(true);- expect(state.remoteSyncPlugin?.currentTask?.sync_task_type).toBe(+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(true);+ expect(state.plugins.remoteSyncPlugin.currentTask?.sync_task_type).toBe("import",);});@@ -130,8 +135,8 @@ describe("remote-sync-listener-middleware", () => {await new Promise((resolve) => setTimeout(resolve, 100));const state = store.getState();- expect(state.remoteSyncPlugin?.showModal).toBe(false);- expect(state.remoteSyncPlugin?.currentTask).toBeNull();+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();});it("should NOT show modal when settings save fails", async () => {@@ -159,8 +164,8 @@ describe("remote-sync-listener-middleware", () => {await new Promise((resolve) => setTimeout(resolve, 100));const state = store.getState();- expect(state.remoteSyncPlugin?.showModal).toBe(false);- expect(state.remoteSyncPlugin?.currentTask).toBeNull();+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();});it("should NOT show modal when disabling remote sync", async () => {@@ -187,8 +192,8 @@ describe("remote-sync-listener-middleware", () => {await new Promise((resolve) => setTimeout(resolve, 100));const state = store.getState();- expect(state.remoteSyncPlugin?.showModal).toBe(false);- expect(state.remoteSyncPlugin?.currentTask).toBeNull();+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();});});@@ -212,12 +217,12 @@ describe("remote-sync-listener-middleware", () => {// The import listener triggers on matchPending, so modal should show immediatelyawait waitForCondition(() => {const state = store.getState();- return state.remoteSyncPlugin?.showModal === true;+ return state.plugins.remoteSyncPlugin.showModal === true;});const state = store.getState();- expect(state.remoteSyncPlugin?.showModal).toBe(true);- expect(state.remoteSyncPlugin?.currentTask?.sync_task_type).toBe(+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(true);+ expect(state.plugins.remoteSyncPlugin.currentTask?.sync_task_type).toBe("import",);});@@ -247,8 +252,8 @@ describe("remote-sync-listener-middleware", () => {await new Promise((resolve) => setTimeout(resolve, 100));const state = store.getState();- expect(state.remoteSyncPlugin?.showModal).toBe(false);- expect(state.remoteSyncPlugin?.currentTask).toBeNull();+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();});it("should set conflict variant to 'setup' when import fails with conflict", async () => {@@ -256,10 +261,13 @@ describe("remote-sync-listener-middleware", () => {status: 200,body: {status: "conflict",+ sync_task_type: "import",+ ended_at: "2026-01-01T00:00:01Z",},});const store = createTestStore();+ store.dispatch(taskStarted({ taskType: "import" }));// Dispatch the mutationstore.dispatch(@@ -271,13 +279,12 @@ describe("remote-sync-listener-middleware", () => {fetchMock.callHistory.done("path:/api/ee/remote-sync/current-task"),);- expect(store.getState().remoteSyncPlugin?.showModal).toBe(false);-await waitFor(() => {- expect(store.getState().remoteSyncPlugin?.syncConflictVariant).toBe(- "setup",- );+ expect(+ store.getState().plugins.remoteSyncPlugin.syncConflictVariant,+ ).toBe("setup");});+ expect(store.getState().plugins.remoteSyncPlugin.showModal).toBe(false);});it("does NOT open the setup modal when an export task ends in conflict", async () => {@@ -285,10 +292,15 @@ describe("remote-sync-listener-middleware", () => {// not by the middleware — so the middleware must not route them to the setup-conflict modal.fetchMock.get("path:/api/ee/remote-sync/current-task", {status: 200,- body: { status: "conflict", sync_task_type: "export" },+ body: {+ status: "conflict",+ sync_task_type: "export",+ ended_at: "2026-01-01T00:00:01Z",+ },});const store = createTestStore();+ store.dispatch(taskStarted({ taskType: "export" }));store.dispatch(remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),@@ -299,11 +311,101 @@ describe("remote-sync-listener-middleware", () => {);await waitFor(() => {- expect(store.getState().remoteSyncPlugin?.showModal).toBe(false);+ expect(+ store.getState().plugins.remoteSyncPlugin.currentTask?.status,+ ).toBe("conflict");+ });+ expect(store.getState().plugins.remoteSyncPlugin.showModal).toBe(false);+ expect(+ store.getState().plugins.remoteSyncPlugin.syncConflictVariant,+ ).not.toBe("setup");+ });+ });++ describe("getRemoteSyncCurrentTask listener", () => {+ const CURRENT_TASK = "path:/api/ee/remote-sync/current-task";+ const settle = () => new Promise((resolve) => setTimeout(resolve, 100));++ it("follows a task found running on load so the tab can watch it", async () => {+ fetchMock.get(CURRENT_TASK, {+ status: 200,+ body: {+ id: 5,+ status: "running",+ sync_task_type: "import",+ progress: 0.4,+ ended_at: null,+ },});- expect(store.getState().remoteSyncPlugin?.syncConflictVariant).not.toBe(- "setup",+ const store = createTestStore();++ store.dispatch(+ remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),);++ await waitFor(() => {+ expect(store.getState().plugins.remoteSyncPlugin.currentTask?.id).toBe(+ 5,+ );+ });+ expect(store.getState().plugins.remoteSyncPlugin.showModal).toBe(true);+ });++ it("leaves a finished task found on load alone", async () => {+ fetchMock.get(CURRENT_TASK, {+ status: 200,+ body: {+ id: 5,+ status: "conflict",+ sync_task_type: "import",+ ended_at: "2026-01-01T00:00:01Z",+ },+ });+ const store = createTestStore();++ store.dispatch(+ remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),+ );++ await waitForCondition(() => fetchMock.callHistory.done(CURRENT_TASK));+ await settle();++ expect(store.getState().plugins.remoteSyncPlugin.currentTask).toBeNull();+ expect(+ store.getState().plugins.remoteSyncPlugin.syncConflictVariant,+ ).toBeNull();+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(1);+ });++ it("refreshes the task once when a watched task ends, not in a loop", async () => {+ fetchMock.get(CURRENT_TASK, {+ status: 200,+ body: {+ id: 5,+ status: "successful",+ sync_task_type: "import",+ progress: 1,+ ended_at: "2026-01-01T00:00:01Z",+ },+ });+ const store = createTestStore();+ store.dispatch(taskStarted({ taskType: "import" }));++ store.dispatch(+ remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),+ );++ await waitFor(() => {+ expect(store.getState().plugins.remoteSyncPlugin.currentTask?.id).toBe(+ 5,+ );+ });+ // the terminal handling invalidates this query's tag, so the subscribed query refetches once+ await waitFor(() => {+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(2);+ });+ await settle();+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(2);});});diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.tsindex 4782d691aaf..f2f6fae3f67 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts@@ -40,9 +40,9 @@ export const getIsError = createSelector((currentTask) => currentTask?.status === "errored",);-export const getIsStalled = createSelector(+export const getIsCancelled = createSelector(getCurrentTask,- (currentTask) => currentTask?.status === "timed-out",+ (currentTask) => currentTask?.status === "cancelled",);export const getLastProgressReportAt = createSelector(@@ -50,6 +50,16 @@ export const getLastProgressReportAt = createSelector((currentTask) => currentTask?.last_progress_report_at ?? null,);+export const getStartedAt = createSelector(+ getCurrentTask,+ (currentTask) => currentTask?.started_at ?? null,+);++export const getInitiatedByUser = createSelector(+ getCurrentTask,+ (currentTask) => currentTask?.initiated_by_user ?? null,+);+export const getIsSuccess = createSelector(getCurrentTask,(currentTask) => currentTask?.status === "successful",diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.unit.spec.tsnew file mode 100644index 00000000000..ba283854ba9--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.unit.spec.ts@@ -0,0 +1,57 @@+import { createMockState } from "__support__/state";+import type { RemoteSyncTask } from "metabase-types/api";++import { getInitiatedByUser, getIsCancelled, getStartedAt } from "./selectors";+import { initialState } from "./sync-task-slice";+import type { RemoteSyncStoreState } from "./types";++const task: RemoteSyncTask = {+ id: 2080,+ sync_task_type: "import",+ status: "cancelled",+ progress: 0.21,+ started_at: "2026-09-15T22:06:23Z",+ ended_at: "2026-09-15T22:40:00Z",+ last_progress_report_at: "2026-09-15T22:08:39Z",+ error_message: "Task cancelled",+ initiated_by: 231,+ initiated_by_user: {+ id: 231,+ first_name: "Cynthia",+ last_name: "Balusek",+ email: "[email protected]",+ },+};++const stateWith = (+ currentTask: RemoteSyncTask | null,+): RemoteSyncStoreState => ({+ ...createMockState(),+ plugins: { remoteSyncPlugin: { ...initialState, currentTask } },+});++describe("remote_sync selectors", () => {+ it("getIsCancelled is true only for a cancelled task", () => {+ expect(getIsCancelled(stateWith(task))).toBe(true);+ expect(getIsCancelled(stateWith({ ...task, status: "successful" }))).toBe(+ false,+ );+ expect(getIsCancelled(stateWith(null))).toBe(false);+ });++ it("getStartedAt returns the start time or null", () => {+ expect(getStartedAt(stateWith(task))).toBe("2026-09-15T22:06:23Z");+ expect(getStartedAt(stateWith(null))).toBeNull();+ });++ it("getInitiatedByUser returns the trimmed user, or null when the task has none", () => {+ expect(getInitiatedByUser(stateWith(task))).toEqual(task.initiated_by_user);+ expect(+ getInitiatedByUser(stateWith({ ...task, initiated_by_user: null })),+ ).toBeNull();+ expect(+ getInitiatedByUser(stateWith({ ...task, initiated_by_user: undefined })),+ ).toBeNull();+ expect(getInitiatedByUser(stateWith(null))).toBeNull();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.tsindex cf83e8d374c..e290edd3920 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.ts@@ -34,6 +34,7 @@ export const remoteSyncSlice = createSlice({started_at: new Date().toISOString(),ended_at: null,last_progress_report_at: null,+ last_heartbeat_at: null,error_message: null,initiated_by: 0,};diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.tsindex 27e36db1a3b..3a71e8b14e5 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts@@ -7,6 +7,7 @@ import type {IconName,RemoteSyncEntityStatus,RemoteSyncRequiredSync,+ RemoteSyncTaskType,SettingDefinition,} from "metabase-types/api";@@ -57,6 +58,33 @@ export const getEnvSettingProps = <T>(return {};};+// Thresholds follow the backend checkpoints: import reports 0.05 after the conflict scan, 0.70 after the+// load and 0.90 after the ledger commit; export reports 0.33 after planning and 0.66 after serializing.+export const getProgressPhaseLabel = (+ progress: number,+ taskType: RemoteSyncTaskType,+): string => {+ if (taskType === "export") {+ if (progress < 0.33) {+ return t`preparing`;+ }+ if (progress < 0.66) {+ return t`exporting content`;+ }+ return t`pushing`;+ }+ if (progress < 0.05) {+ return t`preparing`;+ }+ if (progress < 0.7) {+ return t`importing content`;+ }+ if (progress < 0.9) {+ return t`recording sync state`;+ }+ return t`finishing`;+};+export const getSyncStatusIcon = (status: RemoteSyncEntityStatus): IconName => {switch (status) {case "create":diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.tsindex 0f99c07be9d..a65bedbd9c4 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts@@ -10,6 +10,7 @@ import {getBlockedReason,getCollectionPathSegments,getListedRequiredSyncs,+ getProgressPhaseLabel,getRequiredSyncRow,isTableChildModel,} from "./utils";@@ -201,6 +202,28 @@ describe("remote_sync utils", () => {});});+ describe("getProgressPhaseLabel", () => {+ it("maps import progress onto the backend checkpoints", () => {+ expect(getProgressPhaseLabel(0, "import")).toBe("preparing");+ expect(getProgressPhaseLabel(0.049, "import")).toBe("preparing");+ expect(getProgressPhaseLabel(0.05, "import")).toBe("importing content");+ expect(getProgressPhaseLabel(0.69, "import")).toBe("importing content");+ expect(getProgressPhaseLabel(0.7, "import")).toBe("recording sync state");+ expect(getProgressPhaseLabel(0.89, "import")).toBe(+ "recording sync state",+ );+ expect(getProgressPhaseLabel(0.9, "import")).toBe("finishing");+ expect(getProgressPhaseLabel(1, "import")).toBe("finishing");+ });++ it("maps export progress onto the plan, serialize and push phases", () => {+ expect(getProgressPhaseLabel(0, "export")).toBe("preparing");+ expect(getProgressPhaseLabel(0.33, "export")).toBe("exporting content");+ expect(getProgressPhaseLabel(0.66, "export")).toBe("pushing");+ expect(getProgressPhaseLabel(1, "export")).toBe("pushing");+ });+ });+describe("isTableChildModel", () => {it("should return true for field model", () => {expect(isTableChildModel("field")).toBe(true);diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.tsindex 6564e2b2ce6..f2321e216ca 100644--- a/frontend/src/metabase-types/api/remote-sync.ts+++ b/frontend/src/metabase-types/api/remote-sync.ts@@ -217,6 +217,13 @@ export type RemoteSyncOutcome =| { kind: "push-skipped" }| { kind: "merged"; pulled: number; pushed: number; branch: string };+export type RemoteSyncTaskUser = {+ id: UserId;+ first_name: string | null;+ last_name: string | null;+ email: string;+};+export type RemoteSyncTask = {id: number;sync_task_type: RemoteSyncTaskType;@@ -225,9 +232,12 @@ export type RemoteSyncTask = {started_at: string | null;ended_at: string | null;last_progress_report_at: string | null;+ last_heartbeat_at?: string | null;error_message: string | null;outcome?: RemoteSyncOutcome | null;initiated_by: UserId;+ /** Absent on auto-imports, which have no initiating user. */+ initiated_by_user?: RemoteSyncTaskUser | null;conflicts?: string[];};diff --git a/resources/migrations/065/20260915_remote_sync_task_last_heartbeat_at.yaml b/resources/migrations/065/20260915_remote_sync_task_last_heartbeat_at.yamlnew file mode 100644index 00000000000..b3ac919ad02--- /dev/null+++ b/resources/migrations/065/20260915_remote_sync_task_last_heartbeat_at.yaml@@ -0,0 +1,25 @@+## Add remote_sync_task.last_heartbeat_at: stamped every few seconds by the thread running the task so a+## stale value means the owning process died, independent of how long the current phase stays silent.+## last_progress_report_at keeps meaning "progress last changed", so a live-but-hung task stays+## distinguishable from a dead one (GHY-4593).++databaseChangeLog:++ - changeSet:+ id: v65.2026-09-15T00:00:00+ author: tplude+ comment: Add remote_sync_task.last_heartbeat_at, stamped periodically by the running thread; null on rows written before the column existed or by a worker that died before its first beat+ changes:+ - addColumn:+ tableName: remote_sync_task+ columns:+ - column:+ name: last_heartbeat_at+ type: ${timestamp_type}+ remarks: Heartbeat timestamp stamped periodically while the task runs; a stale value means the owning process died. Null before the first beat.+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: remote_sync_task+ columnName: last_heartbeat_at