Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GHY-4593] Make remote sync robust against restarts (#82553)

Tyler Pludeb08a2f0e6503
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
index 01aa0db5cc1..dec09ed73b6 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
@@ -177,12 +177,24 @@
:force_push_casualties force-push-casualties
:reason (some-> reason name)}))
+(defn- present-task
+ "Hydrate `task` for the API: its status and the initiating user trimmed to what the UI shows."
+ [task]
+ (-> task
+ (t2/hydrate :status :initiated_by_user)
+ (update :initiated_by_user #(some-> % (select-keys [:id :first_name :last_name :email])))))
+
(api.macros/defendpoint :get "/current-task" :- [:maybe remote-sync.schema/SyncTask]
"Get the current sync task"
[]
(api/check-superuser)
- (when-let [task (remote-sync.task/most-recent-task)]
- (t2/hydrate task :status)))
+ (when-let [task (some-> (remote-sync.task/most-recent-task) present-task)]
+ (if (= :timed-out (:status task))
+ ;; The owning worker is gone (its heartbeat stopped), so close the row now rather than leaving it to the
+ ;; next task creation. Idempotent and safe to race across nodes and polling tabs.
+ (do (remote-sync.task/supersede-stale-tasks!)
+ (present-task (remote-sync.task/most-recent-task)))
+ task)))
(api.macros/defendpoint :post "/current-task/cancel" :- remote-sync.schema/SyncTask
"Cancels the current task if one is running"
@@ -191,7 +203,7 @@
(let [task (remote-sync.task/most-recent-task)]
(api/check-400 (and (some? task) (remote-sync.task/running? task)) "No active task to cancel")
(remote-sync.task/cancel-sync-task! (:id task))
- (t2/hydrate (remote-sync.task/most-recent-task) :status)))
+ (present-task (remote-sync.task/most-recent-task))))
(api.macros/defendpoint :post "/test-connection" :- remote-sync.schema/TestConnectionResponse
"Test whether the Remote Sync credentials can reach the git repository.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index cbe6924d7eb..b50621aeb5d 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -649,14 +649,19 @@
[task-id :- ms/PositiveInt]
(t2/select-one-fn :cancelled :model/RemoteSyncTask :id task-id))
+(def ^:private last-alive-at
+ "The time the task's owning thread last proved it was alive: its heartbeat, or its last progress write for
+ rows that predate the heartbeat column or whose worker died before its first beat."
+ [:coalesce :last_heartbeat_at :last_progress_report_at])
+
(mu/defn current-task
- "The newest started, unfinished RemoteSyncTask that reported progress after `progress-cutoff`, or nil."
- [progress-cutoff :- ms/TemporalInstant]
+ "The newest started, unfinished RemoteSyncTask whose owner was alive after `liveness-cutoff`, or nil."
+ [liveness-cutoff :- ms/TemporalInstant]
(t2/select-one :model/RemoteSyncTask
{:where [:and
[:<> :started_at nil]
[:= :ended_at nil]
- [:< progress-cutoff :last_progress_report_at]]
+ [:< liveness-cutoff last-alive-at]]
:limit 1
:order-by [[:started_at :desc]
[:id :desc]]}))
@@ -671,15 +676,16 @@
:order-by [[:started_at :desc]
[:id :desc]]}))
-(mu/defn last-successful-task
- "The newest finished RemoteSyncTask that was neither cancelled nor failed and recorded a version, or nil."
+(mu/defn last-synced-task
+ "The newest RemoteSyncTask whose commit the local content matches, or nil. `version` is written inside the
+ transaction that commits an import or a push, and by the conflict path (which also writes `conflicts`), so
+ `version` set with `conflicts` null identifies a landed commit whatever `ended_at`, `cancelled`, or
+ `error_message` say: a task cancelled or superseded after its transaction committed is still the sync base."
[]
(t2/select-one :model/RemoteSyncTask
{:where [:and
- [:<> nil :ended_at]
- [:= false :cancelled]
- [:= nil :error_message]
- [:<> nil :version]]
+ [:<> nil :version]
+ [:= nil :conflicts]]
:limit 1
:order-by [[:started_at :desc]
[:id :desc]]}))
@@ -707,17 +713,29 @@
progress :- number?]
(t2/update! :model/RemoteSyncTask task-id {:progress progress, :last_progress_report_at :%now}))
-(mu/defn supersede-stale-tasks!
- "Cancel and end now the started, unfinished RemoteSyncTasks that last reported progress before `cutoff`."
- [cutoff :- ms/TemporalInstant]
- (t2/query {:update (t2/table-name :model/RemoteSyncTask)
- :set {:cancelled true
- :ended_at :%now
- :error_message "Superseded after staleness timeout"}
- :where [:and
- [:<> :started_at nil]
- [:= :ended_at nil]
- [:< :last_progress_report_at cutoff]]}))
+(mu/defn touch-task!
+ "Stamp the heartbeat time of the RemoteSyncTask with `task-id` if it has not ended, returning the number of rows
+ updated. Never touches an ended row, so a cancel or supersede is not undone by a late beat."
+ [task-id :- ms/PositiveInt]
+ (t2/update! :model/RemoteSyncTask {:id task-id, :ended_at nil} {:last_heartbeat_at :%now}))
+
+(mu/defn supersede-stale-tasks! :- [:sequential ms/PositiveInt]
+ "Cancel and end now, with `message` as the error message, the started, unfinished RemoteSyncTasks whose owner
+ was last alive before `cutoff`. Returns the ids of the rows ended, empty when none were stale."
+ [cutoff :- ms/TemporalInstant
+ message :- :string]
+ (let [stale [:and
+ [:<> :started_at nil]
+ [:= :ended_at nil]
+ [:< last-alive-at cutoff]]
+ ids (vec (t2/select-pks-vec :model/RemoteSyncTask {:where stale}))]
+ (when (seq ids)
+ (t2/query {:update (t2/table-name :model/RemoteSyncTask)
+ :set {:cancelled true
+ :ended_at :%now
+ :error_message message}
+ :where [:and stale [:in :id ids]]}))
+ ids))
(mu/defn delete-tasks-started-before!
"Delete the RemoteSyncTasks started before `cutoff`, returning the number deleted."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index c8ab89eed2e..f2455b949ce 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -141,25 +141,26 @@
Takes a throwable exception and returns a string message that categorizes the error (network, authentication,
repository not found, branch, or generic) based on the exception type and message content."
[e]
- (let [missing-db (cause-with-error e :metabase.models.serialization.resolve.db/database-not-found)]
+ (let [missing-db (cause-with-error e :metabase.models.serialization.resolve.db/database-not-found)
+ message (or (ex-message e) "")]
(cond
(or (instance? java.net.UnknownHostException e)
(instance? java.net.UnknownHostException (ex-cause e)))
"Network error: Unable to reach git repository host"
- (str/includes? (ex-message e) "Authentication failed")
+ (str/includes? message "Authentication failed")
"Authentication failed: Please check your git credentials"
- (str/includes? (ex-message e) "Repository not found")
+ (str/includes? message "Repository not found")
"Repository not found: Please check the repository URL"
- (str/includes? (ex-message e) "branch")
+ (str/includes? message "branch")
"Branch error: Please check the specified branch exists"
(some-> e ex-cause ex-message (str/includes? "Can't create a tenant collection without tenants enabled"))
"This repository contains tenant collections, but the tenants feature is disabled on your instance."
- (str/includes? (ex-message e) "Missing commit")
+ (str/includes? message "Missing commit")
"Repository cache is stale: the remote repository may have been force-pushed. Please retry the operation."
(= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/not-found)
@@ -188,7 +189,7 @@
reason (str ": " reason))))))
:else
- (format "Failed to reload from git repository: %s" (ex-message e)))))
+ (format "Failed to reload from git repository: %s" (or (ex-message e) (.getName (class e)))))))
(defn- get-conflicts
"Detects conflicts that would prevent or complicate import. Returns a map with two classes:
@@ -310,10 +311,12 @@
(not= pre-task-branch (settings/remote-sync-branch))))
(defn- materialize-data-apps!
- "After a successful content import, materialize data apps from the same
- snapshot. Data apps live under `data_apps/` in the repo (outside the serdes
- paths), so they ride this import rather than having their own sync. Adapts the
- snapshot to plain reader fns; `data-apps.sync/sync-from-snapshot!` never throws."
+ "Materialize data apps from the snapshot a content import is landing. Data apps live under `data_apps/` in
+ the repo (outside the serdes paths), so they ride this import rather than having their own sync. Runs inside
+ the import's commit transaction, before the version is written: a task's `version` is the sync base (see
+ [[remote-sync.db/last-synced-task]]) and gates every later pull, so a version that landed without its data
+ apps would never be re-materialized. Adapts the snapshot to plain reader fns;
+ `data-apps.sync/sync-from-snapshot!` never throws."
[^SourceSnapshot snapshot]
;; `read-file` returns file text (a string) or nil; data-apps.sync converts to
;; bytes on its side, keeping all Java interop out of this namespace.
@@ -322,6 +325,20 @@
:list-dir (fn [path] (source.p/list-dir snapshot path))
:sha (source.p/version snapshot)}))
+(defn- import-progress-reporter
+ "A throttled progress reporter (see `make-progress-reporter`) for the import task `task-id`."
+ [task-id]
+ (remote-sync.task/make-progress-reporter
+ task-id
+ {:write-fn (fn [fraction]
+ ;; Inside a transaction the task row may be invisible to another connection (and on MySQL that
+ ;; connection would block on it), so write on the current one; nobody outside can see it before
+ ;; commit anyway. Otherwise a fresh pool connection keeps the write independent of the load's.
+ (if (mdb/in-transaction?)
+ (remote-sync.task/update-progress! task-id fraction)
+ (t2/with-connection [_conn (mdb/app-db)]
+ (remote-sync.task/update-progress! task-id fraction))))}))
+
(defn load-snapshot!
"Loads a snapshot's serialized entities into the app DB and reconciles local state to match it:
runs `load-metabase!`, toggles the `remote-sync-transforms` setting based on the snapshot's contents,
@@ -335,20 +352,24 @@
reconcile. That keeps the version pointer, RemoteSyncObject statuses, and the reconcile atomic: either
they all commit or all roll back, so a crash can never leave the version advanced past stale local
state or drop captured dirty markers (see [[import-merged!]])."
- [snapshot task-id sync-timestamp & {:keys [finalize!]}]
+ [snapshot report sync-timestamp & {:keys [finalize!]}]
+ (report 0.05 {:force? true})
(let [path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)
base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
has-transforms? (snapshot-has-transforms? base-ingestable)
- ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)
+ ingestable-snapshot (source.ingestable/wrap-progress-ingestable report [0.05 0.7] base-ingestable)
load-result (serdes/with-cache
(serialization/load-metabase! ingestable-snapshot :reindex? false))
seen-paths (:seen load-result)
imported-data (spec/extract-imported-entities seen-paths)]
- (remote-sync.task/update-progress! task-id 0.8)
+ (report 0.7 {:force? true})
(when (and has-transforms?
(not (settings/remote-sync-transforms)))
(log/info "Detected transforms in remote source, enabling remote-sync-transforms setting")
(settings/remote-sync-transforms! true))
+ ;; Reported before the transaction, not inside it: a write inside would hold the task row's lock until
+ ;; commit, blocking the heartbeat for the whole reconcile and hashing phase.
+ (report 0.75 {:force? true})
(t2/with-transaction [_conn]
(remove-unsynced! (spec/all-syncable-collection-ids) imported-data)
;; Replace the RemoteSyncObject table, folding each entity's repo file_path (so later renames/deletes
@@ -358,6 +379,7 @@
(insert-with-metadata! (spec/sync-all-entities! sync-timestamp imported-data)
(source.ingestable/cached-file-paths base-ingestable))
(when finalize! (finalize!)))
+ (report 0.9 {:force? true})
(when (and (not has-transforms?)
(settings/remote-sync-transforms))
(log/info "No transforms in remote source, disabling remote-sync-transforms setting")
@@ -368,7 +390,7 @@
(search/reindex! :async? (not= :h2 (mdb/db-type)))
(catch Exception e
(log/warn e "Search reindex after import failed")))
- (remote-sync.task/update-progress! task-id 0.95)
+ (report 0.95 {:force? true})
imported-data))
;;; ------------------------------------------- Incremental Import Fast-Path -------------------------------------------
@@ -451,11 +473,12 @@
Renames are handled by entity identity, not path: a rename re-loads the same entity_id at the new path
(an add), so the old path's delete is recognized as a rename and the entity is not removed."
- [{:keys [ingestable deleted-rsos] :as _plan} snapshot-version task-id sync-timestamp & {:keys [finalize!]}]
+ [{:keys [ingestable deleted-rsos] :as _plan} snapshot-version report sync-timestamp & {:keys [finalize!]}]
+ (report 0.05 {:force? true})
(let [load-result (when ingestable
(serdes/with-cache
(serialization/load-metabase!
- (source.ingestable/wrap-progress-ingestable task-id 0.7 ingestable)
+ (source.ingestable/wrap-progress-ingestable report [0.05 0.7] ingestable)
:reindex? false)))
imported-data (spec/extract-imported-entities (:seen load-result))
loaded-eid? (fn [model-type eid]
@@ -472,7 +495,9 @@
{:model_type model_type :model_id model_id})
model-key-of (fn [{:keys [model_type]}] (:model-key (spec/spec-for-model-type model_type)))
sync-rows (spec/sync-all-entities! sync-timestamp imported-data)]
- (remote-sync.task/update-progress! task-id 0.8)
+ (report 0.7 {:force? true})
+ ;; Before the transaction for the same reason as in [[load-snapshot!]].
+ (report 0.75 {:force? true})
(t2/with-transaction [_conn]
(doseq [[model-key ds] (group-by model-key-of deletes)]
(remote-sync.db/delete-instances! model-key (mapv :model_id ds)))
@@ -483,12 +508,13 @@
(remote-sync.db/delete-rsos-of-keys! sync-rows)
(insert-with-metadata! sync-rows (when ingestable (source.ingestable/cached-file-paths ingestable))))
(when finalize! (finalize!)))
+ (report 0.9 {:force? true})
;; We skip the whole-appdb reindex the full load runs. Added/modified entities are already
;; re-indexed by the load itself — serdes' t2 insert!/update! fire the :hook/search-index
;; after-insert/after-update hooks. Deletes have no such hook, so remove them explicitly.
(doseq [[model-key ds] (group-by model-key-of deletes)]
(search/delete! model-key (mapv :model_id ds)))
- (remote-sync.task/update-progress! task-id 0.95)
+ (report 0.95 {:force? true})
(log/info "Successfully reloaded entities from git repository")
{:status :success
:version snapshot-version
@@ -521,7 +547,7 @@
- Marks remote changes synced
- Local changes stay dirty
- Sets version to remote tip"
- [snapshot base-snapshot task-id sync-timestamp]
+ [snapshot base-snapshot task-id report sync-timestamp finalize!]
(let [{:keys [conflicts merged summary]} (serdes/with-cache
(let [targets (spec/exportable-entities)]
(source/compute-merge (spec/extract-entities-for-export targets)
@@ -536,12 +562,12 @@
:message "Import blocked: the same content was changed both locally and on the remote branch."})
;; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are disjoint
;; from the remote changes, so restoring them reproduces exactly the local diff vs remote. Restore +
- ;; set-version run inside the load's transaction so a crash can't leave the dirty markers overwritten.
+ ;; finalize! run inside the load's transaction so a crash can't leave the dirty markers overwritten.
(let [dirty-objects (capture-dirty-objects)]
- (load-snapshot! (source/specs->snapshot merged) task-id sync-timestamp
+ (load-snapshot! (source/specs->snapshot merged) report sync-timestamp
:finalize! (fn []
(restore-dirty-objects! dirty-objects sync-timestamp)
- (remote-sync.task/set-version! task-id (source.p/version snapshot))))
+ (finalize!)))
(log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"
(apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)
(count dirty-objects))
@@ -577,15 +603,23 @@
first-import? (nil? last-imported-version)
;; force-deletion? defaults to force? when a caller doesn't pass it.
force-deletion? (if (nil? force-deletion?) force? force-deletion?)
- finalize! #(remote-sync.task/set-version! task-id snapshot-version)
+ da-result (volatile! nil)
+ ;; Data apps before the version: the version write locks the task row until commit, which would
+ ;; block the heartbeat for the whole materialization.
+ finalize! (fn []
+ (vreset! da-result (materialize-data-apps! snapshot))
+ (remote-sync.task/set-version! task-id snapshot-version))
+ report (import-progress-reporter task-id)
path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)
;; First-import conflicts only block the first import; deletion conflicts block every import (an
;; already-configured instance must not silently delete unsynced transforms). The get-conflicts tree scan
;; is itself deferred so it only runs when one of the gates is open (e.g. a forced import with
;; force-deletion? true skips it entirely).
blocking-conflicts (delay
- (let [conflicts (delay (get-conflicts (source.p/->ingestable snapshot {:path-filters path-filters})
- first-import?))]
+ (let [conflicts (delay
+ (report 0.02 {:force? true})
+ (get-conflicts (source.p/->ingestable snapshot {:path-filters path-filters})
+ first-import?))]
(cond-> []
(and first-import? (not force?))
(into (:first-import-conflicts @conflicts))
@@ -616,7 +650,7 @@
:outcome {:kind "pull-skipped"}})
:else
- (import-merged! snapshot base-snapshot task-id sync-timestamp))
+ (import-merged! snapshot base-snapshot task-id report sync-timestamp finalize!))
;; --- Forced reload: bypasses the no-op/incremental guards. Deletion conflicts (when
;; force-deletion? is false) still block; otherwise a full reload. ---
@@ -633,7 +667,7 @@
:else
(let [_ (log/info "Remote sync full import: forced")
- imported-data (load-snapshot! snapshot task-id sync-timestamp :finalize! finalize!)]
+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)]
(log/info "Successfully reloaded entities from git repository")
{:status :success
:version snapshot-version
@@ -654,7 +688,7 @@
;; conflict. It touches only changed files, so it can't wholesale-delete unsynced transforms.
(and (not @dirty?)
(not= :remote-sync/incremental-not-possible @incremental-plan))
- (incremental-load-snapshot! @incremental-plan snapshot-version task-id sync-timestamp :finalize! finalize!)
+ (incremental-load-snapshot! @incremental-plan snapshot-version report sync-timestamp :finalize! finalize!)
(seq @blocking-conflicts)
(let [message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)]
@@ -671,24 +705,22 @@
first-import? "first import"
:else "changes not incrementally loadable")
_ (log/infof "Remote sync full import: %s" reason)
- imported-data (load-snapshot! snapshot task-id sync-timestamp :finalize! finalize!)]
+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)]
(log/info "Successfully reloaded entities from git repository")
{:status :success
:version snapshot-version
:outcome {:kind "pulled"
:count (pulled-change-count imported-data)
:branch (settings/remote-sync-branch)}}))]
- ;; Data apps ride the pull: re-materialize from the real source snapshot
- ;; (the repo file tree under `data_apps/`), not the synthetic merged
- ;; snapshot `load-snapshot!` sees. They're counted outside serdes, so fold
- ;; how many they upserted or removed into the outcome — otherwise a
- ;; data-app-only pull would report `pull-skipped` / `count 0`.
+ ;; Data apps rode the pull inside `finalize!`, materialized from the real source snapshot (the repo
+ ;; file tree under `data_apps/`), not the synthetic merged snapshot `load-snapshot!` sees. They're
+ ;; counted outside serdes, so fold how many they upserted or removed into the outcome — otherwise a
+ ;; data-app-only pull would report `pull-skipped` / `count 0`. Paths that skipped `finalize!` left
+ ;; no result: the version did not move, so neither did the apps.
(if (= :success (:status result))
- (let [da-result (materialize-data-apps! snapshot)
- ;; Removals count too: a pull whose only change is deleting an app
- ;; directory upserts nothing (`:changed` 0) but still changed what
- ;; this instance serves, so it must report as a pull, not skipped.
- da-changed (+ (:changed da-result 0) (:removed da-result 0))]
+ ;; Removals count too: a pull whose only change is deleting an app directory upserts nothing
+ ;; (`:changed` 0) but still changed what this instance serves, so it must report as a pull, not skipped.
+ (let [da-changed (+ (:changed @da-result 0) (:removed @da-result 0))]
(cond-> result
(pos? da-changed) (update :outcome fold-data-app-changes da-changed)))
result))
@@ -732,6 +764,9 @@
(source.p/abort-commit! commit)
(throw e))))))
+(def ^:private export-progress-plan-done 0.33) ; phase 1 (plan) complete / serialize start
+(def ^:private export-progress-serialize 0.66) ; phase 2 (serialize) complete
+
(defn- export-merged!
"Export when the remote branch has advanced beyond the last synced version. Runs an entity-identity 3-way merge of
local state (`models`) against the remote tip:
@@ -767,8 +802,14 @@
;; the version at the old base and a retry re-merges — the push is idempotent — rather than advancing
;; the pointer past un-reconciled local state).
(if-let [merged-snapshot (source.p/snapshot-at source version)]
- (let [pulled (apply + (vals summary))]
- (load-snapshot! merged-snapshot task-id sync-timestamp
+ (let [pulled (apply + (vals summary))
+ report (import-progress-reporter task-id)]
+ ;; The fold-in load reports on the import scale; map it onto the tail of the export bar so the
+ ;; bar never moves backward after the push checkpoints.
+ (load-snapshot! merged-snapshot
+ (fn [fraction & opts]
+ (apply report (+ export-progress-serialize (* fraction (- 1.0 export-progress-serialize))) opts))
+ sync-timestamp
:finalize! (fn []
(remote-sync.db/mark-all-rsos-synced! sync-timestamp)
(remote-sync.task/set-version! task-id version)))
@@ -1092,9 +1133,6 @@
(doseq [id-chunk (partition-all app-db-batch-size ids)]
(remote-sync.db/mark-rsos-synced! id-chunk (select-keys by-id id-chunk) sync-timestamp))))
-(def ^:private export-progress-plan-done 0.33) ; phase 1 (plan) complete / serialize start
-(def ^:private export-progress-serialize 0.66) ; phase 2 (serialize) complete
-
(defn- full-export!
"Re-serialize and commit the entire remote-synced set, then reconcile every RemoteSyncObject.
@@ -1424,13 +1462,78 @@
:details details
:user-id user-id})))
+(defonce ^:private running-tasks
+ (atom #{}))
+
+(defn running-task-ids
+ "The IDs of the RemoteSyncTasks whose worker is running in this JVM. Multi-node safe by construction: another
+ node's tasks are never in here."
+ []
+ @running-tasks)
+
+(defn fail-running-tasks!
+ "Fail every RemoteSyncTask in [[running-task-ids]] with `message`. For graceful shutdown: the workers die with
+ the process and their rows would otherwise stay open until reaped as stale. Goes through `handle-task-result!`
+ so a result that already landed, or a cancel, wins. Never throws."
+ [message]
+ (doseq [task-id (running-task-ids)]
+ (try
+ (log/infof "Failing remote sync task %d: %s" task-id message)
+ (handle-task-result! {:status :error :message message} task-id)
+ (catch Throwable t
+ (log/errorf t "Failed to close remote sync task %d" task-id)))))
+
+(defn- ensure-task-ended!
+ "Close the RemoteSyncTask `task-id` as failed if it is still open after its worker exited. Goes through
+ `handle-task-result!` so a concurrent cancel, or a result that did land, wins. Never throws: the reason the row
+ is still open may be that the app DB is unreachable."
+ [task-id]
+ (try
+ (when (some-> (remote-sync.db/task task-id) remote-sync.task/running?)
+ (log/warnf "Remote sync task %d ended without recording a result; failing it" task-id)
+ (handle-task-result! {:status :error :message "Task ended without recording a result"} task-id))
+ (catch Throwable t
+ (log/errorf t "Failed to close remote sync task %d" task-id))))
+
+(defn run-task-body!
+ "Run `sync-fn` (a fn of task-id returning a result map) for the already-created RemoteSyncTask `task-id` on the
+ current thread, recording the outcome on the row. `branch` is written to the remote-sync-branch setting on
+ success when non-nil; `:on-success` receives [task-id result] after a successful result is recorded.
+
+ Guarantees, whatever `sync-fn` or the bookkeeping does: a heartbeat runs on the row for the duration, the task
+ is registered in [[running-task-ids]] for the duration, and the row is ended on exit. Any `Throwable` from
+ `sync-fn` becomes an `:error` result; an `Error` must not escape the worker thread, where nothing would log it
+ and the row would stay open."
+ [task-id branch sync-fn & {:keys [on-success]}]
+ (let [stop-heartbeat! (remote-sync.task/start-heartbeat! task-id)]
+ (swap! running-tasks conj task-id)
+ (try
+ (let [result (try
+ (sync-fn task-id)
+ (catch Throwable t
+ (log/error t "Remote sync task failed")
+ {:status :error
+ :message (source-error-message t)}))]
+ (handle-task-result! result task-id branch)
+ (when (and on-success (= :success (:status result)))
+ (try
+ (on-success task-id result)
+ (catch Exception e
+ (log/errorf "Remote sync task :on-success function failed: %s" (ex-message e))))))
+ (catch Throwable t
+ (log/errorf t "Remote sync task %d bookkeeping failed" task-id))
+ (finally
+ (stop-heartbeat!)
+ (swap! running-tasks disj task-id)
+ (ensure-task-ended! task-id)))))
+
(defn- run-async!
"Executes a remote sync task asynchronously in a virtual thread.
Takes a task-type string ('import' or 'export'), a branch name to update in settings upon completion, a
sync-fn function that takes a task-id and performs the sync operation, and an optional :on-success callback
that receives [task-id result] after a successful sync. Creates a new task (or errors if one is already
- running), then executes the sync function in a virtual thread with a timeout.
+ running), then runs [[run-task-body!]] in a virtual thread with a timeout.
Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 if a sync task is already in progress."
[task-type branch sync-fn & {:keys [on-success]}]
@@ -1439,18 +1542,7 @@
(u.jvm/in-virtual-thread*
(dh/with-timeout {:interrupt? true
:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}
- (let [result (try
- (sync-fn task-id)
- (catch Exception e
- (log/errorf "Remote sync task failed: %s" (ex-message e))
- {:status :error
- :message (source-error-message e)}))]
- (handle-task-result! result task-id branch)
- (when (and on-success (= :success (:status result)))
- (try
- (on-success task-id result)
- (catch Exception e
- (log/errorf "Remote sync task :on-success function failed: %s" (ex-message e))))))))
+ (run-task-body! task-id branch sync-fn :on-success on-success)))
task))
(defn async-import!
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj
index be26ebd3f12..e49ce66895a 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj
@@ -5,6 +5,7 @@
[metabase-enterprise.remote-sync.events :as events]
[metabase-enterprise.remote-sync.impl :as impl]
[metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]
+ [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.settings :as settings]
[metabase-enterprise.remote-sync.task.import]
[metabase-enterprise.remote-sync.task.table-cleanup]
@@ -39,6 +40,8 @@
(save [] nil))))))
(defn- remote-sync-init []
+ ;; Only stale rows: a fresh open row may belong to a worker on another node sharing this app DB.
+ (remote-sync.task/supersede-stale-tasks!)
(if (settings/remote-sync-enabled)
(do
(when (= :read-only (settings/remote-sync-type))
@@ -77,3 +80,7 @@
(defmethod startup/def-startup-logic! ::remote-sync-setup
[_]
(remote-sync-init))
+
+(defmethod startup/def-shutdown-logic! ::remote-sync-shutdown
+ [_]
+ (impl/fail-running-tasks! "Interrupted by server shutdown"))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
index 70f8e5dab7c..d8093e3af71 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj
@@ -6,6 +6,8 @@
[metabase-enterprise.remote-sync.schema :as remote-sync.schema]
[metabase.models.interface :as mi]
[metabase.settings.core :as setting]
+ [metabase.util.jvm :as u.jvm]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[methodical.core :as methodical]
@@ -114,6 +116,31 @@
(vreset! last-fraction f)
(write-fn f))))))))
+(def default-heartbeat-interval-ms
+ "Ms between heartbeat writes while a task runs. Deliberately not derived from `remote-sync-task-time-limit-ms`:
+ an operator can set that below this interval, and a test would rather fail loudly than beat silently too slowly."
+ 15000)
+
+(defn start-heartbeat!
+ "Start a virtual thread stamping `last_heartbeat_at` on the task with `task-id` every `interval-ms` (default
+ [[default-heartbeat-interval-ms]]) for as long as it runs. Returns a zero-arg fn that stops it.
+
+ A failed write is logged at debug and the loop continues: a DB hiccup must not kill the heartbeat, and a
+ connection wait only makes the beat late. The update count is ignored because under a test transaction the
+ row is invisible to the heartbeat's own connection, and because an ended row is meant to update nothing."
+ ([task-id] (start-heartbeat! task-id default-heartbeat-interval-ms))
+ ([task-id interval-ms]
+ (let [stop (promise)]
+ (u.jvm/in-virtual-thread*
+ (loop []
+ (when (= ::tick (deref stop interval-ms ::tick))
+ (try
+ (remote-sync.db/touch-task! task-id)
+ (catch Throwable t
+ (log/debugf t "Heartbeat write failed for remote sync task %d" task-id)))
+ (recur))))
+ (fn stop-heartbeat! [] (deliver stop ::stop) nil))))
+
(defn set-version!
"Sets the version value for a sync task.
@@ -147,32 +174,44 @@
(remote-sync.db/end-task! task-id
{:error_message error-msg}))
+(defn- liveness-cutoff
+ "The instant before which a running task's last sign of life makes it stale."
+ []
+ (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms))))
+
(defn current-task
"Gets the current active sync task.
- Returns the most recent RemoteSyncTask that is still running (started but not ended, and has reported progress
- within the time limit), or nil if no active task exists."
+ Returns the most recent RemoteSyncTask that is still running (started but not ended, and whose owning thread
+ heartbeat or reported progress within the time limit), or nil if no active task exists."
[]
- (remote-sync.db/current-task (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms)))))
+ (remote-sync.db/current-task (liveness-cutoff)))
(defn supersede-stale-tasks!
- "Marks any genuinely stale task rows as cancelled and terminated.
+ "Marks any genuinely stale task rows as cancelled and terminated, and logs a warning naming each row it ended.
+ Returns the ids of those rows, empty when none were stale.
- A task is considered stale if it has `started_at` set, `ended_at` nil, and `last_progress_report_at`
- is older than `remote-sync-task-time-limit-ms`. The DB schema requires `last_progress_report_at`
- to be non-null with a default of `current_timestamp`, so a brand-new task always has a recent
- value (set on insert) and is not considered stale.
+ A task is considered stale if it has `started_at` set, `ended_at` nil, and its last sign of life
+ (`last_heartbeat_at`, or `last_progress_report_at` when no beat was ever written) is older than
+ `remote-sync-task-time-limit-ms`. The DB schema requires `last_progress_report_at` to be non-null with
+ a default of `current_timestamp`, so a brand-new task always has a recent value (set on insert) and is
+ not considered stale.
- Called from `create-task-with-lock!` before creating a new task, to clean up rows whose owning
- JVM/thread is gone or hung. Returns nothing meaningful.
+ The error message written names the staleness window, so the UI can show it as the reason the sync stopped.
Combined with `handle-task-result!`'s already-terminated check, this means a stale task's thread
that eventually wakes up and tries to complete will detect that its row is terminated and exit
without writing the setting or overwriting bookkeeping."
[]
- (let [cutoff (t/minus (t/offset-date-time)
- (t/millis (setting/get :remote-sync-task-time-limit-ms)))]
- (remote-sync.db/supersede-stale-tasks! cutoff)))
+ (let [cutoff (liveness-cutoff)
+ minutes (max 1 (quot (setting/get :remote-sync-task-time-limit-ms) 60000))
+ ids (remote-sync.db/supersede-stale-tasks!
+ cutoff
+ (format "Sync was interrupted: the server stopped responding for %d minute%s (it may have restarted)"
+ minutes (if (= 1 minutes) "" "s")))]
+ (when (seq ids)
+ (log/warnf "Superseded stale remote sync tasks %s: their owners were last alive before %s" ids cutoff))
+ ids))
(defn most-recent-task
"Gets the most recently run task, including currently running tasks.
@@ -184,10 +223,10 @@
(defn last-version
"Gets the version that any changes are built off of.
- Returns the version string from the most recent successful task (either export or import), or nil if no successful
- tasks exist."
+ Returns the version string of the newest task (import or export) whose commit landed, regardless of how the task
+ row was closed afterwards (see [[remote-sync.db/last-synced-task]]), or nil if no task has ever synced."
[]
- (:version (remote-sync.db/last-successful-task)))
+ (:version (remote-sync.db/last-synced-task)))
(defn running?
"Checks if a task is currently running.
@@ -234,11 +273,12 @@
Takes a RemoteSyncTask instance.
- Returns true if the task is incomplete and has not reported progress within the time limit, false otherwise."
+ Returns true if the task is incomplete and its owning thread has neither heartbeat nor reported progress within
+ the time limit, false otherwise. A row with no heartbeat falls back to its progress stamp."
[task]
(and (nil? (:ended_at task))
- (t/< (:last_progress_report_at task)
- (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms))))))
+ (t/< (or (:last_heartbeat_at task) (:last_progress_report_at task))
+ (liveness-cutoff))))
(defn conflict?
"Checks if a task ended with conflicts.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
index 292d5de062d..d9c73e77200 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
@@ -21,10 +21,16 @@
[:id pos-int?]
[:sync_task_type TaskType]
[:initiated_by {:optional true} [:maybe pos-int?]]
+ [:initiated_by_user {:optional true} [:maybe [:map
+ [:id pos-int?]
+ [:first_name [:maybe :string]]
+ [:last_name [:maybe :string]]
+ [:email :string]]]]
[:progress [:maybe [:float {:min 0.0 :max 1.0}]]]
[:started_at :any]
[:ended_at {:optional true} [:maybe :any]]
[:last_progress_report_at {:optional true} [:maybe :any]]
+ [:last_heartbeat_at {:optional true} [:maybe :any]]
[:version {:optional true} [:maybe :string]]
[:cancelled {:optional true} [:maybe :boolean]]
[:error_message {:optional true} [:maybe :string]]
@@ -228,6 +234,7 @@
[:started_at {:optional true} [:maybe ms/TemporalInstant]]
[:ended_at {:optional true} [:maybe ms/TemporalInstant]]
[:last_progress_report_at {:optional true} [:maybe ms/TemporalInstant]]
+ [:last_heartbeat_at {:optional true} [:maybe ms/TemporalInstant]]
[:initiated_by {:optional true} [:maybe ::lib.schema.id/user]]
[:error_message {:optional true} [:maybe :string]]
[:version {:optional true} [:maybe :string]]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index b7d9272009b..a50e9d8fe56 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -83,7 +83,7 @@
:default 5)
(defsetting remote-sync-task-time-limit-ms
- (deferred-tru "The maximum amount of time a remote sync task will be given to complete")
+ (deferred-tru "How long a remote sync task may go without proving its process is alive (via heartbeat, or progress on rows without one) before it is treated as dead and superseded. A slow but live task is never affected. The task itself is aborted after ten times this limit.")
:type :integer
:visibility :authenticated
:export? false
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
index 30c8176c71d..c643071bf5c 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
@@ -1,15 +1,12 @@
(ns metabase-enterprise.remote-sync.source.ingestable
(:require
[clojure.string :as str]
- [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.serialization.core :as serialization]
- [metabase.app-db.core :as app-db]
[metabase.models.serialization :as serdes]
[metabase.util :as u]
[metabase.util.log :as log]
- [metabase.util.yaml :as yaml]
- [toucan2.core :as t2])
+ [metabase.util.yaml :as yaml])
(:import
(metabase_enterprise.remote_sync.source.protocol SourceSnapshot)
(org.yaml.snakeyaml.error MarkedYAMLException)))
@@ -78,35 +75,23 @@
(serialization/ingest-errors ingestable)))
(defn wrap-progress-ingestable
- "Wraps an Ingestable to track and update progress during ingestion.
+ "Wraps `ingestable` so that ingesting the n-th of its N entities reports the fraction `lo` + n/N * (`hi` - `lo`)
+ through `report`, a fn of a fraction such as one from `make-progress-reporter`.
- Takes a task-id (the integer ID of the RemoteSyncTask model to update with progress), a normalize value (the
- maximum progress ratio value, with progress calculated as a fraction of this number), and an ingestable (an
- Ingestable object to wrap with progress tracking).
-
- Returns a CallbackIngestable instance that updates task progress as items are ingested."
- [task-id normalize ingestable]
- (let [total (count (serialization/ingest-list ingestable))
+ A failed report is logged and ignored so it can never abort the load it tracks; a cancellation raised by the
+ report (see `update-progress!`) propagates and stops the load."
+ [report [lo hi] ingestable]
+ (let [total (max 1 (count (serialization/ingest-list ingestable)))
calls (atom 0)]
(letfn [(progress-callback [item _]
(when item
- (let [progress (* (/ (swap! calls inc) total) normalize)]
- ;; Progress reporting must never abort the ingestion it tracks. The update runs on a
- ;; separate connection, which on some app DBs can contend with the in-flight load (e.g.
- ;; a MySQL lock-wait timeout), so swallow DB failures and keep going — but still honor a
- ;; cancellation signal, which `update-progress!` raises to stop the task.
- (try
- (if (app-db/in-transaction?)
- ;; The separate connection cannot see the task row if the ambient transaction created it, so it
- ;; would block on that row until innodb_lock_wait_timeout before failing. Nobody outside the
- ;; transaction can observe progress until it commits anyway, so stay on the current connection.
- (remote-sync.task/update-progress! task-id progress)
- (t2/with-connection [_conn (app-db/app-db)]
- (remote-sync.task/update-progress! task-id progress)))
- (catch Exception e
- (if (:cancelled? (ex-data e))
- (throw e)
- (log/warn (u/strip-error e "Failed to report import progress; continuing"))))))))]
+ (try
+ ;; counted down from hi so the last entity lands on hi exactly, not a rounding neighbour
+ (report (- hi (* (- hi lo) (/ (- total (swap! calls inc)) total))))
+ (catch Exception e
+ (if (:cancelled? (ex-data e))
+ (throw e)
+ (log/warn (u/strip-error e "Failed to report import progress; continuing")))))))]
(->CallbackIngestable ingestable progress-callback))))
;; Wraps another Ingestable and filters the `list-files` content to only content that has the specified
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj
index f51c7abc09e..83c9d2dfb00 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/task/import.clj
@@ -36,16 +36,11 @@
(dh/with-timeout {:interrupt? true
:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}
(log/info "Auto-importing remote-sync collections")
- (let [result (impl/import! snapshot task-id)]
- (impl/handle-task-result! result task-id)
- (when (= :success (:status result))
- ;; events/publish-event! rethrows handler exceptions; don't let an audit-log
- ;; failure mark an already-successful import as failed
- (try
- (impl/publish-sync-event! :event/remote-sync-import task-id
- {:branch branch :auto true} nil)
- (catch Exception e
- (log/errorf "Failed to publish remote-sync audit event: %s" (ex-message e))))))))))))))
+ (impl/run-task-body! task-id nil
+ (fn [task-id] (impl/import! snapshot task-id))
+ :on-success (fn [task-id _result]
+ (impl/publish-sync-event! :event/remote-sync-import task-id
+ {:branch branch :auto true} nil)))))))))))
(task/defjob ^{:doc "Auto-imports any remote collections."} AutoImport [_]
(auto-import!))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index bea13912a5e..01cd7bbf2d0 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -755,8 +755,67 @@
:error_message "Task cancelled"}
(mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task"))))))
+(deftest current-task-closes-a-stale-open-task-test
+ (testing "GET /api/ee/remote-sync/current-task closes an open task whose owner is gone and returns it cancelled"
+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]
+ (mt/with-temp [:model/RemoteSyncTask {id :id} {:sync_task_type "import"
+ :started_at old-time
+ :last_progress_report_at old-time}]
+ (let [first-response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task")]
+ (is (=? {:id id
+ :status "cancelled"
+ :cancelled true
+ :ended_at some?
+ :error_message #"^Sync was interrupted.*"}
+ first-response))
+ (testing "a second GET returns the same closed row"
+ (is (=? (select-keys first-response [:id :status :cancelled :ended_at :error_message])
+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task")))))))))
+
+(deftest current-task-leaves-a-live-task-running-test
+ (testing "GET /api/ee/remote-sync/current-task returns a task with a fresh heartbeat as running even when its progress is stale"
+ (mt/with-temp [:model/RemoteSyncTask {id :id} {:sync_task_type "import"
+ :started_at (t/minus (t/offset-date-time) (t/hours 1))
+ :last_progress_report_at (t/minus (t/offset-date-time) (t/hours 1))
+ :last_heartbeat_at :%now}]
+ (is (=? {:id id :status "running" :ended_at nil}
+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task"))))))
+
+(deftest current-task-carries-the-trimmed-initiating-user-test
+ (testing "GET /api/ee/remote-sync/current-task carries the initiating user as id, first_name, last_name, email only"
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"
+ :initiated_by (mt/user->id :rasta)
+ :started_at :%now
+ :last_progress_report_at :%now}]
+ (let [user (:initiated_by_user (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task"))]
+ (is (= {:id (mt/user->id :rasta)
+ :first_name "Rasta"
+ :last_name "Toucan"
+ :email "[email protected]"}
+ user))))))
+
+(deftest current-task-initiating-user-is-nil-for-system-tasks-test
+ (testing "GET /api/ee/remote-sync/current-task carries a nil initiating user for a task with no initiator (auto-import)"
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"
+ :initiated_by nil
+ :started_at :%now
+ :last_progress_report_at :%now}]
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/current-task")]
+ (is (contains? response :initiated_by_user))
+ (is (nil? (:initiated_by_user response)))))))
+
;;; ------------------------------------------------- Cancel Task Endpoint -------------------------------------------------
+(deftest cancel-task-carries-the-trimmed-initiating-user-test
+ (testing "POST /api/ee/remote-sync/current-task/cancel returns the cancelled task with its trimmed initiating user"
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "export"
+ :initiated_by (mt/user->id :rasta)
+ :started_at :%now
+ :last_progress_report_at :%now}]
+ (is (=? {:status "cancelled"
+ :initiated_by_user {:id (mt/user->id :rasta) :email "[email protected]"}}
+ (mt/user-http-request :crowberto :post 200 "ee/remote-sync/current-task/cancel"))))))
+
(deftest cancel-task-requires-superuser-test
(testing "POST /api/ee/remote-sync/current-task/cancel requires superuser permissions (GHY-3804)"
(mt/with-temp [:model/RemoteSyncTask {id :id} {:sync_task_type "export"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj
index 5bfd4dba648..6f68dcc8017 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj
@@ -7,7 +7,9 @@
mixed (serdes + data apps), and serdes-only."
(:require
[clojure.test :refer :all]
+ [metabase-enterprise.data-apps.sync :as data-apps.sync]
[metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as rs.test]
[metabase.search.test-util :as search.tu]
@@ -86,3 +88,39 @@
(merge coll-file (app-tree! "ops" "BUNDLE-V2")))]
(is (= "pulled" (:kind outcome)))
(is (= 2 (:count outcome)) "the collection (1) plus the changed data app (1)"))))))))
+
+(deftest data-apps-land-with-the-version-test
+ (testing "a pull records the data apps at the snapshot version it commits as the sync base"
+ (search.tu/with-index-disabled
+ (mt/with-premium-features #{:data-apps-preview}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")
+ "v1" (app-tree! "sales" "BUNDLE-V2")}
+ :current "v0")]
+ (is (= :success (:status (import-at! src "v0" :force? true))))
+ (is (= :success (:status (import-at! src "v1"))))
+ (is (= "v1" (remote-sync.task/last-version)))
+ (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))
+
+(deftest data-app-failure-rolls-back-the-version-test
+ (testing "a crash while materializing data apps rolls the version back, so the next pull redoes the import
+ instead of skipping a version whose data apps never landed"
+ (search.tu/with-index-disabled
+ (mt/with-premium-features #{:data-apps-preview}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")
+ "v1" (app-tree! "sales" "BUNDLE-V2")}
+ :current "v0")]
+ (is (= :success (:status (import-at! src "v0" :force? true))))
+ (let [v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales")]
+ (mt/with-dynamic-fn-redefs [data-apps.sync/sync-from-snapshot! (fn [_] (throw (ex-info "materialization died" {})))]
+ (is (= :error (:status (import-at! src "v1")))))
+ (is (= "v0" (remote-sync.task/last-version)) "the failed pull is not the sync base")
+ (is (= v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales"))
+ "the v0 app is untouched"))
+ (is (=? {:status :success :outcome {:kind "pulled"}} (import-at! src "v1"))
+ "the next pull redoes the import rather than skipping v1")
+ (is (= "v1" (remote-sync.task/last-version)))
+ (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj
index f807462927d..08013308a9a 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_progress_test.clj
@@ -1,13 +1,90 @@
(ns metabase-enterprise.remote-sync.impl-progress-test
- "Serialize-phase progress hook for remote-sync export (GHY-4132)."
+ "Progress reporting through the export serialize phase (GHY-4132) and the import phases (GHY-4593)."
(:require
[clojure.test :refer :all]
[metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.models.remote-sync-task :as rst]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
- [metabase.test :as mt]))
+ [metabase-enterprise.remote-sync.test-helpers :as rs.test]
+ [metabase.search.test-util :as search.tu]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
+(use-fixtures :once (fixtures/initialize :db))
+(use-fixtures :each rs.test/clean-remote-sync-state)
+
+;;; ------------------------------------------------- Import phases --------------------------------------------------
+
+(def ^:private collection-path
+ "collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml")
+
+(def ^:private card-path
+ "collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/f1C68pznmrpN1F5xFDj6d_some_question.yaml")
+
+(defn- tree [card-name]
+ {collection-path (rs.test/generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")
+ card-path (rs.test/generate-card-yaml "f1C68pznmrpN1F5xFDj6d" card-name "M-Q4pcV0qkiyJ0kiSWECl")})
+
+(defn- import-capturing-progress!
+ "Runs `import!` of `snapshot` with the reporter replaced by one that records every fraction it would write
+ (`:throttle-ms` 0 unless `opts` say otherwise). Records the result on the task. Returns [result fractions]."
+ [snapshot & {:keys [force? throttle-ms now-fn] :or {throttle-ms 0 now-fn (constantly 0)}}]
+ ;; the card in [[tree]] references the test-data database, which must exist before the load
+ (mt/db)
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ writes (atom [])]
+ (mt/with-dynamic-fn-redefs [impl/import-progress-reporter
+ (fn [_task-id]
+ (rst/make-progress-reporter task-id {:throttle-ms throttle-ms
+ :now-fn now-fn
+ :write-fn (fn [f] (swap! writes conj f))}))]
+ (let [result (impl/import! snapshot task-id :force? force?)]
+ (impl/handle-task-result! result task-id)
+ [result @writes]))))
+
+(def ^:private checkpoints
+ "Forced writes of a full import, in order: conflict scan, load start, load done, reconcile start, commit, reindex."
+ [0.02 0.05 0.7 0.75 0.9 0.95])
+
+(defn- approx= [expected actual]
+ (and (= (count expected) (count actual))
+ (every? true? (map #(< (abs (- %1 %2)) 1e-9) expected actual))))
+
+(deftest full-import-progress-is-monotonic-and-checkpointed-test
+ (testing "a first import reports 0.02 before the conflict scan, one fraction per loaded entity inside [0.05 0.7], then the fixed checkpoints"
+ (search.tu/with-index-disabled
+ (let [src (rs.test/versioned-source :trees {"v0" (tree "Some Question")} :current "v0")
+ [result writes] (import-capturing-progress! (source.p/snapshot src))]
+ (is (= :success (:status result)))
+ (is (= writes (sort writes)) "never moves backward")
+ ;; two entities: the collection at the halfway point of the load range, the card on its top
+ (is (approx= [0.02 0.05 0.375 0.7 0.7 0.75 0.9 0.95] writes))))))
+
+(deftest per-entity-progress-writes-are-throttled-test
+ (testing "with the real throttle and a frozen clock, only the forced checkpoints are written"
+ (search.tu/with-index-disabled
+ (let [src (rs.test/versioned-source :trees {"v0" (tree "Some Question")} :current "v0")
+ [result writes] (import-capturing-progress! (source.p/snapshot src) :throttle-ms 10000)]
+ (is (= :success (:status result)))
+ (is (= checkpoints writes))))))
+
+(deftest incremental-import-progress-test
+ (testing "an incremental pull skips the conflict-scan checkpoint and reports the changed entity inside the load range"
+ (search.tu/with-index-disabled
+ (let [src (rs.test/versioned-source :trees {"v0" (tree "Some Question") "v1" (tree "Renamed Question")} :current "v0")]
+ (is (= :success (:status (first (import-capturing-progress! (source.p/snapshot-at src "v0") :force? true)))))
+ (let [[result writes] (import-capturing-progress! (source.p/snapshot-at src "v1"))]
+ (is (= :success (:status result)))
+ (is (= {:kind "pulled" :count 1} (dissoc (:outcome result) :branch))
+ "the incremental path ran: one changed entity pulled")
+ ;; no conflict scan; the single changed entity lands on the top of the load range
+ (is (= [0.05 0.7 0.7 0.75 0.9 0.95] writes)))))))
+
+;;; ------------------------------------------------- Export phases --------------------------------------------------
+
;; A CommitBuilder that records staged paths and does nothing else.
(defrecord RecordingCommit [staged]
source.p/CommitBuilder
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index e0bc364d41e..30b2e2b1d4d 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -463,24 +463,24 @@
(is (= :error (:status result)))
(is (re-find #"Network error" (:message result))))))
-(deftest import!-calls-update-progress-with-expected-values-test
- (testing "import! calls update-progress! with expected progress values"
+(deftest import!-reports-progress-checkpoints-test
+ (testing "a first full import writes only the forced checkpoints when the per-entity writes fall inside the throttle window"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]
- (let [mock-source (test-helpers/create-mock-source)
+ (let [mock-source (test-helpers/create-mock-source)
progress-calls (atom [])]
(mt/with-dynamic-fn-redefs [remote-sync.task/update-progress!
(fn [task-id progress]
- (swap! progress-calls conj {:task-id task-id :progress progress}))]
+ (swap! progress-calls conj {:task-id task-id :progress progress}))
+ ;; a frozen clock keeps every per-entity write inside the throttle window
+ remote-sync.task/make-progress-reporter
+ (let [real (mt/original-fn #'remote-sync.task/make-progress-reporter)]
+ (fn [task-id opts] (real task-id (assoc opts :now-fn (constantly 0)))))]
(let [result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- (is (= 5 (count @progress-calls)))
- (is (= task-id (:task-id (first @progress-calls))))
- (is (= task-id (:task-id (second @progress-calls))))
- (is (= task-id (:task-id (nth @progress-calls 2))))
- (is (= 0.7 (:progress (nth @progress-calls 2))))
- (is (= 0.8 (:progress (nth @progress-calls 3))))
- (is (= 0.95 (:progress (nth @progress-calls 4)))))))))))
+ (is (every? #(= task-id (:task-id %)) @progress-calls))
+ (is (= [0.02 0.05 0.7 0.75 0.9 0.95] (mapv :progress @progress-calls))
+ "conflict scan, load start, load done, reconcile start, commit, reindex"))))))))
(deftest import!-runs-a-single-reindex-inside-the-task-test
(testing "a full import runs exactly one reindex, synchronous on H2 and asynchronous elsewhere"
@@ -2320,3 +2320,87 @@ serdes/meta:
"F1's edited description is present")
(is (not (t2/exists? :model/FieldUserSettings :field_id f2-id))
"F2's stale local row, absent from the imported file's :fields, is gone")))))))
+
+;; ---------- run-task-body!: the row is always closed, whatever the worker does ----------------
+
+(defn- new-task-id []
+ (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)}))
+
+(deftest run-task-body!-records-a-successful-result-test
+ (testing "a :success result completes the row, runs :on-success, and unregisters the task"
+ (let [task-id (new-task-id)
+ on-success (atom nil)
+ seen-running (atom nil)]
+ (impl/run-task-body! task-id nil
+ (fn [id]
+ (reset! seen-running (contains? (impl/running-task-ids) id))
+ {:status :success :outcome {:kind "pull-skipped"}})
+ :on-success (fn [id result] (reset! on-success [id (:status result)])))
+ (is (true? @seen-running) "the task is registered while its body runs")
+ (is (not (contains? (impl/running-task-ids) task-id)))
+ (is (= [task-id :success] @on-success))
+ (is (=? {:ended_at some? :cancelled false :error_message nil :outcome {:kind "pull-skipped"}}
+ (t2/select-one :model/RemoteSyncTask :id task-id))))))
+
+(deftest run-task-body!-fails-the-row-when-sync-fn-throws-an-error-test
+ (testing "an Error (not an Exception) from sync-fn still ends the row with a message"
+ (let [task-id (new-task-id)]
+ (impl/run-task-body! task-id nil (fn [_] (throw (Error. "boom"))))
+ (is (=? {:ended_at some? :cancelled false :error_message #".*boom"}
+ (t2/select-one :model/RemoteSyncTask :id task-id)))
+ (is (not (contains? (impl/running-task-ids) task-id))))))
+
+(deftest run-task-body!-fails-the-row-when-the-throwable-has-no-message-test
+ (testing "a throwable with a nil message still produces a non-empty error_message"
+ (let [task-id (new-task-id)]
+ (impl/run-task-body! task-id nil (fn [_] (throw (StackOverflowError.))))
+ (is (=? {:ended_at some? :error_message #".*StackOverflowError"}
+ (t2/select-one :model/RemoteSyncTask :id task-id))))))
+
+(deftest run-task-body!-closes-the-row-when-bookkeeping-throws-test
+ (testing "when handle-task-result! itself throws, the row is still ended by the exit path"
+ (let [task-id (new-task-id)
+ calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [impl/handle-task-result! (let [orig (mt/original-fn #'impl/handle-task-result!)]
+ (fn [& args]
+ (if (= 1 (swap! calls inc))
+ (throw (ex-info "pool exhausted" {}))
+ (apply orig args))))]
+ (impl/run-task-body! task-id nil (fn [_] {:status :success})))
+ (is (= 2 @calls))
+ (is (=? {:ended_at some? :cancelled false :error_message "Task ended without recording a result"}
+ (t2/select-one :model/RemoteSyncTask :id task-id))))))
+
+(deftest run-task-body!-does-not-clobber-a-concurrent-cancel-test
+ (testing "a cancel that lands while sync-fn runs is preserved: neither the result nor the exit path overwrites it"
+ (let [task-id (new-task-id)]
+ (impl/run-task-body! task-id nil
+ (fn [id]
+ (remote-sync.task/cancel-sync-task! id)
+ {:status :success}))
+ (is (=? {:ended_at some? :cancelled true :error_message "Task cancelled"}
+ (t2/select-one :model/RemoteSyncTask :id task-id))))))
+
+(deftest run-task-body!-stops-the-heartbeat-on-exit-test
+ (testing "the heartbeat started for the task is stopped when the body exits, including on failure"
+ (let [task-id (new-task-id)
+ stopped? (atom false)]
+ (mt/with-dynamic-fn-redefs [remote-sync.task/start-heartbeat! (fn [_id] (fn [] (reset! stopped? true)))]
+ (impl/run-task-body! task-id nil (fn [_] (throw (Error. "boom")))))
+ (is (true? @stopped?)))))
+
+(deftest run-task-body!-on-success-failure-does-not-fail-the-row-test
+ (testing "an exception from :on-success is logged and the row stays successful"
+ (let [task-id (new-task-id)]
+ (impl/run-task-body! task-id nil (fn [_] {:status :success})
+ :on-success (fn [_ _] (throw (ex-info "audit log down" {}))))
+ (is (=? {:ended_at some? :cancelled false :error_message nil}
+ (t2/select-one :model/RemoteSyncTask :id task-id))))))
+
+(deftest run-task-body!-writes-the-branch-on-success-test
+ (testing "a non-nil branch is written to the setting on success and left alone on error"
+ (mt/with-temporary-setting-values [remote-sync-branch "main"]
+ (impl/run-task-body! (new-task-id) "feature" (fn [_] {:status :error :message "nope"}))
+ (is (= "main" (remote-sync.settings/remote-sync-branch)))
+ (impl/run-task-body! (new-task-id) "feature" (fn [_] {:status :success}))
+ (is (= "feature" (remote-sync.settings/remote-sync-branch))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
index 42ccd5985ac..1ef132c482e 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
@@ -261,6 +261,24 @@
;;; ------------------------------------------ First import: never incremental ------------------------------------------
+(deftest cancelled-after-commit-keeps-the-sync-base-test
+ (testing "a pull whose task is cancelled after its transaction committed still advances last-version, so the
+ next pull of the same snapshot is skipped instead of re-importing everything"
+ (do-with-bench!
+ (fn [f0]
+ (search.tu/with-index-disabled
+ (let [f1 (update f0 (path-with f0 "card_b") str/replace "display: table" "display: line")
+ src (rs.test/versioned-source :trees {"v0" f0 "v1" f1} :current "v0")]
+ (is (= :success (:status (import-at! src "v0" :force? true))))
+ (let [task (t2/insert-returning-pk! :model/RemoteSyncTask
+ {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
+ (is (= :success (:status (impl/import! (source.p/snapshot-at src "v1") task))))
+ ;; The worker died (or an admin cancelled) between the commit and the result bookkeeping.
+ (remote-sync.task/cancel-sync-task! task)
+ (is (=? {:cancelled true :version "v1"} (t2/select-one :model/RemoteSyncTask :id task))))
+ (is (= "v1" (remote-sync.task/last-version)))
+ (is (=? {:status :success :outcome {:kind "pull-skipped"}} (import-at! src "v1")))))))))
+
(deftest first-import-no-force-uses-full-load-test
(testing "GHY-3779: a first import (no prior version, so last-version is nil) with force? false must NOT
attempt the incremental fast-path. incremental-load-snapshot! assumes local state equals
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj
index e36e2699f02..b54f366192b 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj
@@ -6,11 +6,19 @@
[metabase-enterprise.remote-sync.impl :as impl]
[metabase-enterprise.remote-sync.init :as init]
[metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]
+ [metabase-enterprise.remote-sync.test-helpers :as th]
[metabase.collections.models.collection :as collection]
[metabase.collections.test-utils :as collections.tu]
+ [metabase.startup.core :as startup]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[toucan2.core :as t2]))
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db))
+(use-fixtures :each th/clean-remote-sync-state)
+
(defn- capture-async-import! []
(let [calls (atom [])]
[calls (fn [branch force? args & _opts] (swap! calls conj [branch force? args]) nil)]))
@@ -105,6 +113,55 @@
(#'init/remote-sync-init)
(is (empty? @calls))))))))
+(defn- wait-until
+ "True once `pred` returns truthy, polling every 10 ms for up to `timeout-ms`; false otherwise."
+ [pred timeout-ms]
+ (let [deadline (+ (System/currentTimeMillis) timeout-ms)]
+ (loop []
+ (cond
+ (pred) true
+ (> (System/currentTimeMillis) deadline) false
+ :else (do (Thread/sleep 10) (recur))))))
+
+(defn- new-task-id []
+ (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)}))
+
+(deftest remote-sync-shutdown-fails-this-jvms-running-tasks-test
+ (testing "shutdown fails every task whose worker runs in this JVM and leaves a stale open row from elsewhere untouched"
+ (let [other-id (new-task-id)
+ _ (t2/update! :model/RemoteSyncTask other-id
+ {:last_progress_report_at (t/minus (t/offset-date-time) (t/hours 2))})
+ own-id (new-task-id)
+ release (promise)
+ worker (future (impl/run-task-body! own-id nil
+ (fn [_] @release {:status :success :outcome {:kind "pull-skipped"}})))]
+ (try
+ (is (true? (wait-until #(contains? (impl/running-task-ids) own-id) 5000)))
+ (startup/def-shutdown-logic! ::init/remote-sync-shutdown)
+ (is (=? {:ended_at some? :cancelled false :error_message "Interrupted by server shutdown"}
+ (t2/select-one :model/RemoteSyncTask :id own-id)))
+ (is (=? {:ended_at nil :error_message nil}
+ (t2/select-one :model/RemoteSyncTask :id other-id)))
+ (finally
+ (deliver release nil)
+ (is (not= ::timeout (deref worker 10000 ::timeout)))))
+ (testing "the worker's late result does not overwrite the shutdown bookkeeping"
+ (is (=? {:error_message "Interrupted by server shutdown" :outcome nil}
+ (t2/select-one :model/RemoteSyncTask :id own-id)))))))
+
+(deftest remote-sync-init-supersedes-stale-tasks-test
+ (testing "boot closes open task rows whose owner has been silent past the window and leaves fresh ones alone"
+ (mt/with-temporary-setting-values [:remote-sync-url nil]
+ (let [stale-id (new-task-id)
+ _ (t2/update! :model/RemoteSyncTask stale-id
+ {:last_progress_report_at (t/minus (t/offset-date-time) (t/hours 2))})
+ fresh-id (new-task-id)]
+ (#'init/remote-sync-init)
+ (is (=? {:cancelled true :ended_at some? :error_message #"^Sync was interrupted.*"}
+ (t2/select-one :model/RemoteSyncTask :id stale-id)))
+ (is (=? {:cancelled false :ended_at nil :error_message nil}
+ (t2/select-one :model/RemoteSyncTask :id fresh-id)))))))
+
;;; ------------------------------------------- Glossary ledger backfill -------------------------------------------
(defn- glossary-rso-count []
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj
index cf4245c307e..2c9d21d6800 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj
@@ -3,6 +3,7 @@
(:require
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase-enterprise.remote-sync.db :as remote-sync.db]
[metabase-enterprise.remote-sync.models.remote-sync-task :as rst]
[metabase-enterprise.remote-sync.test-helpers :as th]
[metabase.models.interface :as mi]
@@ -365,6 +366,129 @@
(finally
(rst/complete-sync-task! (:id task)))))))
+(deftest timed-out?-uses-heartbeat-over-progress-test
+ (testing "timed-out? is false when the heartbeat is fresh even though progress is stale"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))
+ old-time (t/minus (t/offset-date-time) (t/hours 2))]
+ (try
+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time
+ :last_heartbeat_at (t/offset-date-time)})
+ (is (false? (rst/timed-out? (t2/select-one :model/RemoteSyncTask :id (:id task)))))
+ (finally
+ (rst/complete-sync-task! (:id task))))))
+ (testing "timed-out? is true when the heartbeat is stale even though progress is fresh"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))
+ old-time (t/minus (t/offset-date-time) (t/hours 2))]
+ (try
+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at (t/offset-date-time)
+ :last_heartbeat_at old-time})
+ (is (true? (rst/timed-out? (t2/select-one :model/RemoteSyncTask :id (:id task)))))
+ (finally
+ (rst/complete-sync-task! (:id task))))))
+ (testing "timed-out? falls back to progress when no heartbeat was ever written"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))
+ old-time (t/minus (t/offset-date-time) (t/hours 2))]
+ (try
+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time
+ :last_heartbeat_at nil})
+ (is (true? (rst/timed-out? (t2/select-one :model/RemoteSyncTask :id (:id task)))))
+ (finally
+ (rst/complete-sync-task! (:id task)))))))
+
+(deftest current-task-uses-heartbeat-over-progress-test
+ (testing "current-task returns a task whose heartbeat is fresh and progress stale"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))
+ old-time (t/minus (t/offset-date-time) (t/hours 2))]
+ (try
+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time
+ :last_heartbeat_at (t/offset-date-time)})
+ (is (= (:id task) (:id (rst/current-task))))
+ (finally
+ (rst/complete-sync-task! (:id task))))))
+ (testing "current-task ignores a task whose heartbeat is stale and progress fresh"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))
+ old-time (t/minus (t/offset-date-time) (t/hours 2))]
+ (try
+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at (t/offset-date-time)
+ :last_heartbeat_at old-time})
+ (is (nil? (rst/current-task)))
+ (finally
+ (rst/complete-sync-task! (:id task)))))))
+
+;;; ------------------------------------------------------------------------------------------------
+;;; Tests for touch-task! and start-heartbeat!
+;;; ------------------------------------------------------------------------------------------------
+
+(defn- wait-until
+ "True once `pred` returns truthy, polling every 10 ms for up to `timeout-ms`; false otherwise."
+ [pred timeout-ms]
+ (let [deadline (+ (System/currentTimeMillis) timeout-ms)]
+ (loop []
+ (cond
+ (pred) true
+ (> (System/currentTimeMillis) deadline) false
+ :else (do (Thread/sleep 10) (recur))))))
+
+(deftest touch-task!-stamps-heartbeat-only-test
+ (testing "touch-task! sets last_heartbeat_at and leaves last_progress_report_at alone on a running task"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))
+ old-time (t/minus (t/offset-date-time) (t/hours 2))]
+ (try
+ (t2/update! :model/RemoteSyncTask (:id task) {:last_progress_report_at old-time})
+ (let [before (t2/select-one :model/RemoteSyncTask :id (:id task))]
+ (is (nil? (:last_heartbeat_at before)))
+ (is (= 1 (remote-sync.db/touch-task! (:id task))))
+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id task))]
+ (is (some? (:last_heartbeat_at after)))
+ (is (= (:last_progress_report_at before) (:last_progress_report_at after)))))
+ (finally
+ (rst/complete-sync-task! (:id task)))))))
+
+(deftest touch-task!-ignores-ended-task-test
+ (testing "touch-task! updates nothing on a cancelled task, so a late beat cannot revive it"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))]
+ (rst/cancel-sync-task! (:id task))
+ (let [before (t2/select-one :model/RemoteSyncTask :id (:id task))]
+ (is (= 0 (remote-sync.db/touch-task! (:id task))))
+ (is (= before (t2/select-one :model/RemoteSyncTask :id (:id task))))))))
+
+(deftest start-heartbeat!-beats-until-stopped-test
+ (testing "the heartbeat writes on every interval and stops writing once the stop fn is called"
+ (let [writes (atom 0)]
+ (mt/with-dynamic-fn-redefs [remote-sync.db/touch-task! (fn [_task-id] (swap! writes inc) 1)]
+ (let [stop! (rst/start-heartbeat! 1 20)]
+ (try
+ (is (true? (wait-until #(>= @writes 3) 2000)))
+ (finally
+ (stop!)))
+ (let [after-stop @writes]
+ (Thread/sleep 100)
+ (is (= after-stop @writes))))))))
+
+(deftest start-heartbeat!-survives-write-failure-test
+ (testing "a failing write does not end the heartbeat loop"
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [remote-sync.db/touch-task! (fn [_task-id]
+ (when (= 1 (swap! calls inc))
+ (throw (ex-info "db down" {})))
+ 1)]
+ (let [stop! (rst/start-heartbeat! 1 20)]
+ (try
+ (is (true? (wait-until #(>= @calls 3) 2000)))
+ (finally
+ (stop!))))))))
+
+(deftest start-heartbeat!-writes-the-row-test
+ (testing "the heartbeat advances last_heartbeat_at on the real row"
+ (let [task (rst/create-sync-task! "import" (mt/user->id :rasta))
+ stop! (rst/start-heartbeat! (:id task) 20)]
+ (try
+ (is (true? (wait-until #(some? (t2/select-one-fn :last_heartbeat_at :model/RemoteSyncTask :id (:id task)))
+ 2000)))
+ (finally
+ (stop!)
+ (rst/complete-sync-task! (:id task)))))))
+
(deftest update-progress-throws-on-cancelled-task-test
(testing "update-progress! throws exception when updating a cancelled task"
(let [task (rst/create-sync-task! "import" (mt/user->id :rasta))]
@@ -375,35 +499,52 @@
(deftest last-version-test
(testing "When there are no tasks, last-version returns nil"
(is (nil? (rst/last-version))))
- (testing "When there are no successful tasks, last-version returns nil"
+ (testing "When no task has recorded a version, last-version returns nil"
(let [task (rst/create-sync-task! "import" (mt/user->id :rasta))]
(rst/fail-sync-task! (:id task) "Test failure")
(is (nil? (rst/last-version)))))
- (testing "Returns last successful version"
+ (testing "Returns the version of the newest task whose commit landed"
(let [successful-task (rst/create-sync-task! "import" (mt/user->id :rasta))]
(rst/complete-sync-task! (:id successful-task))
(rst/set-version! (:id successful-task) "version 1")
(is (= "version 1" (rst/last-version)))
- (testing "Ignores cancelled tasks"
+ (testing "Ignores a cancelled task with no version"
(rst/cancel-sync-task! (:id (rst/create-sync-task! "import" (mt/user->id :rasta))))
(is (= "version 1" (rst/last-version))))
- (testing "Ignores failed tasks"
+ (testing "Ignores a failed task with no version"
(rst/fail-sync-task! (:id (rst/create-sync-task! "import" (mt/user->id :rasta))) "Error")
(is (= "version 1" (rst/last-version))))
- (testing "Ignores incomplete tasks"
- (is (= "version 1" (rst/last-version))))
- (testing "Returns newer successful tasks"
+ (testing "Ignores an open task with no version"
+ (let [open-task (rst/create-sync-task! "import" (mt/user->id :rasta))]
+ (is (= "version 1" (rst/last-version)))
+ (rst/complete-sync-task! (:id open-task))))
+ (testing "Ignores a conflict task even though it records the version it conflicted against"
+ (let [conflict-task (rst/create-sync-task! "import" (mt/user->id :rasta))]
+ (rst/set-version! (:id conflict-task) "version 1.5")
+ (rst/conflict-sync-task! (:id conflict-task) ["some conflict"])
+ (is (= "version 1" (rst/last-version)))))
+ (testing "Returns a newer successful task's version"
(let [new-task (rst/create-sync-task! "import" (mt/user->id :rasta))]
(rst/complete-sync-task! (:id new-task))
(rst/set-version! (:id new-task) "version 2")
- (is (= "version 2" (rst/last-version))))))))
+ (is (= "version 2" (rst/last-version)))))
+ (testing "A task cancelled after its commit landed is still the sync base"
+ (let [cancelled-task (rst/create-sync-task! "import" (mt/user->id :rasta))]
+ (rst/set-version! (:id cancelled-task) "version 3")
+ (rst/cancel-sync-task! (:id cancelled-task))
+ (is (= "version 3" (rst/last-version)))))
+ (testing "A task that failed after its commit landed is still the sync base"
+ (let [failed-task (rst/create-sync-task! "export" (mt/user->id :rasta))]
+ (rst/set-version! (:id failed-task) "version 4")
+ (rst/fail-sync-task! (:id failed-task) "bookkeeping failed")
+ (is (= "version 4" (rst/last-version))))))))
;;; ------------------------------------------------------------------------------------------------
;;; Tests for supersede-stale-tasks!
;;; ------------------------------------------------------------------------------------------------
;;;
-;;; Used by `create-task-with-lock!` (auto-import path) to clean up rows whose owning JVM/thread
-;;; is gone or hung. Must NOT supersede brand-new tasks that haven't reported progress yet.
+;;; Called at boot, by `GET /current-task`, and before a new task is created, to close rows whose owning
+;;; JVM/thread is gone or hung. Must NOT supersede brand-new tasks that haven't reported progress yet.
(defn- insert-task!
"Helper: insert a RemoteSyncTask row with the given fields, bypassing the create-sync-task!
@@ -422,11 +563,22 @@
stale-task (insert-task! {:started_at old-time
:last_progress_report_at old-time
:progress 0.5})]
- (rst/supersede-stale-tasks!)
+ (mt/with-temporary-setting-values [:remote-sync-task-time-limit-ms 120000]
+ (rst/supersede-stale-tasks!))
(let [after (t2/select-one :model/RemoteSyncTask :id (:id stale-task))]
(is (true? (:cancelled after)))
(is (some? (:ended_at after)))
- (is (= "Superseded after staleness timeout" (:error_message after)))))))
+ (is (= "Sync was interrupted: the server stopped responding for 2 minutes (it may have restarted)"
+ (:error_message after)))))))
+
+(deftest supersede-stale-tasks!-message-uses-singular-for-one-minute-test
+ (testing "the interruption message reads '1 minute' when the window is one minute"
+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))
+ stale-task (insert-task! {:started_at old-time :last_progress_report_at old-time})]
+ (mt/with-temporary-setting-values [:remote-sync-task-time-limit-ms 60000]
+ (rst/supersede-stale-tasks!))
+ (is (= "Sync was interrupted: the server stopped responding for 1 minute (it may have restarted)"
+ (t2/select-one-fn :error_message :model/RemoteSyncTask :id (:id stale-task)))))))
(deftest supersede-stale-tasks!-leaves-brand-new-tasks-alone-test
(testing "supersede-stale-tasks! must NOT mark a brand-new task that just inserted —
@@ -458,6 +610,29 @@
"task with recent progress must not be terminated"))
(rst/complete-sync-task! (:id active-task)))))
+(deftest supersede-stale-tasks!-uses-heartbeat-over-progress-test
+ (testing "supersede-stale-tasks! leaves a task with a fresh heartbeat alone even when its progress is stale"
+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))
+ task (insert-task! {:started_at old-time
+ :last_progress_report_at old-time
+ :last_heartbeat_at (t/offset-date-time)
+ :progress 0.5})]
+ (rst/supersede-stale-tasks!)
+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id task))]
+ (is (false? (:cancelled after)))
+ (is (nil? (:ended_at after))))
+ (rst/complete-sync-task! (:id task))))
+ (testing "supersede-stale-tasks! reaps a task with a stale heartbeat even when its progress is fresh"
+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))
+ task (insert-task! {:started_at old-time
+ :last_progress_report_at (t/offset-date-time)
+ :last_heartbeat_at old-time
+ :progress 0.5})]
+ (rst/supersede-stale-tasks!)
+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id task))]
+ (is (true? (:cancelled after)))
+ (is (some? (:ended_at after)))))))
+
(deftest supersede-stale-tasks!-leaves-terminated-tasks-alone-test
(testing "supersede-stale-tasks! must NOT touch tasks that already have ended_at set"
(let [old-time (t/minus (t/offset-date-time) (t/hours 1))
@@ -473,6 +648,24 @@
(is (= (:ended_at before) (:ended_at after))
"ended_at must not be overwritten")))))
+(deftest supersede-stale-tasks!-returns-and-logs-reaped-ids-test
+ (testing "superseding returns the ids of the rows it ended and logs one warning naming them"
+ (let [now (t/offset-date-time)
+ old-time (t/minus now (t/hours 1))
+ stale (insert-task! {:started_at old-time :last_progress_report_at old-time})
+ fresh (insert-task! {:started_at now :last_progress_report_at now})]
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.remote-sync.models.remote-sync-task :warn]]
+ (is (= [(:id stale)] (rst/supersede-stale-tasks!)))
+ (is (=? [{:level :warn
+ :message (re-pattern (str "Superseded stale remote sync tasks \\[" (:id stale) "\\]: .*"))}]
+ (messages))))
+ (is (nil? (:ended_at (t2/select-one :model/RemoteSyncTask :id (:id fresh)))))
+ (testing "a second pass finds nothing and logs nothing"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.remote-sync.models.remote-sync-task :warn]]
+ (is (= [] (rst/supersede-stale-tasks!)))
+ (is (= [] (messages)))))
+ (rst/complete-sync-task! (:id fresh)))))
+
;;; ------------------------------------------------------------------------------------------------
;;; Tests for make-progress-reporter
;;; ------------------------------------------------------------------------------------------------
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj
index a07997f3cd8..01715759e0e 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/ingestable_test.clj
@@ -1,15 +1,14 @@
(ns metabase-enterprise.remote-sync.source.ingestable-test
(:require
[clojure.test :refer :all]
+ [metabase-enterprise.remote-sync.models.remote-sync-task :as rst]
[metabase-enterprise.remote-sync.source.ingestable :as ingestable]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase-enterprise.serialization.core :as serialization]
[metabase.audit-app.events.audit-log]
[metabase.test.fixtures :as fixtures]
- [metabase.test.util :as mt]
- [metabase.users.settings]
- [toucan2.core :as t2]))
+ [metabase.users.settings]))
(comment metabase.audit-app.events.audit-log/keep-me
metabase.users.settings/keep-me)
@@ -66,50 +65,31 @@
(is (= 3 (count @calls)) "Callback should be called three times"))))))
(deftest wrap-progress-ingestable-test
- (testing "wrap-progress-ingestable creates CallbackIngestable with progress tracking"
- ;; This test cannot use with-temp because the callback uses a separate connection to make
- ;; sure it is updating progress outside of the transaction serdes is using
- (mt/with-model-cleanup [:model/User :model/RemoteSyncTask]
- (let [user (first (t2/insert-returning-instances! :model/User {:first_name "Test"
- :last_name "User"
- :password "password123"}))
- task (first (t2/insert-returning-instances! :model/RemoteSyncTask {:sync_task_type "import"
- :initiated_by (:id user)}))
- task-id (:id task)
- mock-source (test-helpers/create-mock-source)
- base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil) (atom []))
- normalize 100]
- (testing "creates a CallbackIngestable"
- (let [wrapped (ingestable/wrap-progress-ingestable task-id normalize base-ingestable)]
- (is (instance? metabase_enterprise.remote_sync.source.ingestable.CallbackIngestable wrapped)
- "Should return CallbackIngestable instance")))
- (testing "updates task progress in database as items are ingested"
- (let [wrapped (ingestable/wrap-progress-ingestable task-id normalize base-ingestable)
- paths (serialization/ingest-list wrapped)
- total-paths (count paths)]
- (is (seq paths) "Should have paths to ingest")
- (let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)]
- (is (nil? (:progress initial-task)) "Progress should be nil initially"))
- (serialization/ingest-one wrapped (first paths))
- (let [task-after-first (t2/select-one :model/RemoteSyncTask :id task-id)]
- (is (some? (:progress task-after-first)) "Progress should be updated after first item")
- (is (< (abs (- (:progress task-after-first) (double (* (/ 1 total-paths) normalize)))) 0.01)
- "Progress should reflect one item ingested")
- (is (some? (:last_progress_report_at task-after-first))
- "last_progress_report_at should be set"))
- (serialization/ingest-one wrapped (second paths))
- (let [task-after-second (t2/select-one :model/RemoteSyncTask :id task-id)]
- (is (< (abs (- (:progress task-after-second) (double (* (/ 2 total-paths) normalize)))) 0.01)
- "Progress should reflect two items ingested"))))
- (testing "progress reaches normalize value when all items ingested"
- (let [wrapped (ingestable/wrap-progress-ingestable task-id normalize base-ingestable)
- paths (serialization/ingest-list wrapped)]
- (doseq [path paths]
- (serialization/ingest-one wrapped path))
- (let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)]
- (is (< (abs (- (:progress final-task) (double normalize))) 0.01)
- "Progress should equal normalize value when all items ingested"))))))))
+ (let [mock-source (test-helpers/create-mock-source)
+ base-ingestable (ingestable/->IngestableSnapshot (source.p/snapshot mock-source) (atom nil) (atom []))
+ paths (serialization/ingest-list base-ingestable)
+ total (count paths)
+ approx= (fn [expected actual] (< (abs (- expected actual)) 1e-9))]
+ (is (pos? total))
+ (testing "the n-th of N ingested entities reports lo + n/N * (hi - lo) through the reporter"
+ (let [writes (atom [])
+ report (rst/make-progress-reporter 1 {:throttle-ms 0 :write-fn (fn [f] (swap! writes conj f))})
+ wrapped (ingestable/wrap-progress-ingestable report [0.05 0.7] base-ingestable)]
+ (is (instance? metabase_enterprise.remote_sync.source.ingestable.CallbackIngestable wrapped))
+ (doseq [path paths]
+ (serialization/ingest-one wrapped path))
+ (is (= total (count @writes)) "one report per ingested entity")
+ (is (every? true? (map approx=
+ (map #(+ 0.05 (* (/ % total) 0.65)) (range 1 (inc total)))
+ @writes)))
+ (is (= 0.7 (last @writes)) "the last entity lands on hi exactly")))
+ (testing "a failing report is logged and the entity is still ingested"
+ (let [wrapped (ingestable/wrap-progress-ingestable (fn [_] (throw (RuntimeException. "db down"))) [0.0 1.0] base-ingestable)]
+ (is (some? (serialization/ingest-one wrapped (first paths))))))
+ (testing "a cancellation raised by the report propagates and stops the ingestion"
+ (let [wrapped (ingestable/wrap-progress-ingestable (fn [_] (throw (ex-info "cancelled" {:cancelled? true}))) [0.0 1.0] base-ingestable)]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"cancelled"
+ (serialization/ingest-one wrapped (first paths))))))))
(deftest root-dependency-ingestable-test
(testing "RootDependencyIngestable filters items based on root dependencies"
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx
index b389347c0ba..47ac85db5a8 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.stories.tsx
@@ -15,13 +15,22 @@ const initialState = createMockState({
const noop = () => {};
-export const Stalled = () => (
+const startedAt = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+
+const initiatedByUser = {
+ id: 1,
+ first_name: "Cynthia",
+ last_name: "Balusek",
+ email: "[email protected]",
+};
+
+export const InProgress = () => (
<ReduxProvider storeInitialState={initialState}>
<SyncProgressModal
taskType="export"
progress={0.3}
- isStalled
- minutesSinceLastUpdate={6}
+ startedAt={startedAt}
+ initiatedByUser={initiatedByUser}
isError={false}
errorMessage=""
isSuccess={false}
@@ -31,11 +40,15 @@ export const Stalled = () => (
</ReduxProvider>
);
-export const InProgress = () => (
+export const Quiet = () => (
<ReduxProvider storeInitialState={initialState}>
<SyncProgressModal
- taskType="export"
- progress={0.3}
+ taskType="import"
+ progress={0.8}
+ isQuiet
+ minutesSinceLastUpdate={3}
+ startedAt={startedAt}
+ initiatedByUser={initiatedByUser}
isError={false}
errorMessage=""
isSuccess={false}
@@ -44,3 +57,20 @@ export const InProgress = () => (
/>
</ReduxProvider>
);
+
+export const Cancelled = () => (
+ <ReduxProvider storeInitialState={initialState}>
+ <SyncProgressModal
+ taskType="import"
+ progress={0.21}
+ isCancelled
+ startedAt={startedAt}
+ initiatedByUser={null}
+ isError={false}
+ errorMessage="Sync was interrupted: the server stopped responding for 5 minutes (it may have restarted)"
+ isSuccess={false}
+ outcome={null}
+ onDismiss={noop}
+ />
+ </ReduxProvider>
+);
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx
index 470268c52c8..5223fed876f 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.tsx
@@ -1,18 +1,29 @@
-import { msgid, ngettext, t } from "ttag";
+import { c, msgid, ngettext, t } from "ttag";
import { ActionButton } from "metabase/common/components/ActionButton";
import { useToast } from "metabase/common/hooks";
import { getUserIsAdmin } from "metabase/current-user";
+import { dayjs } from "metabase/dayjs";
import { useSelector } from "metabase/redux";
import { Button, Group, Modal, Progress, Stack, Text } from "metabase/ui";
+import { getUserName } from "metabase/utils/user";
import { useCancelRemoteSyncCurrentTaskMutation } from "metabase-enterprise/api";
-import type { RemoteSyncOutcome, RemoteSyncTaskType } from "metabase-types/api";
+import type {
+ RemoteSyncOutcome,
+ RemoteSyncTaskType,
+ RemoteSyncTaskUser,
+} from "metabase-types/api";
+
+import { getProgressPhaseLabel } from "../utils";
interface SyncProgressModalProps {
taskType: RemoteSyncTaskType;
progress: number;
- isStalled?: boolean;
+ isQuiet?: boolean;
+ isCancelled?: boolean;
minutesSinceLastUpdate?: number | null;
+ startedAt?: string | null;
+ initiatedByUser?: RemoteSyncTaskUser | null;
isError: boolean;
errorMessage: string;
isSuccess: boolean;
@@ -23,15 +34,18 @@ interface SyncProgressModalProps {
export function SyncProgressModal({
progress,
taskType,
- isStalled = false,
+ isQuiet = false,
+ isCancelled = false,
minutesSinceLastUpdate = null,
+ startedAt = null,
+ initiatedByUser = null,
isError,
errorMessage,
isSuccess,
outcome,
onDismiss,
}: SyncProgressModalProps) {
- const canCancel = useSelector(getUserIsAdmin);
+ const isAdmin = useSelector(getUserIsAdmin);
const [cancelRemoteSyncCurrentTask] =
useCancelRemoteSyncCurrentTaskMutation();
@@ -63,12 +77,17 @@ export function SyncProgressModal({
}
};
+ const startedBy = (
+ <StartedByLine startedAt={startedAt} initiatedByUser={initiatedByUser} />
+ );
+
if (isError) {
return (
<Modal onClose={onDismiss} opened size="md" title={t`Sync failed`}>
<Stack mt="lg" gap="lg">
<Text>{t`An error occurred during sync.`}</Text>
{errorMessage && <Text>{errorMessage}</Text>}
+ {startedBy}
<Group justify="flex-end">
<Button
data-testid="sync-error-close-button"
@@ -100,6 +119,25 @@ export function SyncProgressModal({
);
}
+ if (isCancelled) {
+ return (
+ <Modal onClose={onDismiss} opened size="md" title={t`Sync stopped`}>
+ <Stack mt="lg" gap="lg">
+ <Text>{errorMessage || t`The sync was cancelled.`}</Text>
+ <StoppedAtLine progress={progress} taskType={taskType} />
+ {startedBy}
+ <Group justify="flex-end">
+ <Button
+ data-testid="sync-cancelled-close-button"
+ onClick={onDismiss}
+ variant="filled"
+ >{t`Close`}</Button>
+ </Group>
+ </Stack>
+ </Modal>
+ );
+ }
+
const { title, progressLabel } = getModalContent(taskType);
return (
@@ -111,22 +149,16 @@ export function SyncProgressModal({
withCloseButton={false}
>
<Stack mt="lg" gap="lg">
- {isStalled ? (
- <Text ta="center">{getStalledMessage(minutesSinceLastUpdate)}</Text>
- ) : (
- <>
- <Text ta="center">{progressLabel}</Text>
- <Progress
- value={progress * 100}
- transitionDuration={300}
- animated
- />
- </>
+ <Text ta="center">{progressLabel}</Text>
+ <Progress value={progress * 100} transitionDuration={300} animated />
+ {isQuiet && (
+ <Text ta="center">{getQuietMessage(minutesSinceLastUpdate)}</Text>
)}
+ {startedBy}
<Text size="sm">
{t`Please wait until this finishes before editing content.`}
</Text>
- {canCancel && (
+ {isAdmin && (
<Group justify="flex-end">
<ActionButton
actionFn={onCancel}
@@ -141,15 +173,57 @@ export function SyncProgressModal({
);
}
-function getStalledMessage(minutesSinceLastUpdate: number | null): string {
+function StartedByLine({
+ startedAt,
+ initiatedByUser,
+}: {
+ startedAt: string | null;
+ initiatedByUser: RemoteSyncTaskUser | null;
+}) {
+ if (!startedAt) {
+ return null;
+ }
+
+ const time = dayjs(startedAt).format("LT");
+ const name = initiatedByUser ? getUserName(initiatedByUser) : null;
+
+ return (
+ <Text size="sm" c="text-secondary" data-testid="sync-started-by">
+ {name
+ ? c("{0} is a person's name, {1} is a time of day")
+ .t`Started by ${name} at ${time}`
+ : c("{0} is a time of day").t`Started at ${time}`}
+ </Text>
+ );
+}
+
+function StoppedAtLine({
+ progress,
+ taskType,
+}: {
+ progress: number;
+ taskType: RemoteSyncTaskType;
+}) {
+ const percent = Math.round(progress * 100);
+ const phase = getProgressPhaseLabel(progress, taskType);
+
+ return (
+ <Text size="sm" c="text-secondary" data-testid="sync-stopped-at">
+ {c("{0} is a percentage, {1} is a sync phase such as 'importing content'")
+ .t`Stopped at ${percent}% while ${phase}`}
+ </Text>
+ );
+}
+
+function getQuietMessage(minutesSinceLastUpdate: number | null): string {
if (minutesSinceLastUpdate == null) {
- return t`Still working…`;
+ return t`No progress reported recently. The sync is still running.`;
}
const minutes = minutesSinceLastUpdate;
return ngettext(
- msgid`Still working. No update for ${minutes} minute.`,
- `Still working. No update for ${minutes} minutes.`,
+ msgid`No progress for ${minutes} minute. The sync is still running.`,
+ `No progress for ${minutes} minutes. The sync is still running.`,
minutes,
);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx
index e8553949e9a..bddd59ca3eb 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/SyncProgressModal.unit.spec.tsx
@@ -6,16 +6,32 @@ import {
} from "__support__/server-mocks";
import { createMockState } from "__support__/state";
import { renderWithProviders, screen, waitFor } from "__support__/ui";
-import type { RemoteSyncOutcome } from "metabase-types/api";
+import type { RemoteSyncOutcome, RemoteSyncTaskUser } from "metabase-types/api";
import { createMockUser } from "metabase-types/api/mocks";
import { SyncProgressModal } from "./SyncProgressModal";
+const STARTED_AT = "2026-09-15T22:06:23Z";
+const STARTED_AT_TIME = new Date(STARTED_AT).toLocaleTimeString([], {
+ hour: "numeric",
+ minute: "2-digit",
+});
+
+const INITIATED_BY_USER: RemoteSyncTaskUser = {
+ id: 231,
+ first_name: "Cynthia",
+ last_name: "Balusek",
+ email: "[email protected]",
+};
+
const setup = ({
taskType = "import" as const,
progress = 0.5,
- isStalled = false,
+ isQuiet = false,
+ isCancelled = false,
minutesSinceLastUpdate = null,
+ startedAt = null,
+ initiatedByUser = null,
isError = false,
errorMessage = "",
isSuccess = false,
@@ -26,8 +42,11 @@ const setup = ({
}: {
taskType?: "import" | "export";
progress?: number;
- isStalled?: boolean;
+ isQuiet?: boolean;
+ isCancelled?: boolean;
minutesSinceLastUpdate?: number | null;
+ startedAt?: string | null;
+ initiatedByUser?: RemoteSyncTaskUser | null;
isError?: boolean;
errorMessage?: string;
isSuccess?: boolean;
@@ -46,8 +65,11 @@ const setup = ({
<SyncProgressModal
taskType={taskType}
progress={progress}
- isStalled={isStalled}
+ isQuiet={isQuiet}
+ isCancelled={isCancelled}
minutesSinceLastUpdate={minutesSinceLastUpdate}
+ startedAt={startedAt}
+ initiatedByUser={initiatedByUser}
isError={isError}
errorMessage={errorMessage}
isSuccess={isSuccess}
@@ -107,26 +129,144 @@ describe("SyncProgressModal", () => {
});
});
- describe("stalled progress state (GHY-4132)", () => {
- it("replaces the frozen progress bar with a 'still working' message when the task has stalled", () => {
- // Reproduces GHY-4132: a large push stops reporting progress, the backend marks the still-running
- // task :timed-out, and the UI used to sit on a frozen 'Exporting content…' bar. Instead we report
- // honestly that it is still running but hasn't reported progress recently.
- setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 6 });
+ describe("quiet running state", () => {
+ it("keeps the progress bar and says the sync is still running when progress has not moved", () => {
+ setup({
+ taskType: "export",
+ progress: 0.3,
+ isQuiet: true,
+ minutesSinceLastUpdate: 6,
+ });
expect(
- screen.getByText("Still working. No update for 6 minutes."),
+ screen.getByText(
+ "No progress for 6 minutes. The sync is still running.",
+ ),
+ ).toBeInTheDocument();
+ expect(screen.getByRole("progressbar")).toBeInTheDocument();
+ expect(screen.getByText("Exporting content…")).toBeInTheDocument();
+ expect(
+ screen.getByRole("button", { name: "Cancel" }),
).toBeInTheDocument();
- expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();
- expect(screen.queryByText("Exporting content…")).not.toBeInTheDocument();
});
it("uses the singular 'minute' when only one minute has passed", () => {
- setup({ taskType: "export", isStalled: true, minutesSinceLastUpdate: 1 });
+ setup({ isQuiet: true, minutesSinceLastUpdate: 1 });
+
+ expect(
+ screen.getByText(
+ "No progress for 1 minute. The sync is still running.",
+ ),
+ ).toBeInTheDocument();
+ });
+
+ it("shows no hint while progress is fresh", () => {
+ setup({ isQuiet: false, minutesSinceLastUpdate: 0 });
+
+ expect(screen.queryByText(/still running/)).not.toBeInTheDocument();
+ });
+ });
+
+ describe("cancelled state", () => {
+ it("shows the row's message with a Close button and no progress bar", async () => {
+ const onDismiss = jest.fn();
+ setup({
+ taskType: "import",
+ progress: 0.21,
+ isCancelled: true,
+ errorMessage:
+ "Sync was interrupted: the server stopped responding for 5 minutes (it may have restarted)",
+ onDismiss,
+ });
+ expect(screen.getByText("Sync stopped")).toBeInTheDocument();
expect(
- screen.getByText("Still working. No update for 1 minute."),
+ screen.getByText(
+ "Sync was interrupted: the server stopped responding for 5 minutes (it may have restarted)",
+ ),
).toBeInTheDocument();
+ expect(
+ screen.getByText("Stopped at 21% while importing content"),
+ ).toBeInTheDocument();
+ expect(screen.queryByRole("progressbar")).not.toBeInTheDocument();
+ expect(
+ screen.queryByRole("button", { name: "Cancel" }),
+ ).not.toBeInTheDocument();
+
+ await userEvent.click(screen.getByTestId("sync-cancelled-close-button"));
+
+ expect(onDismiss).toHaveBeenCalled();
+ });
+
+ it("falls back to generic copy when the row has no message", () => {
+ setup({ isCancelled: true, errorMessage: "" });
+
+ expect(screen.getByText("The sync was cancelled.")).toBeInTheDocument();
+ });
+ });
+
+ describe("started-by line", () => {
+ it("names the initiating user and the start time", () => {
+ setup({
+ isCancelled: true,
+ startedAt: STARTED_AT,
+ initiatedByUser: INITIATED_BY_USER,
+ });
+
+ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(
+ `Started by Cynthia Balusek at ${STARTED_AT_TIME}`,
+ );
+ });
+
+ it("falls back to the email when the user has no name", () => {
+ setup({
+ isCancelled: true,
+ startedAt: STARTED_AT,
+ initiatedByUser: {
+ ...INITIATED_BY_USER,
+ first_name: null,
+ last_name: null,
+ },
+ });
+
+ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(
+ `Started by [email protected] at ${STARTED_AT_TIME}`,
+ );
+ });
+
+ it("shows only the start time for a task with no initiating user", () => {
+ setup({ isError: true, startedAt: STARTED_AT, initiatedByUser: null });
+
+ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(
+ `Started at ${STARTED_AT_TIME}`,
+ );
+ });
+
+ it("is shown on the running view so a second admin can see whose task it is", () => {
+ setup({
+ startedAt: STARTED_AT,
+ initiatedByUser: INITIATED_BY_USER,
+ });
+
+ expect(screen.getByTestId("sync-started-by")).toHaveTextContent(
+ "Started by Cynthia Balusek",
+ );
+ });
+
+ it("is omitted when the start time is unknown", () => {
+ setup({ isCancelled: true, startedAt: null });
+
+ expect(screen.queryByTestId("sync-started-by")).not.toBeInTheDocument();
+ });
+
+ it("is omitted from the success view", () => {
+ setup({
+ isSuccess: true,
+ startedAt: STARTED_AT,
+ initiatedByUser: INITIATED_BY_USER,
+ });
+
+ expect(screen.queryByTestId("sync-started-by")).not.toBeInTheDocument();
});
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts
index ae72d53cb60..59028d16eb7 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts
@@ -19,6 +19,8 @@ export const TRANSFORMS_KEY = "remote-sync-transforms";
export const COLLECTIONS_KEY = "collections";
// Used in modal variant when library doesn't exist yet but user wants to sync it
export const SYNC_LIBRARY_PENDING_KEY = "sync-library-pending";
+// A running task whose progress has not moved for this long gets a "still running" hint in the modal
+export const SYNC_QUIET_AFTER_MINUTES = 2;
export const REMOTE_SYNC_SCHEMA = Yup.object({
[REMOTE_SYNC_KEY]: Yup.boolean().nullable().default(true),
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx
index 2c8dffc081f..d17ac4a3a48 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.tsx
@@ -4,21 +4,23 @@ import { useSetting } from "metabase/settings";
import { useGetRemoteSyncCurrentTaskQuery } from "metabase-enterprise/api";
import { SyncProgressModal } from "../components/SyncProgressModal";
-import { REMOTE_SYNC_KEY } from "../constants";
+import { REMOTE_SYNC_KEY, SYNC_QUIET_AFTER_MINUTES } from "../constants";
import {
getErrorMessage,
getHasPendingMutation,
+ getInitiatedByUser,
+ getIsCancelled,
getIsError,
getIsRunning,
- getIsStalled,
getIsSuccess,
getLastProgressReportAt,
getProgress,
getShowModal,
+ getStartedAt,
getTaskOutcome,
getTaskType,
} from "../selectors";
-import { modalDismissed } from "../sync-task-slice";
+import { modalDismissed, taskCleared } from "../sync-task-slice";
const SYNC_STATUS_POLL_INTERVAL = 2000;
@@ -31,8 +33,10 @@ export const useSyncStatus = () => {
const taskType = useSelector(getTaskType);
const progress = useSelector(getProgress);
const isError = useSelector(getIsError);
- const isStalled = useSelector(getIsStalled);
+ const isCancelled = useSelector(getIsCancelled);
const lastProgressReportAt = useSelector(getLastProgressReportAt);
+ const startedAt = useSelector(getStartedAt);
+ const initiatedByUser = useSelector(getInitiatedByUser);
const errorMessage = useSelector(getErrorMessage);
const isSuccess = useSelector(getIsSuccess);
const outcome = useSelector(getTaskOutcome);
@@ -42,26 +46,41 @@ export const useSyncStatus = () => {
? dayjs().diff(dayjs(lastProgressReportAt), "minute")
: null;
+ const isQuiet =
+ isRunning &&
+ minutesSinceLastUpdate !== null &&
+ minutesSinceLastUpdate >= SYNC_QUIET_AFTER_MINUTES;
+
const shouldPoll = isRunning && showModal && !hasPendingMutation;
+ // Always subscribed, so a fresh tab discovers a sync started elsewhere (or a stale row the backend
+ // repairs on read); polling only while a task is being watched.
useGetRemoteSyncCurrentTaskQuery(undefined, {
pollingInterval: shouldPoll ? SYNC_STATUS_POLL_INTERVAL : undefined,
skipPollingIfUnfocused: true,
- skip: !isRemoteSyncEnabled || !shouldPoll,
+ skip: !isRemoteSyncEnabled,
});
+ // A stopped task has nothing left to poll, so drop it instead of hiding the modal; otherwise the next
+ // taskUpdated for the same stale row would reopen it.
+ const onDismiss = () =>
+ dispatch(isCancelled ? taskCleared() : modalDismissed());
+
const progressModal =
showModal && taskType ? (
<SyncProgressModal
taskType={taskType}
progress={progress}
- isStalled={isStalled}
+ isQuiet={isQuiet}
+ isCancelled={isCancelled}
minutesSinceLastUpdate={minutesSinceLastUpdate}
+ startedAt={startedAt}
+ initiatedByUser={initiatedByUser}
isError={isError}
errorMessage={errorMessage}
isSuccess={isSuccess}
outcome={outcome}
- onDismiss={() => dispatch(modalDismissed())}
+ onDismiss={onDismiss}
/>
) : null;
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.unit.spec.tsx
new file mode 100644
index 00000000000..d937528bc1f
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-sync-status.unit.spec.tsx
@@ -0,0 +1,66 @@
+import fetchMock from "fetch-mock";
+
+import { setupEnterprisePlugins } from "__support__/enterprise";
+import { mockSettings } from "__support__/settings";
+import { createMockState } from "__support__/state";
+import { renderHookWithProviders, waitFor } from "__support__/ui";
+import type { RemoteSyncTask } from "metabase-types/api";
+import {
+ createMockTokenFeatures,
+ createMockUser,
+} from "metabase-types/api/mocks";
+
+import { useSyncStatus } from "./use-sync-status";
+
+const CURRENT_TASK = "path:/api/ee/remote-sync/current-task";
+
+const setup = ({
+ task,
+ enabled = true,
+}: {
+ task: Partial<RemoteSyncTask> | null;
+ enabled?: boolean;
+}) => {
+ // the plugin registers its reducer and middleware only once the premium gate passes
+ const settings = mockSettings({
+ "token-features": createMockTokenFeatures({ remote_sync: true }),
+ "remote-sync-enabled": enabled,
+ "remote-sync-type": "read-write",
+ "remote-sync-branch": "main",
+ });
+ setupEnterprisePlugins();
+ fetchMock.get(CURRENT_TASK, task ? { status: 200, body: task } : 204);
+
+ return renderHookWithProviders(() => useSyncStatus(), {
+ storeInitialState: createMockState({
+ currentUser: createMockUser({ is_superuser: true }),
+ settings,
+ }),
+ });
+};
+
+const settle = () => new Promise((resolve) => setTimeout(resolve, 100));
+
+// The test store carries no plugin middleware, so these cover the subscription itself; what a fetched
+// task does to the store is covered by the listener middleware spec.
+describe("useSyncStatus", () => {
+ it("fetches once and stays idle when there is no task", async () => {
+ const { result } = setup({ task: null });
+
+ await waitFor(() => {
+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(1);
+ });
+ await settle();
+
+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(1);
+ expect(result.current.isRunning).toBe(false);
+ expect(result.current.progressModal).toBeNull();
+ });
+
+ it("does not fetch when remote sync is disabled", async () => {
+ setup({ task: null, enabled: false });
+ await settle();
+
+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(0);
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts
index 62959252763..64252324590 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.ts
@@ -9,6 +9,7 @@ import { tag } from "metabase-enterprise/api/tags";
import type { RemoteSyncTaskStatus } from "metabase-types/api";
import { REMOTE_SYNC_INVALIDATION_TAGS } from "../constants";
+import { getCurrentTask } from "../selectors";
import {
modalDismissed,
syncConflictVariantUpdated,
@@ -109,15 +110,29 @@ const terminalTaskStates: RemoteSyncTaskStatus[] = [
"successful",
"errored",
"cancelled",
- "timed-out",
] as const;
remoteSyncListenerMiddleware.startListening({
matcher: remoteSyncApi.endpoints.getRemoteSyncCurrentTask.matchFulfilled,
- effect: async (action, { dispatch }) => {
+ effect: async (action, { dispatch, getOriginalState }) => {
const task = action.payload;
if (task) {
+ // The query is subscribed whenever remote sync is enabled, so a fetch made while nothing is being
+ // watched is discovery, not an event: a task still running elsewhere is picked up so this tab can
+ // follow it, while a finished one is history and is left alone. Treating history as an event would
+ // replay an old conflict on every page load and, since the terminal handling invalidates this
+ // query's own tag, refetch forever.
+ const previous = getCurrentTask(getOriginalState());
+ const wasRunning = previous !== null && previous.ended_at === null;
+
+ if (!wasRunning) {
+ if (task.ended_at === null) {
+ dispatch(taskUpdated(task));
+ }
+ return;
+ }
+
dispatch(taskUpdated(task));
if (task.status === "conflict") {
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.ts
index e53eb0faad1..887a10b456f 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/remote-sync-listener-middleware.unit.spec.ts
@@ -22,7 +22,11 @@ import {
createMockSettings,
} from "metabase-types/api/mocks";
-import { initialState, remoteSyncReducer } from "../sync-task-slice";
+import {
+ initialState,
+ remoteSyncReducer,
+ taskStarted,
+} from "../sync-task-slice";
import { remoteSyncListenerMiddleware } from "./remote-sync-listener-middleware";
@@ -31,12 +35,13 @@ const createTestStore = (
) => {
const store = configureStore({
reducer: combineReducers({
- remoteSyncPlugin: remoteSyncReducer,
+ // nested as the app registers it (PLUGIN_REDUCERS -> state.plugins), which the selectors read
+ plugins: combineReducers({ remoteSyncPlugin: remoteSyncReducer }),
// EnterpriseApi is an enhanced version of Api, so they share the same reducer
[Api.reducerPath]: Api.reducer,
}),
preloadedState: {
- remoteSyncPlugin: initialState,
+ plugins: { remoteSyncPlugin: initialState },
// Settings are served from the getSessionProperties RTK Query cache.
[Api.reducerPath]: seedApiQueryCache(undefined, [
{
@@ -96,12 +101,12 @@ describe("remote-sync-listener-middleware", () => {
// Wait for the request to complete and middleware to process
await waitForCondition(() => {
const state = store.getState();
- return state.remoteSyncPlugin?.showModal === true;
+ return state.plugins.remoteSyncPlugin.showModal === true;
});
const state = store.getState();
- expect(state.remoteSyncPlugin?.showModal).toBe(true);
- expect(state.remoteSyncPlugin?.currentTask?.sync_task_type).toBe(
+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(true);
+ expect(state.plugins.remoteSyncPlugin.currentTask?.sync_task_type).toBe(
"import",
);
});
@@ -130,8 +135,8 @@ describe("remote-sync-listener-middleware", () => {
await new Promise((resolve) => setTimeout(resolve, 100));
const state = store.getState();
- expect(state.remoteSyncPlugin?.showModal).toBe(false);
- expect(state.remoteSyncPlugin?.currentTask).toBeNull();
+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);
+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();
});
it("should NOT show modal when settings save fails", async () => {
@@ -159,8 +164,8 @@ describe("remote-sync-listener-middleware", () => {
await new Promise((resolve) => setTimeout(resolve, 100));
const state = store.getState();
- expect(state.remoteSyncPlugin?.showModal).toBe(false);
- expect(state.remoteSyncPlugin?.currentTask).toBeNull();
+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);
+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();
});
it("should NOT show modal when disabling remote sync", async () => {
@@ -187,8 +192,8 @@ describe("remote-sync-listener-middleware", () => {
await new Promise((resolve) => setTimeout(resolve, 100));
const state = store.getState();
- expect(state.remoteSyncPlugin?.showModal).toBe(false);
- expect(state.remoteSyncPlugin?.currentTask).toBeNull();
+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);
+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();
});
});
@@ -212,12 +217,12 @@ describe("remote-sync-listener-middleware", () => {
// The import listener triggers on matchPending, so modal should show immediately
await waitForCondition(() => {
const state = store.getState();
- return state.remoteSyncPlugin?.showModal === true;
+ return state.plugins.remoteSyncPlugin.showModal === true;
});
const state = store.getState();
- expect(state.remoteSyncPlugin?.showModal).toBe(true);
- expect(state.remoteSyncPlugin?.currentTask?.sync_task_type).toBe(
+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(true);
+ expect(state.plugins.remoteSyncPlugin.currentTask?.sync_task_type).toBe(
"import",
);
});
@@ -247,8 +252,8 @@ describe("remote-sync-listener-middleware", () => {
await new Promise((resolve) => setTimeout(resolve, 100));
const state = store.getState();
- expect(state.remoteSyncPlugin?.showModal).toBe(false);
- expect(state.remoteSyncPlugin?.currentTask).toBeNull();
+ expect(state.plugins.remoteSyncPlugin.showModal).toBe(false);
+ expect(state.plugins.remoteSyncPlugin.currentTask).toBeNull();
});
it("should set conflict variant to 'setup' when import fails with conflict", async () => {
@@ -256,10 +261,13 @@ describe("remote-sync-listener-middleware", () => {
status: 200,
body: {
status: "conflict",
+ sync_task_type: "import",
+ ended_at: "2026-01-01T00:00:01Z",
},
});
const store = createTestStore();
+ store.dispatch(taskStarted({ taskType: "import" }));
// Dispatch the mutation
store.dispatch(
@@ -271,13 +279,12 @@ describe("remote-sync-listener-middleware", () => {
fetchMock.callHistory.done("path:/api/ee/remote-sync/current-task"),
);
- expect(store.getState().remoteSyncPlugin?.showModal).toBe(false);
-
await waitFor(() => {
- expect(store.getState().remoteSyncPlugin?.syncConflictVariant).toBe(
- "setup",
- );
+ expect(
+ store.getState().plugins.remoteSyncPlugin.syncConflictVariant,
+ ).toBe("setup");
});
+ expect(store.getState().plugins.remoteSyncPlugin.showModal).toBe(false);
});
it("does NOT open the setup modal when an export task ends in conflict", async () => {
@@ -285,10 +292,15 @@ describe("remote-sync-listener-middleware", () => {
// not by the middleware — so the middleware must not route them to the setup-conflict modal.
fetchMock.get("path:/api/ee/remote-sync/current-task", {
status: 200,
- body: { status: "conflict", sync_task_type: "export" },
+ body: {
+ status: "conflict",
+ sync_task_type: "export",
+ ended_at: "2026-01-01T00:00:01Z",
+ },
});
const store = createTestStore();
+ store.dispatch(taskStarted({ taskType: "export" }));
store.dispatch(
remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),
@@ -299,11 +311,101 @@ describe("remote-sync-listener-middleware", () => {
);
await waitFor(() => {
- expect(store.getState().remoteSyncPlugin?.showModal).toBe(false);
+ expect(
+ store.getState().plugins.remoteSyncPlugin.currentTask?.status,
+ ).toBe("conflict");
+ });
+ expect(store.getState().plugins.remoteSyncPlugin.showModal).toBe(false);
+ expect(
+ store.getState().plugins.remoteSyncPlugin.syncConflictVariant,
+ ).not.toBe("setup");
+ });
+ });
+
+ describe("getRemoteSyncCurrentTask listener", () => {
+ const CURRENT_TASK = "path:/api/ee/remote-sync/current-task";
+ const settle = () => new Promise((resolve) => setTimeout(resolve, 100));
+
+ it("follows a task found running on load so the tab can watch it", async () => {
+ fetchMock.get(CURRENT_TASK, {
+ status: 200,
+ body: {
+ id: 5,
+ status: "running",
+ sync_task_type: "import",
+ progress: 0.4,
+ ended_at: null,
+ },
});
- expect(store.getState().remoteSyncPlugin?.syncConflictVariant).not.toBe(
- "setup",
+ const store = createTestStore();
+
+ store.dispatch(
+ remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),
);
+
+ await waitFor(() => {
+ expect(store.getState().plugins.remoteSyncPlugin.currentTask?.id).toBe(
+ 5,
+ );
+ });
+ expect(store.getState().plugins.remoteSyncPlugin.showModal).toBe(true);
+ });
+
+ it("leaves a finished task found on load alone", async () => {
+ fetchMock.get(CURRENT_TASK, {
+ status: 200,
+ body: {
+ id: 5,
+ status: "conflict",
+ sync_task_type: "import",
+ ended_at: "2026-01-01T00:00:01Z",
+ },
+ });
+ const store = createTestStore();
+
+ store.dispatch(
+ remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),
+ );
+
+ await waitForCondition(() => fetchMock.callHistory.done(CURRENT_TASK));
+ await settle();
+
+ expect(store.getState().plugins.remoteSyncPlugin.currentTask).toBeNull();
+ expect(
+ store.getState().plugins.remoteSyncPlugin.syncConflictVariant,
+ ).toBeNull();
+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(1);
+ });
+
+ it("refreshes the task once when a watched task ends, not in a loop", async () => {
+ fetchMock.get(CURRENT_TASK, {
+ status: 200,
+ body: {
+ id: 5,
+ status: "successful",
+ sync_task_type: "import",
+ progress: 1,
+ ended_at: "2026-01-01T00:00:01Z",
+ },
+ });
+ const store = createTestStore();
+ store.dispatch(taskStarted({ taskType: "import" }));
+
+ store.dispatch(
+ remoteSyncApi.endpoints.getRemoteSyncCurrentTask.initiate(),
+ );
+
+ await waitFor(() => {
+ expect(store.getState().plugins.remoteSyncPlugin.currentTask?.id).toBe(
+ 5,
+ );
+ });
+ // the terminal handling invalidates this query's tag, so the subscribed query refetches once
+ await waitFor(() => {
+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(2);
+ });
+ await settle();
+ expect(fetchMock.callHistory.calls(CURRENT_TASK)).toHaveLength(2);
});
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts
index 4782d691aaf..f2f6fae3f67 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.ts
@@ -40,9 +40,9 @@ export const getIsError = createSelector(
(currentTask) => currentTask?.status === "errored",
);
-export const getIsStalled = createSelector(
+export const getIsCancelled = createSelector(
getCurrentTask,
- (currentTask) => currentTask?.status === "timed-out",
+ (currentTask) => currentTask?.status === "cancelled",
);
export const getLastProgressReportAt = createSelector(
@@ -50,6 +50,16 @@ export const getLastProgressReportAt = createSelector(
(currentTask) => currentTask?.last_progress_report_at ?? null,
);
+export const getStartedAt = createSelector(
+ getCurrentTask,
+ (currentTask) => currentTask?.started_at ?? null,
+);
+
+export const getInitiatedByUser = createSelector(
+ getCurrentTask,
+ (currentTask) => currentTask?.initiated_by_user ?? null,
+);
+
export const getIsSuccess = createSelector(
getCurrentTask,
(currentTask) => currentTask?.status === "successful",
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.unit.spec.ts
new file mode 100644
index 00000000000..ba283854ba9
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/selectors.unit.spec.ts
@@ -0,0 +1,57 @@
+import { createMockState } from "__support__/state";
+import type { RemoteSyncTask } from "metabase-types/api";
+
+import { getInitiatedByUser, getIsCancelled, getStartedAt } from "./selectors";
+import { initialState } from "./sync-task-slice";
+import type { RemoteSyncStoreState } from "./types";
+
+const task: RemoteSyncTask = {
+ id: 2080,
+ sync_task_type: "import",
+ status: "cancelled",
+ progress: 0.21,
+ started_at: "2026-09-15T22:06:23Z",
+ ended_at: "2026-09-15T22:40:00Z",
+ last_progress_report_at: "2026-09-15T22:08:39Z",
+ error_message: "Task cancelled",
+ initiated_by: 231,
+ initiated_by_user: {
+ id: 231,
+ first_name: "Cynthia",
+ last_name: "Balusek",
+ email: "[email protected]",
+ },
+};
+
+const stateWith = (
+ currentTask: RemoteSyncTask | null,
+): RemoteSyncStoreState => ({
+ ...createMockState(),
+ plugins: { remoteSyncPlugin: { ...initialState, currentTask } },
+});
+
+describe("remote_sync selectors", () => {
+ it("getIsCancelled is true only for a cancelled task", () => {
+ expect(getIsCancelled(stateWith(task))).toBe(true);
+ expect(getIsCancelled(stateWith({ ...task, status: "successful" }))).toBe(
+ false,
+ );
+ expect(getIsCancelled(stateWith(null))).toBe(false);
+ });
+
+ it("getStartedAt returns the start time or null", () => {
+ expect(getStartedAt(stateWith(task))).toBe("2026-09-15T22:06:23Z");
+ expect(getStartedAt(stateWith(null))).toBeNull();
+ });
+
+ it("getInitiatedByUser returns the trimmed user, or null when the task has none", () => {
+ expect(getInitiatedByUser(stateWith(task))).toEqual(task.initiated_by_user);
+ expect(
+ getInitiatedByUser(stateWith({ ...task, initiated_by_user: null })),
+ ).toBeNull();
+ expect(
+ getInitiatedByUser(stateWith({ ...task, initiated_by_user: undefined })),
+ ).toBeNull();
+ expect(getInitiatedByUser(stateWith(null))).toBeNull();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.ts
index cf83e8d374c..e290edd3920 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/sync-task-slice.ts
@@ -34,6 +34,7 @@ export const remoteSyncSlice = createSlice({
started_at: new Date().toISOString(),
ended_at: null,
last_progress_report_at: null,
+ last_heartbeat_at: null,
error_message: null,
initiated_by: 0,
};
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
index 27e36db1a3b..3a71e8b14e5 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.ts
@@ -7,6 +7,7 @@ import type {
IconName,
RemoteSyncEntityStatus,
RemoteSyncRequiredSync,
+ RemoteSyncTaskType,
SettingDefinition,
} from "metabase-types/api";
@@ -57,6 +58,33 @@ export const getEnvSettingProps = <T>(
return {};
};
+// Thresholds follow the backend checkpoints: import reports 0.05 after the conflict scan, 0.70 after the
+// load and 0.90 after the ledger commit; export reports 0.33 after planning and 0.66 after serializing.
+export const getProgressPhaseLabel = (
+ progress: number,
+ taskType: RemoteSyncTaskType,
+): string => {
+ if (taskType === "export") {
+ if (progress < 0.33) {
+ return t`preparing`;
+ }
+ if (progress < 0.66) {
+ return t`exporting content`;
+ }
+ return t`pushing`;
+ }
+ if (progress < 0.05) {
+ return t`preparing`;
+ }
+ if (progress < 0.7) {
+ return t`importing content`;
+ }
+ if (progress < 0.9) {
+ return t`recording sync state`;
+ }
+ return t`finishing`;
+};
+
export const getSyncStatusIcon = (status: RemoteSyncEntityStatus): IconName => {
switch (status) {
case "create":
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts
index 0f99c07be9d..a65bedbd9c4 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/utils.unit.spec.ts
@@ -10,6 +10,7 @@ import {
getBlockedReason,
getCollectionPathSegments,
getListedRequiredSyncs,
+ getProgressPhaseLabel,
getRequiredSyncRow,
isTableChildModel,
} from "./utils";
@@ -201,6 +202,28 @@ describe("remote_sync utils", () => {
});
});
+ describe("getProgressPhaseLabel", () => {
+ it("maps import progress onto the backend checkpoints", () => {
+ expect(getProgressPhaseLabel(0, "import")).toBe("preparing");
+ expect(getProgressPhaseLabel(0.049, "import")).toBe("preparing");
+ expect(getProgressPhaseLabel(0.05, "import")).toBe("importing content");
+ expect(getProgressPhaseLabel(0.69, "import")).toBe("importing content");
+ expect(getProgressPhaseLabel(0.7, "import")).toBe("recording sync state");
+ expect(getProgressPhaseLabel(0.89, "import")).toBe(
+ "recording sync state",
+ );
+ expect(getProgressPhaseLabel(0.9, "import")).toBe("finishing");
+ expect(getProgressPhaseLabel(1, "import")).toBe("finishing");
+ });
+
+ it("maps export progress onto the plan, serialize and push phases", () => {
+ expect(getProgressPhaseLabel(0, "export")).toBe("preparing");
+ expect(getProgressPhaseLabel(0.33, "export")).toBe("exporting content");
+ expect(getProgressPhaseLabel(0.66, "export")).toBe("pushing");
+ expect(getProgressPhaseLabel(1, "export")).toBe("pushing");
+ });
+ });
+
describe("isTableChildModel", () => {
it("should return true for field model", () => {
expect(isTableChildModel("field")).toBe(true);
diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.ts
index 6564e2b2ce6..f2321e216ca 100644
--- a/frontend/src/metabase-types/api/remote-sync.ts
+++ b/frontend/src/metabase-types/api/remote-sync.ts
@@ -217,6 +217,13 @@ export type RemoteSyncOutcome =
| { kind: "push-skipped" }
| { kind: "merged"; pulled: number; pushed: number; branch: string };
+export type RemoteSyncTaskUser = {
+ id: UserId;
+ first_name: string | null;
+ last_name: string | null;
+ email: string;
+};
+
export type RemoteSyncTask = {
id: number;
sync_task_type: RemoteSyncTaskType;
@@ -225,9 +232,12 @@ export type RemoteSyncTask = {
started_at: string | null;
ended_at: string | null;
last_progress_report_at: string | null;
+ last_heartbeat_at?: string | null;
error_message: string | null;
outcome?: RemoteSyncOutcome | null;
initiated_by: UserId;
+ /** Absent on auto-imports, which have no initiating user. */
+ initiated_by_user?: RemoteSyncTaskUser | null;
conflicts?: string[];
};
diff --git a/resources/migrations/065/20260915_remote_sync_task_last_heartbeat_at.yaml b/resources/migrations/065/20260915_remote_sync_task_last_heartbeat_at.yaml
new file mode 100644
index 00000000000..b3ac919ad02
--- /dev/null
+++ b/resources/migrations/065/20260915_remote_sync_task_last_heartbeat_at.yaml
@@ -0,0 +1,25 @@
+## Add remote_sync_task.last_heartbeat_at: stamped every few seconds by the thread running the task so a
+## stale value means the owning process died, independent of how long the current phase stays silent.
+## last_progress_report_at keeps meaning "progress last changed", so a live-but-hung task stays
+## distinguishable from a dead one (GHY-4593).
+
+databaseChangeLog:
+
+ - changeSet:
+ id: v65.2026-09-15T00:00:00
+ author: tplude
+ comment: Add remote_sync_task.last_heartbeat_at, stamped periodically by the running thread; null on rows written before the column existed or by a worker that died before its first beat
+ changes:
+ - addColumn:
+ tableName: remote_sync_task
+ columns:
+ - column:
+ name: last_heartbeat_at
+ type: ${timestamp_type}
+ remarks: Heartbeat timestamp stamped periodically while the task runs; a stale value means the owning process died. Null before the first beat.
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: remote_sync_task
+ columnName: last_heartbeat_at