The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Prevent captured-branch race in remote-sync (GHY-3505) (#73796)
Eric Normanda0666118bd40
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.cljindex 734a911fef9..936e9a61f91 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj@@ -207,13 +207,11 @@_query{:keys [name]} :- [:map [:name ms/NonBlankString]]](api/check-superuser)- (let [base-branch (or (remote-sync.task/last-version) (settings/remote-sync-branch))- source (source/source-from-settings)]- (api/check-400 source "Source not configured")+ (let [base-branch (or (remote-sync.task/last-version) (settings/remote-sync-branch))]+ (api/check-400 (source/source-from-settings) "Source not configured")(api/check-400 base-branch "Base commit not found")(try- (source.p/create-branch source name base-branch)- (settings/remote-sync-branch! name)+ (impl/create-branch! name base-branch)(events/publish-event! :event/remote-sync-create-branch{:details {:branch_name name:base_branch base-branch}@@ -234,21 +232,19 @@[:message ms/NonBlankString]]](api/check-superuser)(api/check-400 (= (settings/remote-sync-type) :read-write) "Stash is only allowed when remote-sync-type is set to 'read-write'")- (let [source (source/source-from-settings)]- (api/check-400 source "Source not configured")- (try- (source.p/create-branch source new-branch (settings/remote-sync-branch))- (let [{task-id :id :as task} (impl/async-export! new-branch false message)]- (events/publish-event! :event/remote-sync-stash- {:object task- :details {:branch new-branch}- :user-id api/*current-user-id*})- {:status "success"- :message (str "Stashing to " new-branch)- :task_id task-id})- (catch Exception e- (throw (ex-info (format "Failed to stash changes to branch: %s" (ex-message e))- {:status-code 400}))))))+ (api/check-400 (source/source-from-settings) "Source not configured")+ (try+ (let [{task-id :id :as task} (impl/stash! new-branch message)]+ (events/publish-event! :event/remote-sync-stash+ {:object task+ :details {:branch new-branch}+ :user-id api/*current-user-id*})+ {:status "success"+ :message (str "Stashing to " new-branch)+ :task_id task-id})+ (catch Exception e+ (throw (ex-info (format "Failed to stash changes to branch: %s" (ex-message e))+ {:status-code 400})))))(def ^{:arglists '([request respond raise])} routes"`/api/ee/remote-sync` routes."diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/core.cljindex 18919622f1f..f89dde6835c 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.remote-sync.core(:require+ [metabase-enterprise.remote-sync.guards :as guards][metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.source.protocol :as source.p]@@ -91,6 +92,7 @@"Sets remote sync to true/false on one or collections in a single transaction. Checks that the remote sync stateafterwards is consistent in terms of dependency rules. Collections are provided as a map of collection-id -> sync state."[collection-states :- [:map-of pos-int? :boolean]]+ (guards/ensure-no-active-task!)(let [{:keys [sync-on sync-off]} (-> (reduce-kv (fn [sync-states collection-id sync-state](if sync-state(update sync-states :sync-on conj collection-id)diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/guards.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/guards.cljnew file mode 100644index 00000000000..f07c59b99ab--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/guards.clj@@ -0,0 +1,35 @@+(ns metabase-enterprise.remote-sync.guards+ "Guard predicates used by mutating remote-sync operations to refuse running while+ another task is in flight, and to opportunistically clean up stale rows so a JVM+ crash or hung thread doesn't block the system permanently.++ Lives in its own namespace because both `impl.clj` and `settings.clj` need to call+ it, and they do not currently share a common ancestor that could host it without+ creating a circular dependency."+ (:require+ [metabase-enterprise.remote-sync.models.remote-sync-task :as rst]))++(defn task-running?+ "Returns true if a remote-sync task is currently *active* — started, not ended, and has+ reported progress recently (within `remote-sync-task-time-limit-ms`). Delegates to+ `current-task`, so a stale task (no progress reports for longer than the timeout) does+ NOT count as running. Stale rows get cleaned up by `ensure-no-active-task!`'s supersession+ step, which means user-driven operations self-heal from JVM crashes and hung threads+ the same way auto-import does."+ []+ (some? (rst/current-task)))++(defn ensure-no-active-task!+ "Throws an ex-info with status-code 400 if a remote-sync task is currently active. After+ the check passes, calls `supersede-stale-tasks!` to mark any stale rows as cancelled ++ terminated so they don't interfere with the operation about to run.++ Used as a guard at the top of mutating remote-sync operations. Combined with+ `handle-task-result!`'s already-terminated check, this means an old task's late-arriving+ thread will detect its row is terminated and exit without writing the setting or+ overwriting bookkeeping."+ []+ (when (task-running?)+ (throw (ex-info "Remote sync task in progress"+ {:status-code 400})))+ (rst/supersede-stale-tasks!))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 8f1db83a234..f4215fcf234 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -3,6 +3,7 @@[clojure.string :as str][diehard.core :as dh][java-time.api :as t]+ [metabase-enterprise.remote-sync.guards :as guards][metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object][metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.settings :as settings]@@ -220,18 +221,36 @@{:conflicts (vec all-conflicts):summary (into #{} (map :category) all-conflicts)}))+(defn- branch-changed-since-scheduling?+ "Returns true if `pre-task-branch` was captured by the async-* function and the+ `remote-sync-branch` setting has since drifted to a different value. Used as a+ defense-in-depth check against any future code path that bypasses the operation-level+ guards and mutates the setting between scheduling and the work running."+ [pre-task-branch]+ (and (some? pre-task-branch)+ (not= pre-task-branch (settings/remote-sync-branch))))+(defn import!"Imports and reloads Metabase entities from a remote snapshot.Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:- :force? - forces import even when the snapshot version matches the last imported version+ - :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs+ from the current setting at task start, the import aborts with `:error` to protect data+ integrity (the load mutates the app DB, so we refuse to proceed when state has drifted)Loads serialized entities, removes entities not in the import, syncs the remote-sync-object table, andoptionally creates a remote-synced collection.Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may bethrown during import and are caught and converted to error status maps."- [^SourceSnapshot snapshot task-id & {:keys [force?]}]+ [^SourceSnapshot snapshot task-id & {:keys [force? pre-task-branch]}]+ (when (branch-changed-since-scheduling? pre-task-branch)+ (log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"+ pre-task-branch (settings/remote-sync-branch))+ (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"+ {:pre-task-branch pre-task-branch+ :current-branch (settings/remote-sync-branch)})))(log/info "Reloading remote entities from the remote source")(analytics/inc! :metabase-remote-sync/imports)(let [sync-timestamp (t/instant)]@@ -296,13 +315,24 @@(defn export!"Exports remote-synced collections to a remote source repository.- Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and a commit message string. Extracts all- remote-synced collections, serializes their content, writes the files to the source, and updates all- RemoteSyncObject statuses to 'synced'.+ Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, a commit message string, and+ optional keyword arguments:+ - :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs+ from the current setting at task start, the export aborts with `:error` (defense-in-depth+ against any path that mutates the setting between scheduling and the work running).++ Extracts all remote-synced collections, serializes their content, writes the files to the source, and+ updates all RemoteSyncObject statuses to 'synced'.Returns a map with :status (either :success or :error), :version, and optionally :message keys. Variousexceptions may be thrown during export and are caught and converted to error status maps."- [^SourceSnapshot snapshot task-id message]+ [^SourceSnapshot snapshot task-id message & {:keys [pre-task-branch]}]+ (when (branch-changed-since-scheduling? pre-task-branch)+ (log/warnf "Aborting export: remote-sync-branch changed from %s to %s since task was scheduled"+ pre-task-branch (settings/remote-sync-branch))+ (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"+ {:pre-task-branch pre-task-branch+ :current-branch (settings/remote-sync-branch)})))(if snapshot(let [sync-timestamp (t/instant)](try@@ -337,13 +367,21 @@"Takes a cluster-wide lock and either returns an existing in-progress RemoteSyncTask ID or creates a new one.Takes a task-type string (either 'import' or 'export'). Returns a RemoteSyncTask with an optional :existing? key.- If a task is already running, returns (assoc existing-task :existing? true). Otherwise creates a new task and- returns it."+ If a task is already running (per `current-task`, which uses the staleness window), returns+ `(assoc existing-task :existing? true)`. Otherwise — i.e., no current task, but stale rows might still+ be hanging around — calls `supersede-stale-tasks!` to mark them terminated, then creates a new task.++ Used directly by the auto-import Quartz job, so auto-imports self-heal after a stale task. User-driven+ endpoints go through `ensure-no-active-task!` first (in `async-import!` / `async-export!` / etc.), which+ uses the stricter `task-running?` predicate and refuses if any task — including stale — is alive. So+ this function only reaches the supersession branch on the auto-import path."[task-type](cluster-lock/with-cluster-lock ::remote-sync-task(if-let [task (remote-sync.task/current-task)](assoc task :existing? true)- (remote-sync.task/create-sync-task! task-type api/*current-user-id*))))+ (do+ (remote-sync.task/supersede-stale-tasks!)+ (remote-sync.task/create-sync-task! task-type api/*current-user-id*)))));;; ------------------------------------------- Remote Changes Check -------------------------------------------@@ -417,20 +455,44 @@RemoteSyncTask ID, and an optional branch name. On success, updates the remote-sync-branch setting (if branchprovided), marks the task complete, and invalidates the remote changes cache. On conflict, sets the version andstores the conflicts. On error, marks the task as failed with the error message. For any other status, marks the- task as failed with 'Unexpected Error'."+ task as failed with 'Unexpected Error'.++ If the task has already been terminated (`ended_at` is set, e.g., because an admin cancelled it+ via POST /current-task/cancel while the virtual thread was still running), this function logs a+ warning and returns without writing anything. This prevents a still-running thread from clobbering+ the cancellation bookkeeping or stomping the branch setting via its captured value.++ The read and the subsequent write happen in a single transaction with `SELECT ... FOR UPDATE` so+ a concurrent cancel cannot slip in between the terminated-check and the result-write."[result task-id & [branch]]- (case (:status result)- :success (do- (t2/with-transaction [_conn]- (when branch- (settings/remote-sync-branch! branch))- (remote-sync.task/complete-sync-task! task-id))- (invalidate-remote-changes-cache!))- :conflict (do- (remote-sync.task/set-version! task-id (:version result))- (remote-sync.task/conflict-sync-task! task-id (:conflicts result)))- :error (remote-sync.task/fail-sync-task! task-id (:message result))- (remote-sync.task/fail-sync-task! task-id "Unexpected Error")))+ (let [proceed?+ (t2/with-transaction [_conn]+ (let [task (t2/select-one :model/RemoteSyncTask :id task-id {:for :update})]+ (cond+ (nil? task)+ (do (log/warnf "Task %s missing during result handling; skipping" task-id)+ false)++ (some? (:ended_at task))+ (do (log/warnf "Task %s already terminated (ended_at=%s); skipping result handling to preserve state"+ task-id (:ended_at task))+ false)++ :else+ (do+ (case (:status result)+ :success (do+ (when branch+ (settings/remote-sync-branch! branch))+ (remote-sync.task/complete-sync-task! task-id))+ :conflict (do+ (remote-sync.task/set-version! task-id (:version result))+ (remote-sync.task/conflict-sync-task! task-id (:conflicts result)))+ :error (remote-sync.task/fail-sync-task! task-id (:message result))+ (remote-sync.task/fail-sync-task! task-id "Unexpected Error"))+ true))))]+ (when (and proceed? (= (:status result) :success))+ (invalidate-remote-changes-cache!))))(defn- run-async!"Executes a remote sync task asynchronously in a virtual thread.@@ -466,13 +528,20 @@Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if thereare unsaved changes and force? is false."[branch force? import-args]- (let [source (source/source-from-settings branch)- has-dirty? (remote-sync.object/dirty?)]+ (guards/ensure-no-active-task!)+ (let [pre-task-branch (settings/remote-sync-branch)+ source (source/source-from-settings branch)+ has-dirty? (remote-sync.object/dirty?)](when (and has-dirty? (not force?))(throw (ex-info "There are unsaved changes in the Remote Sync collection which will be overwritten by the import. Force the import to discard these changes."{:status-code 400:conflicts true})))- (run-async! "import" branch (fn [task-id] (import! (source.p/snapshot source) task-id (assoc import-args :force? force?))))))+ (run-async! "import" branch+ (fn [task-id]+ (import! (source.p/snapshot source) task-id+ (assoc import-args+ :force? force?+ :pre-task-branch pre-task-branch))))))(defn async-export!"Exports the remote-synced collections to the remote source repository asynchronously.@@ -484,15 +553,38 @@Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if thereare new remote changes and force? is false."[branch force? message]- (let [source (source/source-from-settings branch)- last-task-version (remote-sync.task/last-version)- snapshot (source.p/snapshot source)+ (guards/ensure-no-active-task!)+ (let [pre-task-branch (settings/remote-sync-branch)+ source (source/source-from-settings branch)+ last-task-version (remote-sync.task/last-version)+ snapshot (source.p/snapshot source)current-source-version (source.p/version snapshot)](when (and (not force?) (some? last-task-version) (not= last-task-version current-source-version))(throw (ex-info "Cannot export changes that will overwrite new changes in the branch."{:status-code 400:conflicts true})))- (run-async! "export" branch (fn [task-id] (export! snapshot task-id message)))))+ (run-async! "export" branch+ (fn [task-id]+ (export! snapshot task-id message :pre-task-branch pre-task-branch)))))++(defn create-branch!+ "Creates a new remote branch from `base-branch` and switches `remote-sync-branch`+ to the new name. Does not publish events or return a response map; the caller+ is responsible for those concerns."+ [name base-branch]+ (guards/ensure-no-active-task!)+ (let [source (source/source-from-settings)]+ (source.p/create-branch source name base-branch)+ (settings/remote-sync-branch! name)))++(defn stash!+ "Creates a new remote branch from the current `remote-sync-branch` and starts an+ async export to it. Returns the resulting RemoteSyncTask. Does not publish events."+ [new-branch message]+ (guards/ensure-no-active-task!)+ (let [source (source/source-from-settings)]+ (source.p/create-branch source new-branch (settings/remote-sync-branch))+ (async-export! new-branch false message)))(defn finish-remote-config!"Based on the current configuration, fill in any missing settings and finalize remote sync setup.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.cljindex bf2b6fcb8f4..fdb754b8094 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj@@ -2,8 +2,8 @@"Model for tracking remote sync tasks and their progress."(:require[java-time.api :as t]- [metabase-enterprise.remote-sync.settings :as settings][metabase.models.interface :as mi]+ [metabase.settings.core :as setting][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][methodical.core :as methodical]@@ -126,12 +126,38 @@[:<> :started_at nil][:= :ended_at nil][:<- (t/minus (t/offset-date-time) (t/millis (settings/remote-sync-task-time-limit-ms)))+ (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms))):last_progress_report_at]]:limit 1:order-by [[:started_at :desc][:id :desc]]}))+(defn supersede-stale-tasks!+ "Marks any genuinely stale task rows as cancelled and terminated.++ A task is considered stale if it has `started_at` set, `ended_at` nil, and `last_progress_report_at`+ is older than `remote-sync-task-time-limit-ms`. The DB schema requires `last_progress_report_at`+ to be non-null with a default of `current_timestamp`, so a brand-new task always has a recent+ value (set on insert) and is not considered stale.++ Called from `create-task-with-lock!` before creating a new task, to clean up rows whose owning+ JVM/thread is gone or hung. Returns nothing meaningful.++ Combined with `handle-task-result!`'s already-terminated check, this means a stale task's thread+ that eventually wakes up and tries to complete will detect that its row is terminated and exit+ without writing the setting or overwriting bookkeeping."+ []+ (let [cutoff (t/minus (t/offset-date-time)+ (t/millis (setting/get :remote-sync-task-time-limit-ms)))]+ (t2/query {:update (t2/table-name :model/RemoteSyncTask)+ :set {:cancelled true+ :ended_at (mi/now)+ :error_message "Superseded after staleness timeout"}+ :where [:and+ [:<> :started_at nil]+ [:= :ended_at nil]+ [:< :last_progress_report_at cutoff]]})))+(defn most-recent-task"Gets the most recently run task, including currently running tasks.@@ -209,7 +235,7 @@[task](and (nil? (:ended_at task))(t/< (:last_progress_report_at task)- (t/minus (t/offset-date-time) (t/millis (settings/remote-sync-task-time-limit-ms))))))+ (t/minus (t/offset-date-time) (t/millis (setting/get :remote-sync-task-time-limit-ms))))))(defn conflict?"Checks if a task ended with conflicts.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex 34a5359a7ee..bdbf418d186 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][java-time.api :as t]+ [metabase-enterprise.remote-sync.guards :as guards][metabase-enterprise.remote-sync.source.git :as git][metabase.collections.models.collection :as collection][metabase.settings.core :as setting :refer [defsetting]]@@ -205,6 +206,7 @@Throws ExceptionInfo if the git settings are invalid or if unable to connect to the repository."[{:keys [remote-sync-url remote-sync-token] :as settings}]+ (guards/ensure-no-active-task!)(let [git-related-keys #{:remote-sync-url :remote-sync-token :remote-sync-type :remote-sync-branch}updating-git-settings? (some git-related-keys (keys settings))env-set-url (= :env (setting/get-raw-value-source :remote-sync-url))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex 95b3f777a05..75f1f835def 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -3,6 +3,7 @@(:require[clojure.test :refer :all][diehard.core :as dh]+ [java-time.api :as t][metabase-enterprise.remote-sync.core :as remote-sync.core][metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]@@ -140,7 +141,7 @@remote-sync-token "test-token"remote-sync-branch "main"](with-redefs [source/source-from-settings (constantly mock-source)]- (is (= "Remote sync in progress"+ (is (= "Remote sync task in progress"(mt/user-http-request :crowberto :post 400 "ee/remote-sync/import" {})))))))))(deftest import-errors-when-dirty-changes-test@@ -245,7 +246,7 @@remote-sync-token "test-token"remote-sync-branch "main"](with-redefs [source/source-from-settings (constantly mock-source)]- (is (= "Remote sync in progress"+ (is (= "Remote sync task in progress"(mt/user-http-request :crowberto :post 400 "ee/remote-sync/export" {}))))))))))(deftest export-errors-if-external-changes-test@@ -558,6 +559,32 @@:remote-sync-url "https://github.com/test/repo.git":remote-sync-token "test-token"}))))))))+(deftest settings-refuses-while-task-running-test+ (testing "PUT /api/ee/remote-sync/settings refuses with 400 when a RemoteSyncTask is active"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :progress 0.0}]+ (let [check-git-call-count (atom 0)]+ (with-redefs [settings/check-git-settings! (fn [_] (swap! check-git-call-count inc) true)]+ (mt/with-temporary-setting-values [remote-sync-url "file://my/repo.git"+ remote-sync-token nil+ remote-sync-type :read-only+ remote-sync-branch "main"]+ (let [response (mt/user-http-request :crowberto :put 400 "ee/remote-sync/settings"+ {:remote-sync-url "file://different.git"+ :remote-sync-type :read-only+ :remote-sync-branch "feature-x"+ :remote-sync-token nil})]+ (is (= "Remote sync task in progress" (:message response))+ "endpoint should return the guard's error message"))+ (is (= "file://my/repo.git" (settings/remote-sync-url))+ "remote-sync-url must remain unchanged when the guard fires")+ (is (= "main" (settings/remote-sync-branch))+ "remote-sync-branch must remain unchanged when the guard fires")+ (is (zero? @check-git-call-count)+ "check-git-settings! must not be called when the guard fires")))))))+;;; ------------------------------------------- Settings Collections Tests -------------------------------------------(deftest settings-collections-enables-remote-sync-on-single-collection-test@@ -1104,3 +1131,94 @@(is (= 1 (count dirty-items)) "Transforms root should be returned even when setting is disabled")(is (= "Transforms" (:name (first dirty-items))))(is (= "delete" (:sync_status (first dirty-items))))))))))++;; ---------- API-level guard sweep --------------------------------------------------------------+;;+;; Boundary tests verifying that every mutating remote-sync HTTP endpoint surfaces the guard's+;; refusal as a 400 response with `Remote sync task in progress`. Companions to the operation-level+;; tests; this catches the case where an endpoint is edited to bypass the guarded function.++(deftest import-refuses-while-task-running-test+ (testing "POST /api/ee/remote-sync/import returns 400 when a RemoteSyncTask is active"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :progress 0.0}]+ (let [mock-source (test-helpers/create-mock-source)+ tasks-before (t2/count :model/RemoteSyncTask)]+ (with-redefs [source/source-from-settings (constantly mock-source)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ (is (= "Remote sync task in progress"+ (mt/user-http-request :crowberto :post 400 "ee/remote-sync/import" {})))+ (is (= tasks-before (t2/count :model/RemoteSyncTask))+ "no NEW RemoteSyncTask row should be created when the guard fires")))))))++(deftest export-refuses-while-task-running-test+ (testing "POST /api/ee/remote-sync/export returns 400 when a RemoteSyncTask is active"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :progress 0.0}]+ (let [mock-source (test-helpers/create-mock-source)+ tasks-before (t2/count :model/RemoteSyncTask)]+ (with-redefs [source/source-from-settings (constantly mock-source)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ (is (= "Remote sync task in progress"+ (mt/user-http-request :crowberto :post 400 "ee/remote-sync/export"+ {:message "test export"})))+ (is (= tasks-before (t2/count :model/RemoteSyncTask))+ "no NEW RemoteSyncTask row should be created when the guard fires")))))))++(deftest create-branch-refuses-while-task-running-test+ (testing "POST /api/ee/remote-sync/create-branch returns 400 when a RemoteSyncTask is active"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :progress 0.0}]+ (let [mock-source (test-helpers/create-mock-source)+ initial-branches @(:branches-atom mock-source)]+ (with-redefs [source/source-from-settings (constantly mock-source)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ ;; The endpoint wraps the guard's exception with "Failed to create branch: ..." in its catch.+ (is (re-find #"Remote sync task in progress"+ (str (mt/user-http-request :crowberto :post 400 "ee/remote-sync/create-branch"+ {:name "feature-x"}))))+ (is (= "main" (settings/remote-sync-branch))+ "remote-sync-branch must remain unchanged when the guard fires")+ (is (= initial-branches @(:branches-atom mock-source))+ "no new branch should be pushed to the source when the guard fires")))))))++(deftest stash-refuses-while-task-running-test+ (testing "POST /api/ee/remote-sync/stash returns 400 when a RemoteSyncTask is active"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :progress 0.0}]+ (let [mock-source (test-helpers/create-mock-source)+ initial-branches @(:branches-atom mock-source)+ tasks-before (t2/count :model/RemoteSyncTask)]+ (with-redefs [source/source-from-settings (constantly mock-source)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ ;; The endpoint wraps the guard's exception with "Failed to stash changes to branch: ..." in its catch.+ (is (re-find #"Remote sync task in progress"+ (str (mt/user-http-request :crowberto :post 400 "ee/remote-sync/stash"+ {:new_branch "stash-branch"+ :message "stash msg"}))))+ (is (= "main" (settings/remote-sync-branch))+ "remote-sync-branch must remain unchanged when the guard fires")+ (is (= initial-branches @(:branches-atom mock-source))+ "no new branch should be pushed to the source when the guard fires")+ (is (= tasks-before (t2/count :model/RemoteSyncTask))+ "no NEW RemoteSyncTask row should be created when the guard fires")))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.cljindex 30ad2829195..6d1ef16838f 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/core_test.clj@@ -3,6 +3,7 @@(:require[clojure.test :refer :all][metabase-enterprise.remote-sync.core :as core]+ [metabase-enterprise.remote-sync.guards :as guards][metabase.collections.test-utils :refer [with-library-synced with-library-not-synced]][metabase.events.core :as events][metabase.test :as mt]@@ -304,3 +305,19 @@(testing "unknown model returns false for all instances"(is (= {1 false, 2 false}(core/batch-model-eligible? :model/UnknownModel [{:id 1} {:id 2}])))))++;; ---------- Guard contract for bulk-set-remote-sync ---------------------------------------------+;;+;; bulk-set-remote-sync consults `guards/task-running?` and refuses if a task is in flight.++(deftest bulk-set-remote-sync-refuses-while-task-running-test+ (testing "bulk-set-remote-sync must refuse when guards/task-running? returns true,+ without changing the collection's is_remote_synced flag"+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"+ :location "/"+ :is_remote_synced false}]+ (with-redefs [guards/task-running? (constantly true)]+ (is (thrown-with-msg? Exception #"Remote sync task in progress"+ (core/bulk-set-remote-sync {coll-id true})))+ (is (false? (:is_remote_synced (t2/select-one :model/Collection :id coll-id)))+ "collection's is_remote_synced flag must remain unchanged when the guard fires")))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/guards_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/guards_test.cljnew file mode 100644index 00000000000..a9d4a4df932--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/guards_test.clj@@ -0,0 +1,66 @@+(ns metabase-enterprise.remote-sync.guards-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.remote-sync.guards :as guards]+ [metabase-enterprise.remote-sync.test-helpers :as test-helpers]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))+(use-fixtures :each test-helpers/clean-task-table)++(deftest task-running?-returns-false-when-no-tasks-test+ (testing "task-running? returns false when there are no RemoteSyncTask rows"+ (is (false? (guards/task-running?)))))++(deftest task-running?-returns-true-while-task-is-active-test+ (testing "task-running? returns true while a RemoteSyncTask is started, not ended, and reporting+ progress recently"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :last_progress_report_at (t/offset-date-time)+ :progress 0.0}]+ (is (true? (guards/task-running?))))))++(deftest task-running?-returns-false-when-task-has-ended-test+ (testing "task-running? returns false once the RemoteSyncTask has ended_at set"+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :ended_at (t/offset-date-time)+ :progress 1.0}]+ (is (false? (guards/task-running?))))))++(deftest task-running?-returns-false-for-stalled-tasks-test+ (testing "task-running? returns false for tasks that haven't reported progress within the+ timeout window — stale rows should not block operations; ensure-no-active-task!+ cleans them up via supersede-stale-tasks! after the check passes"+ (let [two-hours-ago (t/minus (t/offset-date-time) (t/hours 2))]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at two-hours-ago+ :last_progress_report_at two-hours-ago+ :progress 0.5}]+ (is (false? (guards/task-running?))+ "stalled task must not block operations")))))++(deftest ensure-no-active-task!-supersedes-stale-tasks-test+ (testing "ensure-no-active-task! does not throw for stale tasks AND supersedes them so+ user-driven operations self-heal from JVM-died/stuck-thread orphans"+ (let [two-hours-ago (t/minus (t/offset-date-time) (t/hours 2))+ stale-task (t2/insert-returning-instance!+ :model/RemoteSyncTask+ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at two-hours-ago+ :last_progress_report_at two-hours-ago+ :progress 0.5})]+ (guards/ensure-no-active-task!)+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id stale-task))]+ (is (true? (:cancelled after))+ "stale task must be marked cancelled by ensure-no-active-task!")+ (is (some? (:ended_at after))+ "stale task must be terminated by ensure-no-active-task!")))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 18fef734073..200a6f613b1 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -4,6 +4,7 @@[clojure.string :as str][clojure.test :refer :all][java-time.api :as t]+ [metabase-enterprise.remote-sync.guards :as guards][metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.settings :as remote-sync.settings]@@ -1434,3 +1435,161 @@ serdes/meta:"import should succeed with old-format paths")(is (t2/exists? :model/Collection :entity_id coll-eid)"collection should have been imported from old-format path")))))++;; ---------- GHY-3505: captured-branch race ---------------------------------------------------------+;;+;; The original repros for GHY-3505 demonstrated the captured-branch race by writing the setting+;; directly via `setting/set!`, simulating a path that bypassed all coordination. With the+;; operation-level guards in place (see *-refuses-while-task-running-test below and the+;; corresponding API-level tests in api_test.clj), the race is no longer reachable through any+;; user-facing path: every code that mutates remote-sync state now calls `guards/ensure-no-active-task!`+;; first and refuses with 400 if a RemoteSyncTask is in flight.+;;+;; Coverage of the fix is split across:+;; - guards-test (the predicate itself: catches stalled tasks too)+;; - operation-level guard tests in this file, core_test, settings_test+;; - API-level guard tests in api_test+;;+;; As a defense-in-depth measure for any future code path that bypasses the guards, async-import!+;; and async-export! capture the setting at scheduling time and the work function aborts if it+;; observes a different setting at start. This is not separately tested because the window in+;; which the setting could change between scheduling and start is sub-millisecond.++;; ---------- Guard contract: mutating operations must refuse while a task is running ----------------+;;+;; Every mutating remote-sync operation consults `guards/task-running?` and refuses if it returns+;; true. The tests use `with-redefs` to flip it to `(constantly true)` so they don't depend on+;; actually inserting a RemoteSyncTask row.++(deftest async-import!-refuses-while-task-running-test+ (testing "async-import! must refuse when guards/task-running? returns true,+ without creating a RemoteSyncTask row"+ (mt/with-temporary-setting-values [remote-sync-enabled true+ remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ (with-redefs [guards/task-running? (constantly true)+ source/source-from-settings (fn [& _] (test-helpers/create-mock-source))]+ (is (thrown-with-msg? Exception #"Remote sync task in progress"+ (impl/async-import! "main" true {})))+ (is (zero? (t2/count :model/RemoteSyncTask))+ "no RemoteSyncTask row should be created when the guard fires")))))++(deftest async-export!-refuses-while-task-running-test+ (testing "async-export! must refuse when guards/task-running? returns true,+ without creating a RemoteSyncTask row"+ (mt/with-temporary-setting-values [remote-sync-enabled true+ remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ (with-redefs [guards/task-running? (constantly true)+ source/source-from-settings (fn [& _] (test-helpers/create-mock-source))]+ (is (thrown-with-msg? Exception #"Remote sync task in progress"+ (impl/async-export! "main" true "msg")))+ (is (zero? (t2/count :model/RemoteSyncTask))+ "no RemoteSyncTask row should be created when the guard fires")))))++(deftest create-branch!-refuses-while-task-running-test+ (testing "create-branch! must refuse when guards/task-running? returns true,+ without pushing a branch to the source or changing the setting"+ (mt/with-temporary-setting-values [remote-sync-enabled true+ remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ (let [mock-source (test-helpers/create-mock-source)+ initial-branches @(:branches-atom mock-source)]+ (with-redefs [guards/task-running? (constantly true)+ source/source-from-settings (fn [& _] mock-source)]+ (is (thrown-with-msg? Exception #"Remote sync task in progress"+ (impl/create-branch! "feature-x" "main")))+ (is (= "main" (remote-sync.settings/remote-sync-branch))+ "remote-sync-branch must remain unchanged when the guard fires")+ (is (= initial-branches @(:branches-atom mock-source))+ "no new branch should be pushed to the source when the guard fires"))))))++(deftest stash!-refuses-while-task-running-test+ (testing "stash! must refuse when guards/task-running? returns true,+ without pushing a branch to the source, creating a task, or changing the setting"+ (mt/with-temporary-setting-values [remote-sync-enabled true+ remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "token"+ remote-sync-branch "main"+ remote-sync-type :read-write]+ (let [mock-source (test-helpers/create-mock-source)+ initial-branches @(:branches-atom mock-source)]+ (with-redefs [guards/task-running? (constantly true)+ source/source-from-settings (fn [& _] mock-source)]+ (is (thrown-with-msg? Exception #"Remote sync task in progress"+ (impl/stash! "stash-branch" "stash message")))+ (is (= "main" (remote-sync.settings/remote-sync-branch))+ "remote-sync-branch must remain unchanged when the guard fires")+ (is (= initial-branches @(:branches-atom mock-source))+ "no new branch should be pushed to the source when the guard fires")+ (is (zero? (t2/count :model/RemoteSyncTask))+ "no RemoteSyncTask row should be created when the guard fires"))))))++;; ---------- handle-task-result! is robust against double-handling -----------------------------+;;+;; If an admin cancels a task via POST /current-task/cancel while its virtual thread is still+;; running, the thread will eventually reach handle-task-result!. Without protection, the :success+;; path would write the captured branch (stomping any setting change since cancellation) and+;; complete-sync-task! would overwrite the cancellation bookkeeping. The check at the top of+;; handle-task-result! short-circuits when the task is already terminated.++(deftest handle-task-result!-skips-already-terminated-task-test+ (testing "handle-task-result! must not write the setting or update the task row when the task+ is already terminated (e.g., cancelled by admin while the virtual thread was running)"+ (mt/with-temporary-setting-values [remote-sync-branch "dev"]+ (let [task-id (t2/insert-returning-pk!+ :model/RemoteSyncTask+ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time)+ :ended_at (t/offset-date-time)+ :cancelled true+ :error_message "Cancelled by admin"+ :progress 0.5})]+ (impl/handle-task-result! {:status :success :version "abc"} task-id "feature-x")+ (is (= "dev" (remote-sync.settings/remote-sync-branch))+ "setting must remain unchanged — already-terminated task must not write it")+ (let [task-after (t2/select-one :model/RemoteSyncTask :id task-id)]+ (is (true? (:cancelled task-after))+ "cancellation bookkeeping must be preserved")+ (is (= "Cancelled by admin" (:error_message task-after))+ "error message must be preserved")+ (is (= 0.5 (double (:progress task-after)))+ "progress must not be overwritten to 1.0"))))))++;; ---------- create-task-with-lock! integrates supersession ------------------------------------+;;+;; The auto-import path (task/import.clj) calls create-task-with-lock! directly, without going+;; through the strict ensure-no-active-task! guard. So when an auto-import runs and finds the+;; previous task to be stale, supersede-stale-tasks! cleans up the stale row before inserting+;; the new one. This is the self-healing behavior we want for automatic operations.++(deftest create-task-with-lock!-supersedes-stale-tasks-test+ (testing "create-task-with-lock! marks stale tasks superseded before creating a new task,+ so auto-imports recover after a JVM/thread death"+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))+ stale-task (t2/insert-returning-instance!+ :model/RemoteSyncTask+ {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :started_at old-time+ :last_progress_report_at old-time+ :progress 0.5})+ new-task (impl/create-task-with-lock! "import")]+ (let [stale-after (t2/select-one :model/RemoteSyncTask :id (:id stale-task))]+ (is (true? (:cancelled stale-after))+ "stale task must be marked cancelled")+ (is (some? (:ended_at stale-after))+ "stale task must be terminated"))+ (is (some? (:id new-task))+ "new task must be created")+ (is (nil? (:ended_at new-task))+ "new task must be active")+ (is (not (:existing? new-task))+ "new task must not be marked as existing"))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.cljindex a7c4973b5d6..ae4445bac61 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_task_test.clj@@ -147,7 +147,8 @@(testing "hydrates status"(let [task (rst/create-sync-task! "import" (mt/user->id :rasta))hydrated-task (t2/hydrate task :status)]- (is (= :running (:status hydrated-task))))))+ (is (= :running (:status hydrated-task)))+ (rst/complete-sync-task! (:id task)))));;; ------------------------------------------------------------------------------------------------;;; Tests for Edge Cases and Error Handling@@ -392,3 +393,78 @@(rst/complete-sync-task! (:id new-task))(rst/set-version! (:id new-task) "version 2")(is (= "version 2" (rst/last-version))))))))++;;; ------------------------------------------------------------------------------------------------+;;; Tests for supersede-stale-tasks!+;;; ------------------------------------------------------------------------------------------------+;;;+;;; Used by `create-task-with-lock!` (auto-import path) to clean up rows whose owning JVM/thread+;;; is gone or hung. Must NOT supersede brand-new tasks that haven't reported progress yet.++(defn- insert-task!+ "Helper: insert a RemoteSyncTask row with the given fields, bypassing the create-sync-task!+ helper so we can set arbitrary timestamps for staleness testing."+ [fields]+ (t2/insert-returning-instance! :model/RemoteSyncTask+ (merge {:sync_task_type "import"+ :initiated_by (mt/user->id :rasta)+ :progress 0}+ fields)))++(deftest supersede-stale-tasks!-marks-stale-task-cancelled-test+ (testing "supersede-stale-tasks! marks a stale task (last_progress_report_at older than the+ timeout) as cancelled and terminated"+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))+ stale-task (insert-task! {:started_at old-time+ :last_progress_report_at old-time+ :progress 0.5})]+ (rst/supersede-stale-tasks!)+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id stale-task))]+ (is (true? (:cancelled after)))+ (is (some? (:ended_at after)))+ (is (= "Superseded after staleness timeout" (:error_message after)))))))++(deftest supersede-stale-tasks!-leaves-brand-new-tasks-alone-test+ (testing "supersede-stale-tasks! must NOT mark a brand-new task that just inserted —+ protects against superseding tasks created moments ago by another node in the+ cluster (the DB defaults last_progress_report_at to current_timestamp on insert)"+ (let [now (t/offset-date-time)+ new-task (insert-task! {:started_at now+ :last_progress_report_at now})]+ (rst/supersede-stale-tasks!)+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id new-task))]+ (is (false? (:cancelled after))+ "brand-new task must remain unchanged")+ (is (nil? (:ended_at after))+ "brand-new task must not be terminated"))+ (rst/complete-sync-task! (:id new-task)))))++(deftest supersede-stale-tasks!-leaves-active-tasks-alone-test+ (testing "supersede-stale-tasks! must NOT mark a task that has reported progress recently"+ (let [now (t/offset-date-time)+ old-start (t/minus now (t/hours 1))+ active-task (insert-task! {:started_at old-start+ :last_progress_report_at now+ :progress 0.7})]+ (rst/supersede-stale-tasks!)+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id active-task))]+ (is (false? (:cancelled after))+ "task with recent progress must remain unchanged")+ (is (nil? (:ended_at after))+ "task with recent progress must not be terminated"))+ (rst/complete-sync-task! (:id active-task)))))++(deftest supersede-stale-tasks!-leaves-terminated-tasks-alone-test+ (testing "supersede-stale-tasks! must NOT touch tasks that already have ended_at set"+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))+ ended-task (insert-task! {:started_at old-time+ :ended_at old-time+ :progress 1.0})+ ;; Capture the post-insert ended_at so the comparison is done at the DB's precision+ ;; (Postgres/MySQL truncate nanoseconds to microseconds; H2 keeps full nanos).+ before (t2/select-one :model/RemoteSyncTask :id (:id ended-task))]+ (rst/supersede-stale-tasks!)+ (let [after (t2/select-one :model/RemoteSyncTask :id (:id ended-task))]+ (is (false? (:cancelled after)))+ (is (= (:ended_at before) (:ended_at after))+ "ended_at must not be overwritten")))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.cljindex a2ab049236a..b8677490e3a 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.remote-sync.settings-test(:require[clojure.test :refer :all]+ [metabase-enterprise.remote-sync.guards :as guards][metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.source.git :as git][metabase.collections.models.collection.root :as collection.root]@@ -183,3 +184,31 @@(testing "Root collection for snippets namespace is not remote-synced"(let [root-coll (collection.root/root-collection-with-ui-details :snippets)](is (false? (:is_remote_synced root-coll))))))++;; ---------- Guard contract for settings mutations -----------------------------------------------+;;+;; check-and-update-remote-settings! consults `guards/task-running?` and refuses if a task is+;; in flight.++(deftest check-and-update-remote-settings!-refuses-while-task-running-test+ (testing "check-and-update-remote-settings! must refuse when guards/task-running? returns true,+ without changing any settings or calling git"+ (let [check-git-call-count (atom 0)]+ (with-redefs [guards/task-running? (constantly true)+ settings/check-git-settings! (fn [_] (swap! check-git-call-count inc) true)]+ (mt/with-temporary-setting-values [:remote-sync-url "file://my/repo.git"+ :remote-sync-token nil+ :remote-sync-type :read-only+ :remote-sync-branch "main"]+ (is (thrown-with-msg? Exception #"Remote sync task in progress"+ (settings/check-and-update-remote-settings!+ {:remote-sync-url "file://different.git"+ :remote-sync-type :read-only+ :remote-sync-branch "feature-x"+ :remote-sync-token nil})))+ (is (= "file://my/repo.git" (settings/remote-sync-url))+ "remote-sync-url must remain unchanged when the guard fires")+ (is (= "main" (settings/remote-sync-branch))+ "remote-sync-branch must remain unchanged when the guard fires")+ (is (zero? @check-git-call-count)+ "check-git-settings! must not be called when the guard fires"))))))