Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add /has-remote-changes endpoint to remote-sync API (#67924)

Edward Paget60093c461b22
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
index 775280f2e29..53c983f0e1f 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
@@ -53,6 +53,26 @@
(api/check-superuser)
{:is_dirty (remote-sync.object/dirty-global?)})
+(api.macros/defendpoint :get "/has-remote-changes" :- remote-sync.schema/HasRemoteChangesResponse
+ "Check if there are new changes on the remote branch that can be pulled.
+ Uses in-memory caching (configurable TTL via remote-sync-check-changes-cache-ttl-seconds setting).
+
+ Returns:
+ - has_changes: true if remote version differs from last imported version, or if never imported
+ - remote_version: current Git SHA on remote branch
+ - local_version: Git SHA of last successful import (nil if never imported)
+ - cached: true if result was served from cache"
+ [_route-params
+ {:keys [force-refresh]} :- [:map [:force-refresh {:optional true} :boolean]]
+ _body]
+ (api/check-superuser)
+ (api/check-400 (settings/remote-sync-enabled) "Remote sync is not configured.")
+ (let [result (impl/has-remote-changes? {:force-refresh? force-refresh})]
+ {:has_changes (:has-changes? result)
+ :remote_version (:remote-version result)
+ :local_version (:local-version result)
+ :cached (:cached? result)}))
+
(api.macros/defendpoint :get "/dirty" :- remote-sync.schema/DirtyResponse
"Return all models with changes that have not been pushed to the remote sync source in any
remote-synced collection."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index bab21e401c4..6f17ff0308e 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -388,19 +388,86 @@
(assoc task :existing? true)
(remote-sync.task/create-sync-task! task-type api/*current-user-id*))))
+;;; ------------------------------------------- Remote Changes Check -------------------------------------------
+
+(def ^:private remote-changes-cache
+ "Cache for remote changes check to avoid frequent git operations.
+ Structure: {:last-checked <instant>
+ :branch <branch-name>
+ :remote-version <git-sha>
+ :local-version <git-sha or nil>
+ :has-changes? <boolean>}"
+ (atom nil))
+
+(defn invalidate-remote-changes-cache!
+ "Invalidate the remote changes cache. Call this after import/export."
+ []
+ (reset! remote-changes-cache nil))
+
+(defn- cache-expired?
+ "Check if the cache has expired based on TTL setting."
+ [last-checked]
+ (let [ttl-seconds (settings/remote-sync-check-changes-cache-ttl-seconds)
+ expiry-time (t/plus last-checked (t/seconds ttl-seconds))]
+ (t/after? (t/instant) expiry-time)))
+
+(defn has-remote-changes?
+ "Check if remote has new changes compared to last imported version.
+ Uses cache to avoid frequent git operations. Returns map with:
+ - :has-changes? boolean
+ - :remote-version string (git SHA)
+ - :local-version string (git SHA of last import, or nil)
+ - :cached? boolean (whether result came from cache)
+
+ Cache is invalidated if:
+ - TTL has expired
+ - Branch setting has changed
+ - force-refresh? is true"
+ ([]
+ (has-remote-changes? nil))
+ ([{:keys [force-refresh?]}]
+ (let [cache-state @remote-changes-cache
+ current-branch (settings/remote-sync-branch)
+ cache-valid? (and cache-state
+ (not force-refresh?)
+ (= current-branch (:branch cache-state))
+ (not (cache-expired? (:last-checked cache-state))))]
+ (if cache-valid?
+ (assoc cache-state :cached? true)
+ (let [last-imported (remote-sync.task/last-version)
+ source (source/source-from-settings current-branch)
+ snapshot (source.p/snapshot source)
+ current-remote (source.p/version snapshot)
+ ;; has-changes? is true if:
+ ;; - never imported (last-imported is nil), OR
+ ;; - remote version differs from local version
+ has-changes? (or (nil? last-imported)
+ (not= last-imported current-remote))
+ result {:last-checked (t/instant)
+ :branch current-branch
+ :remote-version current-remote
+ :local-version last-imported
+ :has-changes? has-changes?}]
+ (reset! remote-changes-cache result)
+ (assoc result :cached? false))))))
+
+;;; ------------------------------------------- Task Result Handling -------------------------------------------
+
(defn handle-task-result!
"Handles the outcome of running import! or export! by updating the RemoteSyncTask record.
Takes a result map with a :status key (either :success or :error) and optional :message key, a RemoteSyncTask ID,
- and an optional branch name. On success, updates the remote-sync-branch setting (if branch provided) and marks the
- task complete. On error, marks the task as failed with the error message. For any other status, marks the task as
- failed with 'Unexpected Error'."
+ and an optional branch name. On success, updates the remote-sync-branch setting (if branch provided), marks the
+ task complete, and invalidates the remote changes cache. On error, marks the task as failed with the error message.
+ For any other status, marks the task as failed with 'Unexpected Error'."
[result task-id & [branch]]
(case (:status result)
- :success (t2/with-transaction [_conn]
- (when branch
- (settings/remote-sync-branch! branch))
- (remote-sync.task/complete-sync-task! task-id))
+ :success (do
+ (t2/with-transaction [_conn]
+ (when branch
+ (settings/remote-sync-branch! branch))
+ (remote-sync.task/complete-sync-task! task-id))
+ (invalidate-remote-changes-cache!))
:error (remote-sync.task/fail-sync-task! task-id (:message result))
(remote-sync.task/fail-sync-task! task-id "Unexpected Error")))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
index fb163aaee1c..74675d564d0 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj
@@ -93,3 +93,11 @@
[:status :string]
[:message :string]
[:task_id pos-int?]])
+
+(def HasRemoteChangesResponse
+ "Schema for GET /has-remote-changes response."
+ [:map
+ [:has_changes :boolean]
+ [:remote_version [:maybe :string]]
+ [:local_version [:maybe :string]]
+ [:cached :boolean]])
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index 6bc91c963dc..abdd6117dc9 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -86,6 +86,14 @@
:encryption :no
:default (* 1000 60 5))
+(defsetting remote-sync-check-changes-cache-ttl-seconds
+ (deferred-tru "Time-to-live in seconds for the remote changes check cache. Default is 60 seconds.")
+ :type :integer
+ :visibility :admin
+ :export? false
+ :encryption :no
+ :default 60)
+
(defn check-git-settings!
"Validates git repository settings by attempting to connect and retrieve the default branch.
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index a049192cd82..123e15bfb54 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -780,6 +780,147 @@
(set (source.p/branches mock-source))))
(is (= 1 @export-calls)))))))
+;;; ------------------------------------------------- Has Remote Changes Endpoint -------------------------------------------------
+
+(deftest has-remote-changes-requires-superuser-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes requires superuser permissions"
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"]
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :get 403 "ee/remote-sync/has-remote-changes"))))))
+
+(deftest has-remote-changes-errors-when-not-configured-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes errors when remote sync is not configured"
+ (mt/with-temporary-setting-values [remote-sync-url nil]
+ (is (= "Remote sync is not configured."
+ (mt/user-http-request :crowberto :get 400 "ee/remote-sync/has-remote-changes"))))))
+
+(deftest has-remote-changes-returns-true-when-never-imported-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes returns true when never imported"
+ (let [mock-source (test-helpers/create-mock-source)]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"]
+ (with-redefs [source/source-from-settings (constantly mock-source)]
+ ;; Clear cache before test
+ (impl/invalidate-remote-changes-cache!)
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]
+ (is (true? (:has_changes response)))
+ (is (= "mock-version" (:remote_version response)))
+ (is (nil? (:local_version response)))
+ (is (= false (:cached response)))))))))
+
+(deftest has-remote-changes-returns-true-when-versions-differ-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes returns true when remote version differs from local"
+ (let [mock-source (test-helpers/create-mock-source)]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"]
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"
+ :ended_at :%now
+ :version "old-version"}]
+ (with-redefs [source/source-from-settings (constantly mock-source)]
+ ;; Clear cache before test
+ (impl/invalidate-remote-changes-cache!)
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]
+ (is (true? (:has_changes response)))
+ (is (= "mock-version" (:remote_version response)))
+ (is (= "old-version" (:local_version response)))
+ (is (= false (:cached response))))))))))
+
+(deftest has-remote-changes-returns-false-when-versions-match-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes returns false when versions match"
+ (let [mock-source (test-helpers/create-mock-source)]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"]
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"
+ :ended_at :%now
+ :version "mock-version"}]
+ (with-redefs [source/source-from-settings (constantly mock-source)]
+ ;; Clear cache before test
+ (impl/invalidate-remote-changes-cache!)
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]
+ (is (= false (:has_changes response)))
+ (is (= "mock-version" (:remote_version response)))
+ (is (= "mock-version" (:local_version response)))
+ (is (= false (:cached response))))))))))
+
+(deftest has-remote-changes-uses-cache-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes uses cache on subsequent calls"
+ (let [mock-source (test-helpers/create-mock-source)
+ call-count (atom 0)]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"
+ remote-sync-check-changes-cache-ttl-seconds 60]
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"
+ :ended_at :%now
+ :version "mock-version"}]
+ (with-redefs [source/source-from-settings (fn [& _args]
+ (swap! call-count inc)
+ mock-source)]
+ ;; Clear cache before test
+ (impl/invalidate-remote-changes-cache!)
+ ;; First call - should hit the source
+ (let [response1 (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]
+ (is (= false (:cached response1)))
+ (is (= 1 @call-count)))
+ ;; Second call - should use cache
+ (let [response2 (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]
+ (is (true? (:cached response2)))
+ (is (= 1 @call-count)))))))))
+
+(deftest has-remote-changes-force-refresh-bypasses-cache-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes?force-refresh=true bypasses cache"
+ (let [mock-source (test-helpers/create-mock-source)
+ call-count (atom 0)]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"
+ remote-sync-check-changes-cache-ttl-seconds 60]
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"
+ :ended_at :%now
+ :version "mock-version"}]
+ (with-redefs [source/source-from-settings (fn [& _args]
+ (swap! call-count inc)
+ mock-source)]
+ ;; Clear cache before test
+ (impl/invalidate-remote-changes-cache!)
+ ;; First call - should hit the source
+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")
+ (is (= 1 @call-count))
+ ;; Second call with force-refresh=true - should bypass cache
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes"
+ :force-refresh true)]
+ (is (= false (:cached response)))
+ (is (= 2 @call-count)))))))))
+
+(deftest has-remote-changes-cache-invalidated-on-branch-change-test
+ (testing "GET /api/ee/remote-sync/has-remote-changes invalidates cache when branch changes"
+ (let [mock-source (test-helpers/create-mock-source)
+ call-count (atom 0)]
+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"
+ remote-sync-check-changes-cache-ttl-seconds 60]
+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"
+ :ended_at :%now
+ :version "mock-version"}]
+ (with-redefs [source/source-from-settings (fn [& _args]
+ (swap! call-count inc)
+ mock-source)]
+ ;; Clear cache before test
+ (impl/invalidate-remote-changes-cache!)
+ ;; First call with main branch
+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")
+ (is (= 1 @call-count))
+ ;; Change branch setting
+ (mt/with-temporary-setting-values [remote-sync-branch "develop"]
+ ;; Should hit source again due to branch change
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]
+ (is (= false (:cached response)))
+ (is (= 2 @call-count))))))))))
+
;;; ------------------------------------------- Token Preservation Tests -------------------------------------------
(deftest settings-preserves-token-when-switching-to-read-only-test