The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add /has-remote-changes endpoint to remote-sync API (#67924)
Edward Paget60093c461b22
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.cljindex 775280f2e29..53c983f0e1f 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj@@ -53,6 +53,26 @@(api/check-superuser){:is_dirty (remote-sync.object/dirty-global?)})+(api.macros/defendpoint :get "/has-remote-changes" :- remote-sync.schema/HasRemoteChangesResponse+ "Check if there are new changes on the remote branch that can be pulled.+ Uses in-memory caching (configurable TTL via remote-sync-check-changes-cache-ttl-seconds setting).++ Returns:+ - has_changes: true if remote version differs from last imported version, or if never imported+ - remote_version: current Git SHA on remote branch+ - local_version: Git SHA of last successful import (nil if never imported)+ - cached: true if result was served from cache"+ [_route-params+ {:keys [force-refresh]} :- [:map [:force-refresh {:optional true} :boolean]]+ _body]+ (api/check-superuser)+ (api/check-400 (settings/remote-sync-enabled) "Remote sync is not configured.")+ (let [result (impl/has-remote-changes? {:force-refresh? force-refresh})]+ {:has_changes (:has-changes? result)+ :remote_version (:remote-version result)+ :local_version (:local-version result)+ :cached (:cached? result)}))+(api.macros/defendpoint :get "/dirty" :- remote-sync.schema/DirtyResponse"Return all models with changes that have not been pushed to the remote sync source in anyremote-synced collection."diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex bab21e401c4..6f17ff0308e 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -388,19 +388,86 @@(assoc task :existing? true)(remote-sync.task/create-sync-task! task-type api/*current-user-id*))))+;;; ------------------------------------------- Remote Changes Check -------------------------------------------++(def ^:private remote-changes-cache+ "Cache for remote changes check to avoid frequent git operations.+ Structure: {:last-checked <instant>+ :branch <branch-name>+ :remote-version <git-sha>+ :local-version <git-sha or nil>+ :has-changes? <boolean>}"+ (atom nil))++(defn invalidate-remote-changes-cache!+ "Invalidate the remote changes cache. Call this after import/export."+ []+ (reset! remote-changes-cache nil))++(defn- cache-expired?+ "Check if the cache has expired based on TTL setting."+ [last-checked]+ (let [ttl-seconds (settings/remote-sync-check-changes-cache-ttl-seconds)+ expiry-time (t/plus last-checked (t/seconds ttl-seconds))]+ (t/after? (t/instant) expiry-time)))++(defn has-remote-changes?+ "Check if remote has new changes compared to last imported version.+ Uses cache to avoid frequent git operations. Returns map with:+ - :has-changes? boolean+ - :remote-version string (git SHA)+ - :local-version string (git SHA of last import, or nil)+ - :cached? boolean (whether result came from cache)++ Cache is invalidated if:+ - TTL has expired+ - Branch setting has changed+ - force-refresh? is true"+ ([]+ (has-remote-changes? nil))+ ([{:keys [force-refresh?]}]+ (let [cache-state @remote-changes-cache+ current-branch (settings/remote-sync-branch)+ cache-valid? (and cache-state+ (not force-refresh?)+ (= current-branch (:branch cache-state))+ (not (cache-expired? (:last-checked cache-state))))]+ (if cache-valid?+ (assoc cache-state :cached? true)+ (let [last-imported (remote-sync.task/last-version)+ source (source/source-from-settings current-branch)+ snapshot (source.p/snapshot source)+ current-remote (source.p/version snapshot)+ ;; has-changes? is true if:+ ;; - never imported (last-imported is nil), OR+ ;; - remote version differs from local version+ has-changes? (or (nil? last-imported)+ (not= last-imported current-remote))+ result {:last-checked (t/instant)+ :branch current-branch+ :remote-version current-remote+ :local-version last-imported+ :has-changes? has-changes?}]+ (reset! remote-changes-cache result)+ (assoc result :cached? false))))))++;;; ------------------------------------------- Task Result Handling -------------------------------------------+(defn handle-task-result!"Handles the outcome of running import! or export! by updating the RemoteSyncTask record.Takes a result map with a :status key (either :success or :error) and optional :message key, a RemoteSyncTask ID,- and an optional branch name. On success, updates the remote-sync-branch setting (if branch provided) and marks the- task complete. On error, marks the task as failed with the error message. For any other status, marks the task as- failed with 'Unexpected Error'."+ and an optional branch name. On success, updates the remote-sync-branch setting (if branch provided), marks the+ task complete, and invalidates the remote changes cache. On error, marks the task as failed with the error message.+ For any other status, marks the task as failed with 'Unexpected Error'."[result task-id & [branch]](case (:status result)- :success (t2/with-transaction [_conn]- (when branch- (settings/remote-sync-branch! branch))- (remote-sync.task/complete-sync-task! task-id))+ :success (do+ (t2/with-transaction [_conn]+ (when branch+ (settings/remote-sync-branch! branch))+ (remote-sync.task/complete-sync-task! task-id))+ (invalidate-remote-changes-cache!)):error (remote-sync.task/fail-sync-task! task-id (:message result))(remote-sync.task/fail-sync-task! task-id "Unexpected Error")))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.cljindex fb163aaee1c..74675d564d0 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj@@ -93,3 +93,11 @@[:status :string][:message :string][:task_id pos-int?]])++(def HasRemoteChangesResponse+ "Schema for GET /has-remote-changes response."+ [:map+ [:has_changes :boolean]+ [:remote_version [:maybe :string]]+ [:local_version [:maybe :string]]+ [:cached :boolean]])diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex 6bc91c963dc..abdd6117dc9 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -86,6 +86,14 @@:encryption :no:default (* 1000 60 5))+(defsetting remote-sync-check-changes-cache-ttl-seconds+ (deferred-tru "Time-to-live in seconds for the remote changes check cache. Default is 60 seconds.")+ :type :integer+ :visibility :admin+ :export? false+ :encryption :no+ :default 60)+(defn check-git-settings!"Validates git repository settings by attempting to connect and retrieve the default branch.diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex a049192cd82..123e15bfb54 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -780,6 +780,147 @@(set (source.p/branches mock-source))))(is (= 1 @export-calls)))))))+;;; ------------------------------------------------- Has Remote Changes Endpoint -------------------------------------------------++(deftest has-remote-changes-requires-superuser-test+ (testing "GET /api/ee/remote-sync/has-remote-changes requires superuser permissions"+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"]+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :get 403 "ee/remote-sync/has-remote-changes"))))))++(deftest has-remote-changes-errors-when-not-configured-test+ (testing "GET /api/ee/remote-sync/has-remote-changes errors when remote sync is not configured"+ (mt/with-temporary-setting-values [remote-sync-url nil]+ (is (= "Remote sync is not configured."+ (mt/user-http-request :crowberto :get 400 "ee/remote-sync/has-remote-changes"))))))++(deftest has-remote-changes-returns-true-when-never-imported-test+ (testing "GET /api/ee/remote-sync/has-remote-changes returns true when never imported"+ (let [mock-source (test-helpers/create-mock-source)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"]+ (with-redefs [source/source-from-settings (constantly mock-source)]+ ;; Clear cache before test+ (impl/invalidate-remote-changes-cache!)+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (true? (:has_changes response)))+ (is (= "mock-version" (:remote_version response)))+ (is (nil? (:local_version response)))+ (is (= false (:cached response)))))))))++(deftest has-remote-changes-returns-true-when-versions-differ-test+ (testing "GET /api/ee/remote-sync/has-remote-changes returns true when remote version differs from local"+ (let [mock-source (test-helpers/create-mock-source)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :ended_at :%now+ :version "old-version"}]+ (with-redefs [source/source-from-settings (constantly mock-source)]+ ;; Clear cache before test+ (impl/invalidate-remote-changes-cache!)+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (true? (:has_changes response)))+ (is (= "mock-version" (:remote_version response)))+ (is (= "old-version" (:local_version response)))+ (is (= false (:cached response))))))))))++(deftest has-remote-changes-returns-false-when-versions-match-test+ (testing "GET /api/ee/remote-sync/has-remote-changes returns false when versions match"+ (let [mock-source (test-helpers/create-mock-source)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :ended_at :%now+ :version "mock-version"}]+ (with-redefs [source/source-from-settings (constantly mock-source)]+ ;; Clear cache before test+ (impl/invalidate-remote-changes-cache!)+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (= false (:has_changes response)))+ (is (= "mock-version" (:remote_version response)))+ (is (= "mock-version" (:local_version response)))+ (is (= false (:cached response))))))))))++(deftest has-remote-changes-uses-cache-test+ (testing "GET /api/ee/remote-sync/has-remote-changes uses cache on subsequent calls"+ (let [mock-source (test-helpers/create-mock-source)+ call-count (atom 0)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-check-changes-cache-ttl-seconds 60]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :ended_at :%now+ :version "mock-version"}]+ (with-redefs [source/source-from-settings (fn [& _args]+ (swap! call-count inc)+ mock-source)]+ ;; Clear cache before test+ (impl/invalidate-remote-changes-cache!)+ ;; First call - should hit the source+ (let [response1 (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (= false (:cached response1)))+ (is (= 1 @call-count)))+ ;; Second call - should use cache+ (let [response2 (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (true? (:cached response2)))+ (is (= 1 @call-count)))))))))++(deftest has-remote-changes-force-refresh-bypasses-cache-test+ (testing "GET /api/ee/remote-sync/has-remote-changes?force-refresh=true bypasses cache"+ (let [mock-source (test-helpers/create-mock-source)+ call-count (atom 0)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-check-changes-cache-ttl-seconds 60]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :ended_at :%now+ :version "mock-version"}]+ (with-redefs [source/source-from-settings (fn [& _args]+ (swap! call-count inc)+ mock-source)]+ ;; Clear cache before test+ (impl/invalidate-remote-changes-cache!)+ ;; First call - should hit the source+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")+ (is (= 1 @call-count))+ ;; Second call with force-refresh=true - should bypass cache+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes"+ :force-refresh true)]+ (is (= false (:cached response)))+ (is (= 2 @call-count)))))))))++(deftest has-remote-changes-cache-invalidated-on-branch-change-test+ (testing "GET /api/ee/remote-sync/has-remote-changes invalidates cache when branch changes"+ (let [mock-source (test-helpers/create-mock-source)+ call-count (atom 0)]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-check-changes-cache-ttl-seconds 60]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :ended_at :%now+ :version "mock-version"}]+ (with-redefs [source/source-from-settings (fn [& _args]+ (swap! call-count inc)+ mock-source)]+ ;; Clear cache before test+ (impl/invalidate-remote-changes-cache!)+ ;; First call with main branch+ (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")+ (is (= 1 @call-count))+ ;; Change branch setting+ (mt/with-temporary-setting-values [remote-sync-branch "develop"]+ ;; Should hit source again due to branch change+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (= false (:cached response)))+ (is (= 2 @call-count))))))))))+;;; ------------------------------------------- Token Preservation Tests -------------------------------------------(deftest settings-preserves-token-when-switching-to-read-only-test