The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Unblock remote sync when configured branch is deleted upstream (#72885)
Edward Paget06f43840ce54
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.cljindex 86b154eda85..538a30e4720 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj@@ -81,10 +81,11 @@(api/check-superuser)(api/check-400 (settings/remote-sync-enabled) "Remote sync is not configured.")(let [result (impl/has-remote-changes? {:force-refresh? force-refresh})]- {:has_changes (:has-changes? result)- :remote_version (:remote-version result)- :local_version (:local-version result)- :cached (:cached? result)}))+ (cond-> {:has_changes (:has-changes? result)+ :remote_version (:remote-version result)+ :local_version (:local-version result)+ :cached (:cached? result)}+ (:branch-missing? result) (assoc :branch_missing true))))(api.macros/defendpoint :get "/dirty" :- remote-sync.schema/DirtyResponse"Return all models with changes that have not been pushed to the remote sync source in anydiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 735e93f6c9f..4310dc1ebcb 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -405,6 +405,51 @@expiry-time (t/plus last-checked (t/seconds ttl-seconds))](t/after? (t/instant) expiry-time)))+(defn- cache-valid? [cache-state current-branch force-refresh?]+ (and cache-state+ (not force-refresh?)+ (= current-branch (:branch cache-state))+ (not (cache-expired? (:last-checked cache-state)))))++(defn- snapshot-or-missing-branch+ "Returns a snapshot of `source`, or `::missing-branch` if the configured branch+ no longer exists on the remote. Other exceptions propagate."+ [source]+ (try+ (source.p/snapshot source)+ (catch Exception e+ (case (:error-type (ex-data e))+ :missing-branch ::missing-branch+ (throw e)))))++(defn- branch-missing-result+ "Response for a configured branch that no longer exists upstream. Intentionally+ not cached so the next call re-checks after the user switches branches."+ [current-branch last-imported]+ {:last-checked (t/instant)+ :branch current-branch+ :remote-version nil+ :local-version last-imported+ :has-changes? false+ :branch-missing? true+ :cached? false})++(defn- fresh-result!+ "Computes a has-remote-changes? response from a successful snapshot, caches it,+ and returns it tagged as uncached."+ [current-branch last-imported snapshot]+ (let [current-remote (source.p/version snapshot)+ result {:last-checked (t/instant)+ :branch current-branch+ :remote-version current-remote+ :local-version last-imported+ ;; has-changes? is true if nothing's been imported yet, or if+ ;; remote moved past local.+ :has-changes? (or (nil? last-imported)+ (not= last-imported current-remote))}]+ (reset! remote-changes-cache result)+ (assoc result :cached? false)))+(defn has-remote-changes?"Check if remote has new changes compared to last imported version.Uses cache to avoid frequent git operations. Returns map with:@@ -412,6 +457,9 @@- :remote-version string (git SHA)- :local-version string (git SHA of last import, or nil)- :cached? boolean (whether result came from cache)+ - :branch-missing? boolean (true if the configured branch no longer exists+ on the remote; the result is not cached in that case so the next call+ picks up a branch switch immediately)Cache is invalidated if:- TTL has expired@@ -421,29 +469,15 @@(has-remote-changes? nil))([{:keys [force-refresh?]}](let [cache-state @remote-changes-cache- current-branch (settings/remote-sync-branch)- cache-valid? (and cache-state- (not force-refresh?)- (= current-branch (:branch cache-state))- (not (cache-expired? (:last-checked cache-state))))]- (if cache-valid?+ current-branch (settings/remote-sync-branch)]+ (if (cache-valid? cache-state current-branch force-refresh?)(assoc cache-state :cached? true)(let [last-imported (remote-sync.task/last-version)source (source/source-from-settings current-branch)- snapshot (source.p/snapshot source)- current-remote (source.p/version snapshot)- ;; has-changes? is true if:- ;; - never imported (last-imported is nil), OR- ;; - remote version differs from local version- has-changes? (or (nil? last-imported)- (not= last-imported current-remote))- result {:last-checked (t/instant)- :branch current-branch- :remote-version current-remote- :local-version last-imported- :has-changes? has-changes?}]- (reset! remote-changes-cache result)- (assoc result :cached? false))))))+ snapshot (snapshot-or-missing-branch source)]+ (if (= ::missing-branch snapshot)+ (branch-missing-result current-branch last-imported)+ (fresh-result! current-branch last-imported snapshot)))))));;; ------------------------------------------- Task Result Handling -------------------------------------------diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.cljindex 8a6d49e91a6..ef7c298bbf6 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj@@ -132,7 +132,8 @@[:has_changes :boolean][:remote_version [:maybe :string]][:local_version [:maybe :string]]- [:cached :boolean]])+ [:cached :boolean]+ [:branch_missing {:optional true} :boolean]])(def TestConnectionResponse"Schema for POST /test-connection response."diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex bcd92ed211e..3f0879092cf 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -89,13 +89,15 @@Takes a git-source map containing a :git Git instance and optional :token for authentication. Returns the resultof the git fetch operation. Uses the 'origin' remote which is configured by ensure-origin-configured!.+ Prunes local refs that no longer exist on the remote so deleted branches are reflected locally.Throws ExceptionInfo if the fetch operation fails."[{:keys [^Git git] :as git-source}](when (some? git)(log/info "Fetching repository" {:repo (str git)})- (u/prog1 (call-remote-command (.fetch git) git-source))- (log/info "Successfully fetched repository")))+ (u/prog1 (call-remote-command (.. git fetch (setRemoveDeletedRefs true))+ git-source)+ (log/info "Successfully fetched repository"))))(defn- repo-path [{:keys [^String remote-url ^String token]}](io/file (System/getProperty "java.io.tmpdir") "metabase-git" (-> (str/join ":" [remote-url token]) buddy-hash/sha1 codecs/bytes->hex)))@@ -438,7 +440,9 @@(let [version (commit-sha source (:branch source))](if version(->GitSnapshot (:git source) (:remote-url source) (:branch source) version (:token source) (:managed-dirs source))- (throw (ex-info (str "Invalid branch: " (:branch source)) {})))))+ (throw (ex-info (str "Invalid branch: " (:branch source))+ {:error-type :missing-branch+ :branch (:branch source)})))))(defn- snapshot"Creates a snapshot, recovering from stale cache errors by re-cloning."diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex f68c208c727..5f9685cb4d1 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -1092,6 +1092,80 @@(is (= false (:cached response)))(is (= 2 @call-count))))))))))+(defn- missing-branch-source+ "Source whose snapshot throws the typed :missing-branch ex-info — simulating a+ configured branch that has been deleted upstream."+ [branch]+ (reify source.p/Source+ (branches [_] ["main"])+ (create-branch [_ _ _] nil)+ (default-branch [_] "main")+ (snapshot [_]+ (throw (ex-info (str "Invalid branch: " branch)+ {:error-type :missing-branch+ :branch branch})))))++(deftest has-remote-changes-returns-branch-missing-gracefully-test+ (testing "GET /has-remote-changes returns branch_missing=true instead of 500 when the branch has been deleted upstream"+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "gone"+ remote-sync-check-changes-cache-ttl-seconds 60]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :ended_at :%now+ :version "some-prior-version"}]+ (with-redefs [source/source-from-settings (fn [& _] (missing-branch-source "gone"))]+ (impl/invalidate-remote-changes-cache!)+ (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (false? (:has_changes response)))+ (is (true? (:branch_missing response)))+ (is (nil? (:remote_version response)))+ (is (= "some-prior-version" (:local_version response)))+ (is (false? (:cached response)))))))))++(deftest has-remote-changes-does-not-cache-branch-missing-test+ (testing "Branch-missing results are not cached so a subsequent call re-checks"+ (let [snapshot-calls (atom 0)+ ;; First call returns missing-branch; second call succeeds.+ source-fn (fn [& _]+ (swap! snapshot-calls inc)+ (if (= 1 @snapshot-calls)+ (missing-branch-source "gone")+ (test-helpers/create-mock-source)))]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "gone"+ remote-sync-check-changes-cache-ttl-seconds 60]+ (mt/with-temp [:model/RemoteSyncTask _ {:sync_task_type "import"+ :ended_at :%now+ :version "mock-version"}]+ (with-redefs [source/source-from-settings source-fn]+ (impl/invalidate-remote-changes-cache!)+ (let [first-response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (true? (:branch_missing first-response)))+ (is (false? (:cached first-response))))+ ;; Second call re-hits the source; cache was not populated by the first call.+ (let [second-response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/has-remote-changes")]+ (is (not (:branch_missing second-response)))+ (is (false? (:cached second-response)))+ (is (= 2 @snapshot-calls)))))))))++(deftest has-remote-changes-still-propagates-other-errors-test+ (testing "Non-missing-branch failures still propagate (the graceful handler is :missing-branch-only)"+ (let [failing-source (reify source.p/Source+ (branches [_] ["main"])+ (create-branch [_ _ _] nil)+ (default-branch [_] "main")+ (snapshot [_] (throw (RuntimeException. "boom"))))]+ (mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"+ remote-sync-check-changes-cache-ttl-seconds 60]+ (with-redefs [source/source-from-settings (fn [& _] failing-source)]+ (impl/invalidate-remote-changes-cache!)+ (is (thrown-with-msg? RuntimeException #"boom"+ (impl/has-remote-changes?))))))))+;;; ------------------------------------------- Token Preservation Tests -------------------------------------------(deftest settings-preserves-token-when-switching-to-read-only-testdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 4fab35c8070..c882901738d 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -458,3 +458,54 @@(testing "Bitbucket URL uses x-token-auth"(let [provider (#'git/credentials-provider "https://bitbucket.org/org/repo" "my-token")](is (instance? UsernamePasswordCredentialsProvider provider)))))++;; ---------------------------------------------------------------------------+;; Missing remote branch tests (issue #72778)+;; ---------------------------------------------------------------------------++(defn- delete-remote-branch!+ "Deletes a branch on the 'remote' repo used by a test."+ [{:keys [^Git git]} ^String branch]+ (-> (.branchDelete git)+ (.setBranchNames ^"[Ljava.lang.String;" (into-array String [branch]))+ (.setForce true)+ (.call)))++(deftest fetch!-prunes-deleted-remote-branches-test+ (mt/with-temp-dir [remote-dir nil]+ (let [[source remote] (init-source! "master" remote-dir :branches ["branch-1"])]+ (is (some? (git/commit-sha source "branch-1"))+ "Precondition: branch-1 is resolvable locally after initial clone")+ (delete-remote-branch! remote "branch-1")+ (git/fetch! source)+ (is (nil? (git/commit-sha source "branch-1"))+ "branch-1 ref is pruned locally after the remote branch is deleted")+ (is (some? (git/commit-sha source "master"))+ "other refs are unaffected"))))++(deftest snapshot-throws-missing-branch-ex-data-test+ (mt/with-temp-dir [remote-dir nil]+ (let [[_master remote] (init-source! "master" remote-dir+ :files {"master.txt" "x"})+ bad-source (->source! "does-not-exist" remote)]+ (try+ (source.p/snapshot bad-source)+ (is false "snapshot should have thrown")+ (catch clojure.lang.ExceptionInfo e+ (is (= "Invalid branch: does-not-exist" (ex-message e)))+ (is (= :missing-branch (:error-type (ex-data e))))+ (is (= "does-not-exist" (:branch (ex-data e)))))))))++(deftest snapshot-throws-missing-branch-after-remote-delete-test+ (mt/with-temp-dir [remote-dir nil]+ (let [[_master remote] (init-source! "master" remote-dir :branches ["branch-1"])+ source-on-branch-1 (->source! "branch-1" remote)]+ (is (some? (source.p/snapshot source-on-branch-1))+ "Precondition: snapshot works before the branch is deleted")+ (delete-remote-branch! remote "branch-1")+ (try+ (source.p/snapshot source-on-branch-1)+ (is false "snapshot should have thrown after the remote branch was deleted")+ (catch clojure.lang.ExceptionInfo e+ (is (= :missing-branch (:error-type (ex-data e))))+ (is (= "branch-1" (:branch (ex-data e)))))))))