Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3727 add network timeout to remote sync git operations (#75047)

Eric Normandd30920ac1af7
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index 4fc3d9fb097..32fdde1ba20 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -89,6 +89,14 @@
:encryption :no
:default (* 1000 60 5))
+(defsetting remote-sync-git-timeout-seconds
+ (deferred-tru "Network timeout (in seconds) for remote git operations such as fetch, push, clone, and ls-remote. A stalled connection would otherwise hang a sync indefinitely.")
+ :type :integer
+ :visibility :authenticated
+ :export? false
+ :encryption :no
+ :default 60)
+
(def ^:const transforms-root-id
"Sentinel value for the virtual Transforms root collection.
Used to represent the entire transforms feature being enabled/disabled."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index 028e37701f9..bcd92ed211e 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -6,6 +6,7 @@
[clojure.string :as str]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase.analytics-interface.core :as analytics]
+ [metabase.settings.core :as setting]
[metabase.util :as u]
[metabase.util.log :as log])
(:import
@@ -69,8 +70,10 @@
credentials-provider (when token (credentials-provider remote-url token))]
(analytics/inc! :metabase-remote-sync/git-operations analytics-labels)
(try
- (-> command
- (.setCredentialsProvider credentials-provider)
+ (-> (doto command
+ ;; bound the network operation so a stalled connection can't hang the sync forever (GHY-3727)
+ (.setTimeout (int (setting/get :remote-sync-git-timeout-seconds)))
+ (.setCredentialsProvider credentials-provider))
(.call))
(catch Exception e
(analytics/inc! :metabase-remote-sync/git-operations-failed analytics-labels)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index 2b62a6ef773..4fab35c8070 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -10,7 +10,7 @@
[metabase.util :as u])
(:import (java.io File)
(org.apache.commons.io FileUtils)
- (org.eclipse.jgit.api Git)
+ (org.eclipse.jgit.api Git TransportCommand)
(org.eclipse.jgit.lib PersonIdent)
(org.eclipse.jgit.transport UsernamePasswordCredentialsProvider)))
@@ -93,10 +93,39 @@
(let [remote-repo (apply init-remote! dir config)]
[(->source! branch remote-repo) remote-repo]))
+(defn- command-timeout
+ "Reads the protected `timeout` field (in seconds) that JGit applies to a TransportCommand's
+ network operations. 0 means no timeout (JGit's default), i.e. the operation can hang forever."
+ [^TransportCommand cmd]
+ (let [f (.getDeclaredField TransportCommand "timeout")]
+ (.setAccessible f true)
+ (.getInt f cmd)))
+
(deftest qualify-branch-test
(is (= "refs/heads/main" (#'git/qualify-branch "main")))
(is (= "refs/heads/main" (#'git/qualify-branch "refs/heads/main"))))
+(deftest call-remote-command-applies-network-timeout-test
+ (testing "Remote git operations get a positive network timeout so a stalled connection can't hang
+ the sync thread forever (GHY-3727: pull/push gets stuck at progress 0 and 0.3)"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[source _remote] (init-source! "master" remote-dir :files {"master.txt" "File in master"})
+ ^Git git (:git source)
+ cmd (.lsRemote git)]
+ (#'git/call-remote-command cmd source)
+ (is (pos? (command-timeout cmd))
+ "TransportCommand should have a positive (non-zero) timeout configured before .call")))))
+
+(deftest call-remote-command-respects-timeout-setting-test
+ (testing "The network timeout applied to remote git operations is driven by remote-sync-git-timeout-seconds"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[source _remote] (init-source! "master" remote-dir :files {"master.txt" "File in master"})
+ ^Git git (:git source)
+ cmd (.lsRemote git)]
+ (mt/with-temporary-setting-values [remote-sync-git-timeout-seconds 17]
+ (#'git/call-remote-command cmd source)
+ (is (= 17 (command-timeout cmd))))))))
+
(deftest log
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir :branches ["branch-1" "branch-2"])