Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add connection and read timeouts to python-runner requests (#80080)

Ngoc Khuat04b84f8b7734
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 9bdc7f4c453..57931d51471 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -18,7 +18,7 @@
[toucan2.core :as t2])
(:import
(java.io Closeable)
- (java.net SocketException)
+ (java.net SocketException SocketTimeoutException)
(java.time Duration)))
(set! *warn-on-reflection* true)
@@ -45,7 +45,12 @@
(log/debug "Message update loop interrupted")
(do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]
(when (pos? sleep-ms) (Thread/sleep sleep-ms)))
- (let [{:keys [status body]} (python-runner/get-logs run-id)]
+ ;; a read timeout here is usually transient, so keep polling rather than ending the loop
+ (if-let [{:keys [status body]} (try
+ (python-runner/get-logs run-id)
+ (catch SocketTimeoutException _
+ (log/debugf "Timed out polling for logs, run-id: %s" run-id)
+ nil))]
(cond
(<= 200 status 299)
(let [{:keys [execution_id events]} body]
@@ -63,7 +68,8 @@
:else
(do
(log/warnf "Unexpected status polling for logs %s, run-id: %s" status run-id)
- (log/debug "Exiting due to poll error")))))))
+ (log/debug "Exiting due to poll error")))
+ (recur)))))
(catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
(catch InterruptedException _)
(catch Throwable e
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index 4fd602ea6fb..93d3c5d95f5 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -41,15 +41,25 @@
{:error-type :configuration-error}))
{}))))
+(def ^:private connection-timeout-ms
+ "Connection timeout for runner requests."
+ (u/seconds->ms 10))
+
+(def ^:private socket-timeout-ms
+ "Read timeout for the short runner requests (logs, cancel); /execute adds it as a margin on top of the run timeout."
+ (u/seconds->ms 60))
+
(defn- python-runner-request
"Helper function for making HTTP requests to the python runner service."
[server-url method endpoint request-options & extra-args]
(let [url (str server-url "/v1" endpoint)
- base-options {:content-type :json
- :accept :json
- :throw-exceptions false
- :as :json
- :headers (authorization-headers)}]
+ base-options {:content-type :json
+ :accept :json
+ :throw-exceptions false
+ :as :json
+ :connection-timeout connection-timeout-ms
+ :socket-timeout socket-timeout-ms
+ :headers (authorization-headers)}]
(apply http/request (merge base-options request-options {:method method, :url url}) extra-args)))
(defn root-type
@@ -197,6 +207,14 @@
[~job-run-id]
(^:once fn* [] ~@body)))
+(defn cancel-python-code-http-call!
+ "Calls the /cancel endpoint of the python runner. Returns immediately."
+ [server-url run-id]
+ (python-runner-request server-url :post "/cancel" {:body (json/encode {:request_id run-id})
+ :async? true}
+ #_success (fn [_] (log/debug "Python runner cancel request completed"))
+ #_failure #(log/errorf "Python runner cancel request failed: %s" (ex-message %))))
+
(defn execute-python-code-http-call!
"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
the response from the server."
@@ -210,9 +228,10 @@
table-name->manifest-url (into {} (map (fn [{:keys [alias table_id]}]
[alias (url-for-path [:table table_id :manifest])]))
source-tables)
+ run-timeout-secs (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))
payload {:code code
:library (t2/select-fn->fn :path :source :model/PythonLibrary)
- :timeout (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))
+ :timeout run-timeout-secs
:request_id (or request-id run-id)
:output_url (:url output)
:output_manifest_url (:url output-manifest)
@@ -220,7 +239,17 @@
:table_mapping table-name->url
:manifest_mapping table-name->manifest-url}
response (with-python-api-timing [run-id]
- (python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
+ (try
+ (python-runner-request server-url :post "/execute"
+ {:body (json/encode payload)
+ :socket-timeout (+ (u/seconds->ms run-timeout-secs)
+ socket-timeout-ms)})
+ ;; a connect/read timeout counts as a runner timeout. we stop waiting, so tell
+ ;; the runner to stop too, otherwise it keeps working on an abandoned run
+ (catch java.io.InterruptedIOException _
+ (u/ignore-exceptions
+ (cancel-python-code-http-call! server-url (or request-id run-id)))
+ {:status 408 :body {:timeout true}})))]
;; when a 500 is returned we observe a string in the body (despite the python returning json)
;; always try to parse the returned string as json before yielding (could tighten this up at some point)
(update response :body (fn [string-if-error]
@@ -250,14 +279,6 @@
^InputStream [{:keys [s3-client bucket-name objects]}]
(s3/open-object s3-client bucket-name (:path (:output objects))))
-(defn cancel-python-code-http-call!
- "Calls the /cancel endpoint of the python runner. Returns immediately."
- [server-url run-id]
- (python-runner-request server-url :post "/cancel" {:body (json/encode {:request_id run-id})
- :async? true}
- #_success (fn [_] (log/debug "Python runner cancel request completed"))
- #_failure #(log/errorf "Python runner cancel request failed: %s" (ex-message %))))
-
(defn- safe-delete
"Safely delete a file."
[^File file]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
index 859d1228997..d8f4a44a61f 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
+ [metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase.test :as mt]
[metabase.test.util :as test.util]
[metabase.transforms-base.util :as transforms-base.u]
@@ -10,6 +11,8 @@
[metabase.transforms.util :as transforms.u]
[toucan2.core :as t2])
(:import
+ (java.net SocketTimeoutException)
+ (java.time Duration)
(java.util.concurrent CountDownLatch)))
(set! *warn-on-reflection* true)
@@ -172,3 +175,20 @@
clojure.lang.ExceptionInfo
#"Tables not found: missing_table"
(transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))
+
+(deftest log-poll-survives-transient-timeout-test
+ (testing "a read timeout while polling for logs is retried instead of ending the loop"
+ (let [calls (atom 0)
+ saved (atom nil)]
+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis 0)
+ python-runner/get-logs
+ (fn [run-id]
+ (condp = (swap! calls inc)
+ 1 (throw (SocketTimeoutException. "read timed out"))
+ 2 {:status 200 :body {:execution_id run-id :events [{:message "hi"}]}}
+ {:status 500 :body {}}))
+ transforms-python.execute/save-log-to-transform-run-message!
+ (fn [_run-id message-log] (reset! saved @message-log))]
+ (#'transforms-python.execute/python-message-update-loop! 42 (atom {}))
+ (is (= 3 @calls) "polling carried on past the timeout")
+ (is (= [{:message "hi"}] (:python @saved)) "and the logs that arrived after it were still captured")))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
index e8728a53448..86f1f3d8132 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
@@ -1,5 +1,6 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.python-runner-test
(:require
+ [clj-http.client :as http]
[clojure.core.async :as a]
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
@@ -508,3 +509,34 @@
(testing "Script should timeout after 5 seconds"
(is (contains? result :error))
(is (str/includes? (:error result) "timeout"))))))))
+
+(deftest python-runner-client-timeout-test
+ (testing "runner requests are bounded so a slow or unreachable runner cannot pin the calling thread"
+ (let [captured (atom nil)
+ ok (fn [opts & _] (reset! captured opts) {:status 200 :body {}})]
+ (testing "every request carries a connection and read timeout, and callers can lengthen the read timeout"
+ (with-redefs [http/request ok]
+ (#'python-runner/python-runner-request "http://runner" :get "/logs" {})
+ (is (pos-int? (:connection-timeout @captured)))
+ (is (pos-int? (:socket-timeout @captured)))
+ (#'python-runner/python-runner-request "http://runner" :post "/execute" {:socket-timeout 1234567})
+ (is (= 1234567 (:socket-timeout @captured))
+ "a caller-supplied read timeout (as /execute uses) wins over the default")))
+ (testing "/execute waits the run timeout plus the margin, and a client timeout reads back as a runner timeout"
+ (let [storage {:objects {:output {:url "u"} :output-manifest {:url "u"} :events {:url "u"}}}
+ args {:server-url "http://runner" :code "x" :request-id "r" :run-id 1
+ :source-tables [] :timeout-secs 7 :shared-storage storage}]
+ (with-redefs [http/request ok
+ t2/select-fn->fn (fn [& _] {})]
+ (python-runner/execute-python-code-http-call! args)
+ (is (= (+ (* 7 1000) @#'python-runner/socket-timeout-ms) (:socket-timeout @captured))))
+ (let [cancelled (atom nil)]
+ (with-redefs [http/request (fn [opts & _]
+ (if (str/ends-with? (:url opts) "/cancel")
+ (do (reset! cancelled opts) {:status 200 :body {}})
+ (throw (java.net.SocketTimeoutException. "read timed out"))))
+ t2/select-fn->fn (fn [& _] {})]
+ (is (:timeout (:body (python-runner/execute-python-code-http-call! args)))
+ "a hung runner surfaces as a timeout result, not a thrown error")
+ (is (= {:request_id "r"} (some-> @cancelled :body json/decode+kw))
+ "and the runner is told to drop the run we stopped waiting for"))))))))