The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add connection and read timeouts to python-runner requests (#80080)
Ngoc Khuat04b84f8b7734
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 9bdc7f4c453..57931d51471 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -18,7 +18,7 @@[toucan2.core :as t2])(:import(java.io Closeable)- (java.net SocketException)+ (java.net SocketException SocketTimeoutException)(java.time Duration)))(set! *warn-on-reflection* true)@@ -45,7 +45,12 @@(log/debug "Message update loop interrupted")(do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)](when (pos? sleep-ms) (Thread/sleep sleep-ms)))- (let [{:keys [status body]} (python-runner/get-logs run-id)]+ ;; a read timeout here is usually transient, so keep polling rather than ending the loop+ (if-let [{:keys [status body]} (try+ (python-runner/get-logs run-id)+ (catch SocketTimeoutException _+ (log/debugf "Timed out polling for logs, run-id: %s" run-id)+ nil))](cond(<= 200 status 299)(let [{:keys [execution_id events]} body]@@ -63,7 +68,8 @@:else(do(log/warnf "Unexpected status polling for logs %s, run-id: %s" status run-id)- (log/debug "Exiting due to poll error")))))))+ (log/debug "Exiting due to poll error")))+ (recur)))))(catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))(catch InterruptedException _)(catch Throwable ediff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 4fd602ea6fb..93d3c5d95f5 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -41,15 +41,25 @@{:error-type :configuration-error})){}))))+(def ^:private connection-timeout-ms+ "Connection timeout for runner requests."+ (u/seconds->ms 10))++(def ^:private socket-timeout-ms+ "Read timeout for the short runner requests (logs, cancel); /execute adds it as a margin on top of the run timeout."+ (u/seconds->ms 60))+(defn- python-runner-request"Helper function for making HTTP requests to the python runner service."[server-url method endpoint request-options & extra-args](let [url (str server-url "/v1" endpoint)- base-options {:content-type :json- :accept :json- :throw-exceptions false- :as :json- :headers (authorization-headers)}]+ base-options {:content-type :json+ :accept :json+ :throw-exceptions false+ :as :json+ :connection-timeout connection-timeout-ms+ :socket-timeout socket-timeout-ms+ :headers (authorization-headers)}](apply http/request (merge base-options request-options {:method method, :url url}) extra-args)))(defn root-type@@ -197,6 +207,14 @@[~job-run-id](^:once fn* [] ~@body)))+(defn cancel-python-code-http-call!+ "Calls the /cancel endpoint of the python runner. Returns immediately."+ [server-url run-id]+ (python-runner-request server-url :post "/cancel" {:body (json/encode {:request_id run-id})+ :async? true}+ #_success (fn [_] (log/debug "Python runner cancel request completed"))+ #_failure #(log/errorf "Python runner cancel request failed: %s" (ex-message %))))+(defn execute-python-code-http-call!"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returnsthe response from the server."@@ -210,9 +228,10 @@table-name->manifest-url (into {} (map (fn [{:keys [alias table_id]}][alias (url-for-path [:table table_id :manifest])]))source-tables)+ run-timeout-secs (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))payload {:code code:library (t2/select-fn->fn :path :source :model/PythonLibrary)- :timeout (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))+ :timeout run-timeout-secs:request_id (or request-id run-id):output_url (:url output):output_manifest_url (:url output-manifest)@@ -220,7 +239,17 @@:table_mapping table-name->url:manifest_mapping table-name->manifest-url}response (with-python-api-timing [run-id]- (python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]+ (try+ (python-runner-request server-url :post "/execute"+ {:body (json/encode payload)+ :socket-timeout (+ (u/seconds->ms run-timeout-secs)+ socket-timeout-ms)})+ ;; a connect/read timeout counts as a runner timeout. we stop waiting, so tell+ ;; the runner to stop too, otherwise it keeps working on an abandoned run+ (catch java.io.InterruptedIOException _+ (u/ignore-exceptions+ (cancel-python-code-http-call! server-url (or request-id run-id)))+ {:status 408 :body {:timeout true}})))];; when a 500 is returned we observe a string in the body (despite the python returning json);; always try to parse the returned string as json before yielding (could tighten this up at some point)(update response :body (fn [string-if-error]@@ -250,14 +279,6 @@^InputStream [{:keys [s3-client bucket-name objects]}](s3/open-object s3-client bucket-name (:path (:output objects))))-(defn cancel-python-code-http-call!- "Calls the /cancel endpoint of the python runner. Returns immediately."- [server-url run-id]- (python-runner-request server-url :post "/cancel" {:body (json/encode {:request_id run-id})- :async? true}- #_success (fn [_] (log/debug "Python runner cancel request completed"))- #_failure #(log/errorf "Python runner cancel request failed: %s" (ex-message %))))-(defn- safe-delete"Safely delete a file."[^File file]diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.cljindex 859d1228997..d8f4a44a61f 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase-enterprise.transforms-python.execute :as transforms-python.execute]+ [metabase-enterprise.transforms-python.python-runner :as python-runner][metabase.test :as mt][metabase.test.util :as test.util][metabase.transforms-base.util :as transforms-base.u]@@ -10,6 +11,8 @@[metabase.transforms.util :as transforms.u][toucan2.core :as t2])(:import+ (java.net SocketTimeoutException)+ (java.time Duration)(java.util.concurrent CountDownLatch)))(set! *warn-on-reflection* true)@@ -172,3 +175,20 @@clojure.lang.ExceptionInfo#"Tables not found: missing_table"(transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))++(deftest log-poll-survives-transient-timeout-test+ (testing "a read timeout while polling for logs is retried instead of ending the loop"+ (let [calls (atom 0)+ saved (atom nil)]+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis 0)+ python-runner/get-logs+ (fn [run-id]+ (condp = (swap! calls inc)+ 1 (throw (SocketTimeoutException. "read timed out"))+ 2 {:status 200 :body {:execution_id run-id :events [{:message "hi"}]}}+ {:status 500 :body {}}))+ transforms-python.execute/save-log-to-transform-run-message!+ (fn [_run-id message-log] (reset! saved @message-log))]+ (#'transforms-python.execute/python-message-update-loop! 42 (atom {}))+ (is (= 3 @calls) "polling carried on past the timeout")+ (is (= [{:message "hi"}] (:python @saved)) "and the logs that arrived after it were still captured")))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.cljindex e8728a53448..86f1f3d8132 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj@@ -1,5 +1,6 @@(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.python-runner-test(:require+ [clj-http.client :as http][clojure.core.async :as a][clojure.java.jdbc :as jdbc][clojure.string :as str]@@ -508,3 +509,34 @@(testing "Script should timeout after 5 seconds"(is (contains? result :error))(is (str/includes? (:error result) "timeout"))))))))++(deftest python-runner-client-timeout-test+ (testing "runner requests are bounded so a slow or unreachable runner cannot pin the calling thread"+ (let [captured (atom nil)+ ok (fn [opts & _] (reset! captured opts) {:status 200 :body {}})]+ (testing "every request carries a connection and read timeout, and callers can lengthen the read timeout"+ (with-redefs [http/request ok]+ (#'python-runner/python-runner-request "http://runner" :get "/logs" {})+ (is (pos-int? (:connection-timeout @captured)))+ (is (pos-int? (:socket-timeout @captured)))+ (#'python-runner/python-runner-request "http://runner" :post "/execute" {:socket-timeout 1234567})+ (is (= 1234567 (:socket-timeout @captured))+ "a caller-supplied read timeout (as /execute uses) wins over the default")))+ (testing "/execute waits the run timeout plus the margin, and a client timeout reads back as a runner timeout"+ (let [storage {:objects {:output {:url "u"} :output-manifest {:url "u"} :events {:url "u"}}}+ args {:server-url "http://runner" :code "x" :request-id "r" :run-id 1+ :source-tables [] :timeout-secs 7 :shared-storage storage}]+ (with-redefs [http/request ok+ t2/select-fn->fn (fn [& _] {})]+ (python-runner/execute-python-code-http-call! args)+ (is (= (+ (* 7 1000) @#'python-runner/socket-timeout-ms) (:socket-timeout @captured))))+ (let [cancelled (atom nil)]+ (with-redefs [http/request (fn [opts & _]+ (if (str/ends-with? (:url opts) "/cancel")+ (do (reset! cancelled opts) {:status 200 :body {}})+ (throw (java.net.SocketTimeoutException. "read timed out"))))+ t2/select-fn->fn (fn [& _] {})]+ (is (:timeout (:body (python-runner/execute-python-code-http-call! args)))+ "a hung runner surfaces as a timeout result, not a thrown error")+ (is (= {:request_id "r"} (some-> @cancelled :body json/decode+kw))+ "and the runner is told to drop the run we stopped waiting for"))))))))