Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Stop timed-out mage commands and their descendants (#81463)

Chris Truterd7a519ab6072
diff --git a/mage/src/mage/shell.clj b/mage/src/mage/shell.clj
index ab6926f608e..a1dfbbb8eaa 100644
--- a/mage/src/mage/shell.clj
+++ b/mage/src/mage/shell.clj
@@ -3,7 +3,8 @@
[clojure.string :as str]
[mage.util :as u])
(:import
- (java.io BufferedReader File InputStreamReader)))
+ (java.io BufferedReader File InputStreamReader)
+ (java.lang ProcessHandle)))
(set! *warn-on-reflection* true)
@@ -19,9 +20,44 @@
(defn- deref-with-timeout [dereffable timeout-ms]
(let [result (deref dereffable timeout-ms ::timed-out)]
(when (= result ::timed-out)
- (throw (ex-info (format "Timed out after %d ms." timeout-ms) {})))
+ (throw (ex-info (format "Timed out after %d ms." timeout-ms) {:timed-out? true})))
result))
+(def ^:private ns-per-ms 1000000)
+
+;; How long a signalled process gets to wind itself down before the next, harsher signal.
+(def ^:private kill-grace-period-ms (* 5 1000)) ; 5 seconds
+
+;; How often we re-ask a signalled process whether it has exited yet.
+(def ^:private exit-poll-interval-ms 50)
+
+(defn- wait-for-exit
+ "Wait up to `timeout-ms` for every process in `handles` to exit."
+ [handles timeout-ms]
+ ;; nanoTime rather than wall-clock time, so an NTP correction or a DST shift cannot move the deadline.
+ (let [deadline (+ (System/nanoTime) (* timeout-ms ns-per-ms))]
+ (loop []
+ (when (and (some #(.isAlive ^ProcessHandle %) handles)
+ (< (System/nanoTime) deadline))
+ (Thread/sleep exit-poll-interval-ms)
+ (recur)))))
+
+(defn- kill-process!
+ "Stop `proc` and every process it started, so a timed-out command cannot keep running behind the caller.
+ The descendants are listed before anything is signalled, and each survivor is force-killed on its own,
+ so a child that ignores the first signal dies even after its parent has gone.
+ A child that had already outlived its parent is not reachable from the root handle."
+ [^Process proc]
+ (let [root (.toHandle proc)
+ handles (cons root (iterator-seq (.iterator (.descendants root))))]
+ (run! #(.destroy ^ProcessHandle %) handles)
+ (wait-for-exit handles kill-grace-period-ms)
+ (run! (fn [^ProcessHandle handle]
+ (when (.isAlive handle)
+ (.destroyForcibly handle)))
+ handles)
+ (wait-for-exit handles kill-grace-period-ms)))
+
(def ^:private command-timeout-ms (* 15 60 1000)) ; 15 minutes
(defn sh*
@@ -41,8 +77,10 @@
* `quiet?` -- whether to suppress output from this shell command.
+ * `timeout-ms` -- how long to wait for the command before killing it and throwing; defaults to 15 minutes.
+
* If you set MAGE_VERBOSE env var to true , the command will be printed before running it."
- {:arglists '([cmd & args] [{:keys [env dir quiet?]} cmd & args])}
+ {:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}
[& args]
(when (u/env "MAGE_VERBOSE" (constantly nil))
(println (str "$ " (str/join " " (map (comp pr-str str) (if (map? (first args))
@@ -54,7 +92,8 @@
opts (merge
{:dir u/project-root-directory}
opts)
- {:keys [env dir]} opts
+ {:keys [env dir timeout-ms]
+ :or {timeout-ms command-timeout-ms}} opts
cmd-array (into-array (map str args))
env-array (when env
(assert (map? env))
@@ -72,16 +111,21 @@
(let [exit-code (future (.waitFor proc))
out (future (read-lines out-reader opts))
err (future (read-lines err-reader opts))]
- {:exit (deref-with-timeout exit-code command-timeout-ms)
- :out (deref-with-timeout out command-timeout-ms)
- :err (deref-with-timeout err command-timeout-ms)}))))
+ (try
+ {:exit (deref-with-timeout exit-code timeout-ms)
+ :out (deref-with-timeout out timeout-ms)
+ :err (deref-with-timeout err timeout-ms)}
+ (catch clojure.lang.ExceptionInfo e
+ (when (:timed-out? (ex-data e))
+ (kill-process! proc))
+ (throw e)))))))
(defn sh
"Like [[sh*]], but throws an Exception if the command exits with a non-zero status. Options are the same as `sh*` --
see its documentation for more information.
Returns sequence of output lines."
- {:arglists '([cmd & args] [{:keys [env dir quiet?]} cmd & args])}
+ {:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}
[& args]
(let [{:keys [exit out err], :as response} (apply sh* args)]
(if (zero? exit)
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index 937d0f912fd..b79aa8fc82f 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -15,6 +15,7 @@
[mage.kondo-ratchet-test]
[mage.merge-yaml-migrations-test :as merge-yaml-migrations-test]
[mage.modules-test]
+ [mage.shell-test]
[mage.token-scan-test]
[mage.util :as u]
[mage.util-test]))
@@ -26,6 +27,7 @@
mage.kondo-ratchet-test/keep-me
mage.util-test/keep-me
mage.modules-test/keep-me
+ mage.shell-test/keep-me
merge-yaml-migrations-test/keep-me
token-scan-test/keep-me)
diff --git a/mage/test/mage/shell_test.clj b/mage/test/mage/shell_test.clj
new file mode 100644
index 00000000000..34a88d72740
--- /dev/null
+++ b/mage/test/mage/shell_test.clj
@@ -0,0 +1,60 @@
+(ns mage.shell-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is]]
+ [mage.shell :as shell])
+ (:import
+ (java.io File)))
+
+;; Referenced by core_test.clj to ensure namespace is loaded
+(def keep-me :loaded)
+
+(set! *warn-on-reflection* true)
+
+(def ^:private ns-per-ms 1000000)
+
+;; How long a killed process gets to disappear from `ps` before we call it a survivor.
+(def ^:private gone-timeout-ms (* 2 1000)) ; 2 seconds
+
+(def ^:private gone-poll-interval-ms 50)
+
+(defn- gone?
+ "Whether the process `pid` has exited within [[gone-timeout-ms]], once its parent has reaped it."
+ [pid]
+ (let [deadline (+ (System/nanoTime) (* gone-timeout-ms ns-per-ms))]
+ (loop []
+ (let [state (str/trim (str/join (:out (shell/sh* {:quiet? true} "ps" "-o" "stat=" "-p" (str pid)))))]
+ (cond
+ (or (str/blank? state) (str/starts-with? state "Z")) true
+ (< (System/nanoTime) deadline) (do (Thread/sleep gone-poll-interval-ms) (recur))
+ :else false)))))
+
+;; Short, so the test spends no longer than it has to waiting for the timeout to fire.
+(def ^:private command-timeout-ms 200)
+
+;; Long enough that a run which merely waits the command out cannot come in under [[kill-bound-ms]].
+(def ^:private grandchild-sleep-seconds 30)
+
+;; The timeout plus both of `kill-process!`'s grace periods, with slack for a loaded machine.
+(def ^:private kill-bound-ms (* 15 1000)) ; 15 seconds
+
+(deftest timeout-kills-the-command-and-its-descendants-test
+ (let [pids (File/createTempFile "mage-shell-timeout" ".pids")
+ started (System/nanoTime)]
+ (try
+ ;; The sleep is a grandchild that ignores SIGTERM, so it only dies when the kill is aimed at it
+ ;; directly, after its parent shell has already gone.
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo (re-pattern (str "Timed out after " command-timeout-ms " ms"))
+ (shell/sh* {:quiet? true, :timeout-ms command-timeout-ms}
+ "sh" "-c"
+ (str "echo $$ > \"$0\"; sh -c 'trap \"\" TERM; sleep "
+ grandchild-sleep-seconds "' & echo $! >> \"$0\"; wait")
+ (str pids))))
+ (is (< (quot (- (System/nanoTime) started) ns-per-ms) kill-bound-ms)
+ "the timed-out command was waited out instead of killed")
+ (let [[root child] (map parse-long (str/split-lines (slurp pids)))]
+ (is (some? child) "the command did not record its descendant")
+ (is (gone? root) "the timed-out shell was left running")
+ (is (gone? child) "the shell's descendant was left running"))
+ (finally
+ (.delete pids)))))