The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Stop timed-out mage commands and their descendants (#81463)
Chris Truterd7a519ab6072
diff --git a/mage/src/mage/shell.clj b/mage/src/mage/shell.cljindex ab6926f608e..a1dfbbb8eaa 100644--- a/mage/src/mage/shell.clj+++ b/mage/src/mage/shell.clj@@ -3,7 +3,8 @@[clojure.string :as str][mage.util :as u])(:import- (java.io BufferedReader File InputStreamReader)))+ (java.io BufferedReader File InputStreamReader)+ (java.lang ProcessHandle)))(set! *warn-on-reflection* true)@@ -19,9 +20,44 @@(defn- deref-with-timeout [dereffable timeout-ms](let [result (deref dereffable timeout-ms ::timed-out)](when (= result ::timed-out)- (throw (ex-info (format "Timed out after %d ms." timeout-ms) {})))+ (throw (ex-info (format "Timed out after %d ms." timeout-ms) {:timed-out? true})))result))+(def ^:private ns-per-ms 1000000)++;; How long a signalled process gets to wind itself down before the next, harsher signal.+(def ^:private kill-grace-period-ms (* 5 1000)) ; 5 seconds++;; How often we re-ask a signalled process whether it has exited yet.+(def ^:private exit-poll-interval-ms 50)++(defn- wait-for-exit+ "Wait up to `timeout-ms` for every process in `handles` to exit."+ [handles timeout-ms]+ ;; nanoTime rather than wall-clock time, so an NTP correction or a DST shift cannot move the deadline.+ (let [deadline (+ (System/nanoTime) (* timeout-ms ns-per-ms))]+ (loop []+ (when (and (some #(.isAlive ^ProcessHandle %) handles)+ (< (System/nanoTime) deadline))+ (Thread/sleep exit-poll-interval-ms)+ (recur)))))++(defn- kill-process!+ "Stop `proc` and every process it started, so a timed-out command cannot keep running behind the caller.+ The descendants are listed before anything is signalled, and each survivor is force-killed on its own,+ so a child that ignores the first signal dies even after its parent has gone.+ A child that had already outlived its parent is not reachable from the root handle."+ [^Process proc]+ (let [root (.toHandle proc)+ handles (cons root (iterator-seq (.iterator (.descendants root))))]+ (run! #(.destroy ^ProcessHandle %) handles)+ (wait-for-exit handles kill-grace-period-ms)+ (run! (fn [^ProcessHandle handle]+ (when (.isAlive handle)+ (.destroyForcibly handle)))+ handles)+ (wait-for-exit handles kill-grace-period-ms)))+(def ^:private command-timeout-ms (* 15 60 1000)) ; 15 minutes(defn sh*@@ -41,8 +77,10 @@* `quiet?` -- whether to suppress output from this shell command.+ * `timeout-ms` -- how long to wait for the command before killing it and throwing; defaults to 15 minutes.+* If you set MAGE_VERBOSE env var to true , the command will be printed before running it."- {:arglists '([cmd & args] [{:keys [env dir quiet?]} cmd & args])}+ {:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}[& args](when (u/env "MAGE_VERBOSE" (constantly nil))(println (str "$ " (str/join " " (map (comp pr-str str) (if (map? (first args))@@ -54,7 +92,8 @@opts (merge{:dir u/project-root-directory}opts)- {:keys [env dir]} opts+ {:keys [env dir timeout-ms]+ :or {timeout-ms command-timeout-ms}} optscmd-array (into-array (map str args))env-array (when env(assert (map? env))@@ -72,16 +111,21 @@(let [exit-code (future (.waitFor proc))out (future (read-lines out-reader opts))err (future (read-lines err-reader opts))]- {:exit (deref-with-timeout exit-code command-timeout-ms)- :out (deref-with-timeout out command-timeout-ms)- :err (deref-with-timeout err command-timeout-ms)}))))+ (try+ {:exit (deref-with-timeout exit-code timeout-ms)+ :out (deref-with-timeout out timeout-ms)+ :err (deref-with-timeout err timeout-ms)}+ (catch clojure.lang.ExceptionInfo e+ (when (:timed-out? (ex-data e))+ (kill-process! proc))+ (throw e)))))))(defn sh"Like [[sh*]], but throws an Exception if the command exits with a non-zero status. Options are the same as `sh*` --see its documentation for more information.Returns sequence of output lines."- {:arglists '([cmd & args] [{:keys [env dir quiet?]} cmd & args])}+ {:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}[& args](let [{:keys [exit out err], :as response} (apply sh* args)](if (zero? exit)diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex 937d0f912fd..b79aa8fc82f 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -15,6 +15,7 @@[mage.kondo-ratchet-test][mage.merge-yaml-migrations-test :as merge-yaml-migrations-test][mage.modules-test]+ [mage.shell-test][mage.token-scan-test][mage.util :as u][mage.util-test]))@@ -26,6 +27,7 @@mage.kondo-ratchet-test/keep-memage.util-test/keep-memage.modules-test/keep-me+ mage.shell-test/keep-memerge-yaml-migrations-test/keep-metoken-scan-test/keep-me)diff --git a/mage/test/mage/shell_test.clj b/mage/test/mage/shell_test.cljnew file mode 100644index 00000000000..34a88d72740--- /dev/null+++ b/mage/test/mage/shell_test.clj@@ -0,0 +1,60 @@+(ns mage.shell-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer [deftest is]]+ [mage.shell :as shell])+ (:import+ (java.io File)))++;; Referenced by core_test.clj to ensure namespace is loaded+(def keep-me :loaded)++(set! *warn-on-reflection* true)++(def ^:private ns-per-ms 1000000)++;; How long a killed process gets to disappear from `ps` before we call it a survivor.+(def ^:private gone-timeout-ms (* 2 1000)) ; 2 seconds++(def ^:private gone-poll-interval-ms 50)++(defn- gone?+ "Whether the process `pid` has exited within [[gone-timeout-ms]], once its parent has reaped it."+ [pid]+ (let [deadline (+ (System/nanoTime) (* gone-timeout-ms ns-per-ms))]+ (loop []+ (let [state (str/trim (str/join (:out (shell/sh* {:quiet? true} "ps" "-o" "stat=" "-p" (str pid)))))]+ (cond+ (or (str/blank? state) (str/starts-with? state "Z")) true+ (< (System/nanoTime) deadline) (do (Thread/sleep gone-poll-interval-ms) (recur))+ :else false)))))++;; Short, so the test spends no longer than it has to waiting for the timeout to fire.+(def ^:private command-timeout-ms 200)++;; Long enough that a run which merely waits the command out cannot come in under [[kill-bound-ms]].+(def ^:private grandchild-sleep-seconds 30)++;; The timeout plus both of `kill-process!`'s grace periods, with slack for a loaded machine.+(def ^:private kill-bound-ms (* 15 1000)) ; 15 seconds++(deftest timeout-kills-the-command-and-its-descendants-test+ (let [pids (File/createTempFile "mage-shell-timeout" ".pids")+ started (System/nanoTime)]+ (try+ ;; The sleep is a grandchild that ignores SIGTERM, so it only dies when the kill is aimed at it+ ;; directly, after its parent shell has already gone.+ (is (thrown-with-msg? clojure.lang.ExceptionInfo (re-pattern (str "Timed out after " command-timeout-ms " ms"))+ (shell/sh* {:quiet? true, :timeout-ms command-timeout-ms}+ "sh" "-c"+ (str "echo $$ > \"$0\"; sh -c 'trap \"\" TERM; sleep "+ grandchild-sleep-seconds "' & echo $! >> \"$0\"; wait")+ (str pids))))+ (is (< (quot (- (System/nanoTime) started) ns-per-ms) kill-bound-ms)+ "the timed-out command was waited out instead of killed")+ (let [[root child] (map parse-long (str/split-lines (slurp pids)))]+ (is (some? child) "the command did not record its descendant")+ (is (gone? root) "the timed-out shell was left running")+ (is (gone? child) "the shell's descendant was left running"))+ (finally+ (.delete pids)))))