Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Prevent mage commands hanging on orphaned output pipes (#81699)

Chris Truterb57dc3f98e95
diff --git a/mage/src/mage/bot/preflight.clj b/mage/src/mage/bot/preflight.clj
index 633b627f3d4..43efbe9eab1 100644
--- a/mage/src/mage/bot/preflight.clj
+++ b/mage/src/mage/bot/preflight.clj
@@ -184,13 +184,13 @@
[port]
(let [url (str "http://localhost:" port "/api/health")
timeout-ms (* 5 60 1000)
- interval 10000
- start (System/currentTimeMillis)]
+ interval (* 10 1000)
+ start (System/nanoTime)]
(loop []
(let [{:keys [exit out]} (shell/sh* {:quiet? true}
"curl" "-s" "-o" "/dev/null" "-w" "%{http_code}" url)
healthy? (and (zero? exit) (= (str/trim (str/join out)) "200"))
- elapsed (- (System/currentTimeMillis) start)]
+ elapsed (quot (- (System/nanoTime) start) 1000000)] ; nanos -> millis
(cond
healthy?
(println (c/green (str "Backend healthy on port " port)))
@@ -234,4 +234,3 @@
(check-playwright!)
(check-node-modules!)
(println (c/green "All preflight checks passed."))))
-
diff --git a/mage/src/mage/shell.clj b/mage/src/mage/shell.clj
index a1dfbbb8eaa..2940da99594 100644
--- a/mage/src/mage/shell.clj
+++ b/mage/src/mage/shell.clj
@@ -8,78 +8,124 @@
(set! *warn-on-reflection* true)
-(defn- read-lines [^java.io.BufferedReader reader {:keys [quiet?]}]
+(defn- start-process!
+ "Starts `args` as a subprocess, using `dir` as its working directory when provided.
+ When non-nil, `env` must be a map and completely replaces the parent environment."
+ ^Process [args env dir]
+ (let [builder (ProcessBuilder. ^java.util.List (mapv str args))]
+ (when dir
+ (.directory builder (File. ^String dir)))
+ (when env
+ (assert (map? env))
+ (doto (.environment builder)
+ (.clear)
+ (.putAll (into {} (map (fn [[k v]] [(name k) (str v)])) env))))
+ (.start builder)))
+
+(defn- read-lines
+ "Reads `reader` to the end and returns its lines, printing each one unless `quiet?` is true."
+ [^BufferedReader reader {:keys [quiet?]}]
(loop [lines []]
(if-let [line (.readLine reader)]
(do
(when-not quiet?
- (println line))
+ ;; `println` writes the text and newline separately, so lock around both writes.
+ ;; The lock is the destination itself, so commands printing to the same place take turns
+ ;; and commands printing elsewhere do not wait on each other.
+ (locking *out*
+ (println line)))
(recur (conj lines line)))
lines)))
-(defn- deref-with-timeout [dereffable timeout-ms]
- (let [result (deref dereffable timeout-ms ::timed-out)]
- (when (= result ::timed-out)
- (throw (ex-info (format "Timed out after %d ms." timeout-ms) {:timed-out? true})))
- result))
-
(def ^:private ns-per-ms 1000000)
;; How long a signalled process gets to wind itself down before the next, harsher signal.
-(def ^:private kill-grace-period-ms (* 5 1000)) ; 5 seconds
+(def ^:private kill-grace-period-ms (* 5 1000))
-;; How often we re-ask a signalled process whether it has exited yet.
+;; How often we check whether a signalled process has exited.
(def ^:private exit-poll-interval-ms 50)
+;; After a command exits, its output readers receive at least this much time to finish.
+(def ^:private drain-floor-ms (* 2 1000))
+
+(def ^:private drain-failed-message
+ ;; Returning partial output as complete would hide data loss, so this is an error.
+ "Command exited, but something it left running is holding its output pipe open.")
+
+(defn- deadline-in
+ "Returns the `System/nanoTime` deadline `timeout-ms` milliseconds from now."
+ [timeout-ms]
+ ;; A monotonic reading, so an NTP correction or a manual clock change cannot move the deadline.
+ (+ (System/nanoTime) (* timeout-ms ns-per-ms)))
+
+(defn- remaining-ms
+ "Returns the milliseconds left before `deadline`, or zero once it has passed."
+ [deadline]
+ (max 0 (quot (- deadline (System/nanoTime)) ns-per-ms)))
+
+(defn- deref-by-deadline
+ "Returns the value of `dereffable` if it becomes available by `deadline`.
+ Throws `ExceptionInfo` with `message` and `:timed-out? true` if the deadline passes first."
+ [dereffable deadline message]
+ (let [result (deref dereffable (remaining-ms deadline) ::timed-out)]
+ (when (= result ::timed-out)
+ (throw (ex-info message {:timed-out? true})))
+ result))
+
+(defn- close-quietly [^java.io.Closeable closeable]
+ (try
+ (.close closeable)
+ (catch java.io.IOException _ nil)))
+
(defn- wait-for-exit
- "Wait up to `timeout-ms` for every process in `handles` to exit."
+ "Waits until every process in `handles` exits or `timeout-ms` elapses."
[handles timeout-ms]
- ;; nanoTime rather than wall-clock time, so an NTP correction or a DST shift cannot move the deadline.
- (let [deadline (+ (System/nanoTime) (* timeout-ms ns-per-ms))]
+ (let [deadline (deadline-in timeout-ms)]
(loop []
(when (and (some #(.isAlive ^ProcessHandle %) handles)
- (< (System/nanoTime) deadline))
+ (pos? (remaining-ms deadline)))
(Thread/sleep exit-poll-interval-ms)
(recur)))))
(defn- kill-process!
- "Stop `proc` and every process it started, so a timed-out command cannot keep running behind the caller.
- The descendants are listed before anything is signalled, and each survivor is force-killed on its own,
- so a child that ignores the first signal dies even after its parent has gone.
- A child that had already outlived its parent is not reachable from the root handle."
+ "Attempts to stop `proc` and every descendant reachable from it when this function begins.
+ Returns after every captured process exits or after two grace periods.
+ A process that has already outlived its parent is unreachable and cannot be stopped."
[^Process proc]
(let [root (.toHandle proc)
+ ;; Capture descendants before signalling the root, which can make them unreachable when it exits.
handles (cons root (iterator-seq (.iterator (.descendants root))))]
(run! #(.destroy ^ProcessHandle %) handles)
(wait-for-exit handles kill-grace-period-ms)
+ ;; Signal every survivor directly because its parent may already have exited.
(run! (fn [^ProcessHandle handle]
(when (.isAlive handle)
(.destroyForcibly handle)))
handles)
(wait-for-exit handles kill-grace-period-ms)))
-(def ^:private command-timeout-ms (* 15 60 1000)) ; 15 minutes
+(def ^:private command-timeout-ms (* 15 60 1000))
(defn sh*
- "Run a shell command. Like [[clojure.java.shell/sh]], but prints output to stdout/stderr and returns a map with keys
- `:exit`, `:out`, and `:err` (`:out` and `:err` are vectors of lines). Does not throw Exception if process exits with
- non-zero status code.
+ "Runs a command and returns a map with `:exit`, `:out`, and `:err`; output values are vectors of lines.
+ Unlike [[clojure.java.shell/sh]], prints output as it arrives and does not throw for a nonzero exit status.
Options:
- * `env` -- environment variables (as a map) to use when running `cmd`. If `:env` is `nil`, the default parent
- environment (i.e., the environment in which this Clojure code itself is ran) will be used; if `:env` IS passed, it
- completely replaces the parent environment in which this script is ran -- make sure you pass anything that might be
- needed such as `JAVA_HOME` and `PATH` if you do this
+ * `:env` replaces the parent environment when it is a map; `nil` uses the parent environment.
+ Include variables such as `JAVA_HOME` and `PATH` when replacing the environment.
- * `dir` -- current directory to use when running the shell command. If not specified, command is run in the repo
- root directory.
+ * `:dir` sets the working directory and defaults to the repository root.
- * `quiet?` -- whether to suppress output from this shell command.
+ * `:quiet?` suppresses output while the command runs.
- * `timeout-ms` -- how long to wait for the command before killing it and throwing; defaults to 15 minutes.
+ * `:timeout-ms` is the total budget for the command to exit and for stdout and stderr to be collected.
+ It defaults to 15 minutes.
+ On timeout, kills the command and its reachable descendants and throws `ExceptionInfo`.
+ After the command exits, allows at least two seconds to finish collecting its output.
+ Throws rather than return partial output if another process keeps an output pipe open.
- * If you set MAGE_VERBOSE env var to true , the command will be printed before running it."
+ When `MAGE_VERBOSE` is present in the environment, prints the command before running it."
{:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}
[& args]
(when (u/env "MAGE_VERBOSE" (constantly nil))
@@ -92,39 +138,48 @@
opts (merge
{:dir u/project-root-directory}
opts)
- {:keys [env dir timeout-ms]
- :or {timeout-ms command-timeout-ms}} opts
- cmd-array (into-array (map str args))
- env-array (when env
- (assert (map? env))
- (into-array String (for [[k v] env]
- (format "%s=%s" (name k) (str v)))))
- proc (.exec (Runtime/getRuntime)
- ^"[Ljava.lang.String;" cmd-array
- ^"[Ljava.lang.String;" env-array
- ^File (when dir (File. ^String dir)))]
- ;; Close child stdin so subprocesses that read from it see EOF immediately
- ;; instead of blocking forever on the JVM-owned pipe.
+ {:keys [env dir]} opts
+ ;; `or` selects the default for a missing key or an explicit nil.
+ ;; Destructuring `:or` only covers a missing key.
+ timeout-ms (or (:timeout-ms opts) command-timeout-ms)
+ proc (start-process! args env dir)]
+ ;; Closing child stdin tells subprocesses that no input is coming, so they do not wait forever.
(.close (.getOutputStream proc))
- (with-open [out-reader (BufferedReader. (InputStreamReader. (.getInputStream proc)))
- err-reader (BufferedReader. (InputStreamReader. (.getErrorStream proc)))]
- (let [exit-code (future (.waitFor proc))
- out (future (read-lines out-reader opts))
- err (future (read-lines err-reader opts))]
- (try
- {:exit (deref-with-timeout exit-code timeout-ms)
- :out (deref-with-timeout out timeout-ms)
- :err (deref-with-timeout err timeout-ms)}
- (catch clojure.lang.ExceptionInfo e
- (when (:timed-out? (ex-data e))
- (kill-process! proc))
- (throw e)))))))
+ (let [out-stream (.getInputStream proc)
+ err-stream (.getErrorStream proc)
+ out-reader (BufferedReader. (InputStreamReader. out-stream))
+ err-reader (BufferedReader. (InputStreamReader. err-stream))
+ ;; The command exit and both output reads share one deadline.
+ deadline (deadline-in timeout-ms)
+ exit-code (future (.waitFor proc))
+ out (future (read-lines out-reader opts))
+ err (future (read-lines err-reader opts))]
+ (try
+ (let [exit (deref-by-deadline exit-code deadline (format "Timed out after %d ms." timeout-ms))
+ ;; A process can exit near its deadline while readers still have buffered output to consume.
+ ;; Give them a small minimum window so a successful command is not mistaken for a timeout.
+ drain (max deadline (deadline-in drain-floor-ms))]
+ {:exit exit
+ :out (deref-by-deadline out drain drain-failed-message)
+ :err (deref-by-deadline err drain drain-failed-message)})
+ ;; Any exception can leave the command running, including a reader failure or thread interruption.
+ ;; Stop it whenever it is still alive, regardless of which exception reached us.
+ (catch Throwable e
+ (when (.isAlive proc)
+ (kill-process! proc))
+ (throw e))
+ (finally
+ ;; Close the streams rather than the readers.
+ ;; `BufferedReader.close` waits for the lock a blocked `readLine` holds, so it cannot return until EOF.
+ ;; A process left holding the pipe can withhold EOF for as long as it lives; closing the stream
+ ;; underneath ends the read at once.
+ (close-quietly out-stream)
+ (close-quietly err-stream))))))
(defn sh
- "Like [[sh*]], but throws an Exception if the command exits with a non-zero status. Options are the same as `sh*` --
- see its documentation for more information.
-
- Returns sequence of output lines."
+ "Runs [[sh*]] and returns its stdout lines followed by its stderr lines.
+ Throws `ExceptionInfo` containing the response and command arguments when the exit status is nonzero.
+ Accepts the same options as [[sh*]]."
{:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}
[& args]
(let [{:keys [exit out err], :as response} (apply sh* args)]
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index 22cd68bbd96..6ca8321c14e 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -1,4 +1,7 @@
(ns mage.core-test
+ "Tests for the mage CLI itself.
+ Requires every other mage test namespace: `./bin/mage -test` runs `clojure.test/run-all-tests`, which
+ only sees namespaces that are already loaded."
(:require
[babashka.fs :as fs]
[babashka.tasks :as bt]
@@ -9,12 +12,14 @@
[mage.bot.autobot-test]
[mage.bot.dev-env-core-test]
[mage.bot.env-test]
+ [mage.bot.pr-env-test]
[mage.bot.prompt-test]
+ [mage.bot.repl-eval-test]
[mage.doctor-test]
[mage.fix-unused-requires-test]
[mage.kondo-ratchet-test]
[mage.merge-kondo-ratchets-test]
- [mage.merge-yaml-migrations-test :as merge-yaml-migrations-test]
+ [mage.merge-yaml-migrations-test]
[mage.modules-test]
[mage.project-tests-test]
[mage.shell-test]
@@ -22,19 +27,6 @@
[mage.util :as u]
[mage.util-test]))
-(comment
- ;; Load test namespaces to ensure code coverage
- mage.doctor-test/keep-me
- mage.fix-unused-requires-test/keep-me
- mage.kondo-ratchet-test/keep-me
- mage.merge-kondo-ratchets-test/keep-me
- mage.util-test/keep-me
- mage.modules-test/keep-me
- mage.project-tests-test/keep-me
- mage.shell-test/keep-me
- merge-yaml-migrations-test/keep-me
- token-scan-test/keep-me)
-
(set! *warn-on-reflection* true)
(deftest bin-mage-has-help-test
diff --git a/mage/test/mage/shell_test.clj b/mage/test/mage/shell_test.clj
index 34a88d72740..2b54b07cb59 100644
--- a/mage/test/mage/shell_test.clj
+++ b/mage/test/mage/shell_test.clj
@@ -6,20 +6,20 @@
(:import
(java.io File)))
-;; Referenced by core_test.clj to ensure namespace is loaded
+;; core_test.clj references this var to ensure this namespace is loaded.
(def keep-me :loaded)
(set! *warn-on-reflection* true)
(def ^:private ns-per-ms 1000000)
-;; How long a killed process gets to disappear from `ps` before we call it a survivor.
-(def ^:private gone-timeout-ms (* 2 1000)) ; 2 seconds
+;; The test waits this long for a killed process to disappear from `ps`.
+(def ^:private gone-timeout-ms (* 2 1000))
(def ^:private gone-poll-interval-ms 50)
(defn- gone?
- "Whether the process `pid` has exited within [[gone-timeout-ms]], once its parent has reaped it."
+ "Returns true if `pid` exits or becomes a zombie within [[gone-timeout-ms]]."
[pid]
(let [deadline (+ (System/nanoTime) (* gone-timeout-ms ns-per-ms))]
(loop []
@@ -29,21 +29,22 @@
(< (System/nanoTime) deadline) (do (Thread/sleep gone-poll-interval-ms) (recur))
:else false)))))
-;; Short, so the test spends no longer than it has to waiting for the timeout to fire.
-(def ^:private command-timeout-ms 200)
+;; The timeout keeps the test quick while still giving a loaded runner time to fork twice and record
+;; both pids before the kill.
+(def ^:private command-timeout-ms 1000)
-;; Long enough that a run which merely waits the command out cannot come in under [[kill-bound-ms]].
+;; The grandchild lives long enough that waiting it out cannot finish within [[kill-bound-ms]].
(def ^:private grandchild-sleep-seconds 30)
-;; The timeout plus both of `kill-process!`'s grace periods, with slack for a loaded machine.
-(def ^:private kill-bound-ms (* 15 1000)) ; 15 seconds
+;; The command's timeout plus both grace periods, with room to spare on a loaded machine.
+(def ^:private kill-bound-ms (* 15 1000))
(deftest timeout-kills-the-command-and-its-descendants-test
(let [pids (File/createTempFile "mage-shell-timeout" ".pids")
started (System/nanoTime)]
(try
- ;; The sleep is a grandchild that ignores SIGTERM, so it only dies when the kill is aimed at it
- ;; directly, after its parent shell has already gone.
+ ;; The sleep is a grandchild that inherits its parent shell's decision to ignore SIGTERM.
+ ;; Its parent may exit before the forced-kill pass, so the kill must be aimed directly at the sleep.
(is (thrown-with-msg? clojure.lang.ExceptionInfo (re-pattern (str "Timed out after " command-timeout-ms " ms"))
(shell/sh* {:quiet? true, :timeout-ms command-timeout-ms}
"sh" "-c"
@@ -58,3 +59,32 @@
(is (gone? child) "the shell's descendant was left running"))
(finally
(.delete pids)))))
+
+;; The orphan outlives the command by design.
+;; Once it is reparented, the command's process handle can no longer reach it, so the test records its
+;; pid for cleanup.
+(def ^:private orphan-sleep-seconds 20)
+
+(def ^:private orphan-timeout-ms 500)
+
+;; The bound sits above the drain floor but far below the orphan's lifetime.
+;; If `sh*` exceeds it, it waited for the orphan instead of closing the pipe beneath the blocked read.
+(def ^:private orphan-bound-ms (* 8 1000))
+
+(deftest orphan-holding-the-output-pipe-does-not-hold-the-caller-test
+ (let [pid-file (File/createTempFile "mage-shell-orphan" ".pid")
+ started (System/nanoTime)]
+ (try
+ ;; The command exits at once but leaves an orphan holding stdout open, so the pipe never reaches EOF.
+ ;; Closing the reader would wait out the orphan's full 20 seconds, far past [[orphan-bound-ms]].
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"holding its output pipe open"
+ (shell/sh* {:quiet? true, :timeout-ms orphan-timeout-ms}
+ "sh" "-c"
+ (str "sleep " orphan-sleep-seconds " & echo $! > \"$0\"; exit 0")
+ (str pid-file))))
+ (is (< (quot (- (System/nanoTime) started) ns-per-ms) orphan-bound-ms)
+ "sh* waited out the orphan instead of closing the pipe")
+ (finally
+ (when-let [pid (parse-long (str/trim (slurp pid-file)))]
+ (shell/sh* {:quiet? true} "kill" "-9" (str pid)))
+ (.delete pid-file)))))