The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Prevent mage commands hanging on orphaned output pipes (#81699)
Chris Truterb57dc3f98e95
diff --git a/mage/src/mage/bot/preflight.clj b/mage/src/mage/bot/preflight.cljindex 633b627f3d4..43efbe9eab1 100644--- a/mage/src/mage/bot/preflight.clj+++ b/mage/src/mage/bot/preflight.clj@@ -184,13 +184,13 @@[port](let [url (str "http://localhost:" port "/api/health")timeout-ms (* 5 60 1000)- interval 10000- start (System/currentTimeMillis)]+ interval (* 10 1000)+ start (System/nanoTime)](loop [](let [{:keys [exit out]} (shell/sh* {:quiet? true}"curl" "-s" "-o" "/dev/null" "-w" "%{http_code}" url)healthy? (and (zero? exit) (= (str/trim (str/join out)) "200"))- elapsed (- (System/currentTimeMillis) start)]+ elapsed (quot (- (System/nanoTime) start) 1000000)] ; nanos -> millis(condhealthy?(println (c/green (str "Backend healthy on port " port)))@@ -234,4 +234,3 @@(check-playwright!)(check-node-modules!)(println (c/green "All preflight checks passed."))))-diff --git a/mage/src/mage/shell.clj b/mage/src/mage/shell.cljindex a1dfbbb8eaa..2940da99594 100644--- a/mage/src/mage/shell.clj+++ b/mage/src/mage/shell.clj@@ -8,78 +8,124 @@(set! *warn-on-reflection* true)-(defn- read-lines [^java.io.BufferedReader reader {:keys [quiet?]}]+(defn- start-process!+ "Starts `args` as a subprocess, using `dir` as its working directory when provided.+ When non-nil, `env` must be a map and completely replaces the parent environment."+ ^Process [args env dir]+ (let [builder (ProcessBuilder. ^java.util.List (mapv str args))]+ (when dir+ (.directory builder (File. ^String dir)))+ (when env+ (assert (map? env))+ (doto (.environment builder)+ (.clear)+ (.putAll (into {} (map (fn [[k v]] [(name k) (str v)])) env))))+ (.start builder)))++(defn- read-lines+ "Reads `reader` to the end and returns its lines, printing each one unless `quiet?` is true."+ [^BufferedReader reader {:keys [quiet?]}](loop [lines []](if-let [line (.readLine reader)](do(when-not quiet?- (println line))+ ;; `println` writes the text and newline separately, so lock around both writes.+ ;; The lock is the destination itself, so commands printing to the same place take turns+ ;; and commands printing elsewhere do not wait on each other.+ (locking *out*+ (println line)))(recur (conj lines line)))lines)))-(defn- deref-with-timeout [dereffable timeout-ms]- (let [result (deref dereffable timeout-ms ::timed-out)]- (when (= result ::timed-out)- (throw (ex-info (format "Timed out after %d ms." timeout-ms) {:timed-out? true})))- result))-(def ^:private ns-per-ms 1000000);; How long a signalled process gets to wind itself down before the next, harsher signal.-(def ^:private kill-grace-period-ms (* 5 1000)) ; 5 seconds+(def ^:private kill-grace-period-ms (* 5 1000))-;; How often we re-ask a signalled process whether it has exited yet.+;; How often we check whether a signalled process has exited.(def ^:private exit-poll-interval-ms 50)+;; After a command exits, its output readers receive at least this much time to finish.+(def ^:private drain-floor-ms (* 2 1000))++(def ^:private drain-failed-message+ ;; Returning partial output as complete would hide data loss, so this is an error.+ "Command exited, but something it left running is holding its output pipe open.")++(defn- deadline-in+ "Returns the `System/nanoTime` deadline `timeout-ms` milliseconds from now."+ [timeout-ms]+ ;; A monotonic reading, so an NTP correction or a manual clock change cannot move the deadline.+ (+ (System/nanoTime) (* timeout-ms ns-per-ms)))++(defn- remaining-ms+ "Returns the milliseconds left before `deadline`, or zero once it has passed."+ [deadline]+ (max 0 (quot (- deadline (System/nanoTime)) ns-per-ms)))++(defn- deref-by-deadline+ "Returns the value of `dereffable` if it becomes available by `deadline`.+ Throws `ExceptionInfo` with `message` and `:timed-out? true` if the deadline passes first."+ [dereffable deadline message]+ (let [result (deref dereffable (remaining-ms deadline) ::timed-out)]+ (when (= result ::timed-out)+ (throw (ex-info message {:timed-out? true})))+ result))++(defn- close-quietly [^java.io.Closeable closeable]+ (try+ (.close closeable)+ (catch java.io.IOException _ nil)))+(defn- wait-for-exit- "Wait up to `timeout-ms` for every process in `handles` to exit."+ "Waits until every process in `handles` exits or `timeout-ms` elapses."[handles timeout-ms]- ;; nanoTime rather than wall-clock time, so an NTP correction or a DST shift cannot move the deadline.- (let [deadline (+ (System/nanoTime) (* timeout-ms ns-per-ms))]+ (let [deadline (deadline-in timeout-ms)](loop [](when (and (some #(.isAlive ^ProcessHandle %) handles)- (< (System/nanoTime) deadline))+ (pos? (remaining-ms deadline)))(Thread/sleep exit-poll-interval-ms)(recur)))))(defn- kill-process!- "Stop `proc` and every process it started, so a timed-out command cannot keep running behind the caller.- The descendants are listed before anything is signalled, and each survivor is force-killed on its own,- so a child that ignores the first signal dies even after its parent has gone.- A child that had already outlived its parent is not reachable from the root handle."+ "Attempts to stop `proc` and every descendant reachable from it when this function begins.+ Returns after every captured process exits or after two grace periods.+ A process that has already outlived its parent is unreachable and cannot be stopped."[^Process proc](let [root (.toHandle proc)+ ;; Capture descendants before signalling the root, which can make them unreachable when it exits.handles (cons root (iterator-seq (.iterator (.descendants root))))](run! #(.destroy ^ProcessHandle %) handles)(wait-for-exit handles kill-grace-period-ms)+ ;; Signal every survivor directly because its parent may already have exited.(run! (fn [^ProcessHandle handle](when (.isAlive handle)(.destroyForcibly handle)))handles)(wait-for-exit handles kill-grace-period-ms)))-(def ^:private command-timeout-ms (* 15 60 1000)) ; 15 minutes+(def ^:private command-timeout-ms (* 15 60 1000))(defn sh*- "Run a shell command. Like [[clojure.java.shell/sh]], but prints output to stdout/stderr and returns a map with keys- `:exit`, `:out`, and `:err` (`:out` and `:err` are vectors of lines). Does not throw Exception if process exits with- non-zero status code.+ "Runs a command and returns a map with `:exit`, `:out`, and `:err`; output values are vectors of lines.+ Unlike [[clojure.java.shell/sh]], prints output as it arrives and does not throw for a nonzero exit status.Options:- * `env` -- environment variables (as a map) to use when running `cmd`. If `:env` is `nil`, the default parent- environment (i.e., the environment in which this Clojure code itself is ran) will be used; if `:env` IS passed, it- completely replaces the parent environment in which this script is ran -- make sure you pass anything that might be- needed such as `JAVA_HOME` and `PATH` if you do this+ * `:env` replaces the parent environment when it is a map; `nil` uses the parent environment.+ Include variables such as `JAVA_HOME` and `PATH` when replacing the environment.- * `dir` -- current directory to use when running the shell command. If not specified, command is run in the repo- root directory.+ * `:dir` sets the working directory and defaults to the repository root.- * `quiet?` -- whether to suppress output from this shell command.+ * `:quiet?` suppresses output while the command runs.- * `timeout-ms` -- how long to wait for the command before killing it and throwing; defaults to 15 minutes.+ * `:timeout-ms` is the total budget for the command to exit and for stdout and stderr to be collected.+ It defaults to 15 minutes.+ On timeout, kills the command and its reachable descendants and throws `ExceptionInfo`.+ After the command exits, allows at least two seconds to finish collecting its output.+ Throws rather than return partial output if another process keeps an output pipe open.- * If you set MAGE_VERBOSE env var to true , the command will be printed before running it."+ When `MAGE_VERBOSE` is present in the environment, prints the command before running it."{:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}[& args](when (u/env "MAGE_VERBOSE" (constantly nil))@@ -92,39 +138,48 @@opts (merge{:dir u/project-root-directory}opts)- {:keys [env dir timeout-ms]- :or {timeout-ms command-timeout-ms}} opts- cmd-array (into-array (map str args))- env-array (when env- (assert (map? env))- (into-array String (for [[k v] env]- (format "%s=%s" (name k) (str v)))))- proc (.exec (Runtime/getRuntime)- ^"[Ljava.lang.String;" cmd-array- ^"[Ljava.lang.String;" env-array- ^File (when dir (File. ^String dir)))]- ;; Close child stdin so subprocesses that read from it see EOF immediately- ;; instead of blocking forever on the JVM-owned pipe.+ {:keys [env dir]} opts+ ;; `or` selects the default for a missing key or an explicit nil.+ ;; Destructuring `:or` only covers a missing key.+ timeout-ms (or (:timeout-ms opts) command-timeout-ms)+ proc (start-process! args env dir)]+ ;; Closing child stdin tells subprocesses that no input is coming, so they do not wait forever.(.close (.getOutputStream proc))- (with-open [out-reader (BufferedReader. (InputStreamReader. (.getInputStream proc)))- err-reader (BufferedReader. (InputStreamReader. (.getErrorStream proc)))]- (let [exit-code (future (.waitFor proc))- out (future (read-lines out-reader opts))- err (future (read-lines err-reader opts))]- (try- {:exit (deref-with-timeout exit-code timeout-ms)- :out (deref-with-timeout out timeout-ms)- :err (deref-with-timeout err timeout-ms)}- (catch clojure.lang.ExceptionInfo e- (when (:timed-out? (ex-data e))- (kill-process! proc))- (throw e)))))))+ (let [out-stream (.getInputStream proc)+ err-stream (.getErrorStream proc)+ out-reader (BufferedReader. (InputStreamReader. out-stream))+ err-reader (BufferedReader. (InputStreamReader. err-stream))+ ;; The command exit and both output reads share one deadline.+ deadline (deadline-in timeout-ms)+ exit-code (future (.waitFor proc))+ out (future (read-lines out-reader opts))+ err (future (read-lines err-reader opts))]+ (try+ (let [exit (deref-by-deadline exit-code deadline (format "Timed out after %d ms." timeout-ms))+ ;; A process can exit near its deadline while readers still have buffered output to consume.+ ;; Give them a small minimum window so a successful command is not mistaken for a timeout.+ drain (max deadline (deadline-in drain-floor-ms))]+ {:exit exit+ :out (deref-by-deadline out drain drain-failed-message)+ :err (deref-by-deadline err drain drain-failed-message)})+ ;; Any exception can leave the command running, including a reader failure or thread interruption.+ ;; Stop it whenever it is still alive, regardless of which exception reached us.+ (catch Throwable e+ (when (.isAlive proc)+ (kill-process! proc))+ (throw e))+ (finally+ ;; Close the streams rather than the readers.+ ;; `BufferedReader.close` waits for the lock a blocked `readLine` holds, so it cannot return until EOF.+ ;; A process left holding the pipe can withhold EOF for as long as it lives; closing the stream+ ;; underneath ends the read at once.+ (close-quietly out-stream)+ (close-quietly err-stream))))))(defn sh- "Like [[sh*]], but throws an Exception if the command exits with a non-zero status. Options are the same as `sh*` --- see its documentation for more information.-- Returns sequence of output lines."+ "Runs [[sh*]] and returns its stdout lines followed by its stderr lines.+ Throws `ExceptionInfo` containing the response and command arguments when the exit status is nonzero.+ Accepts the same options as [[sh*]]."{:arglists '([cmd & args] [{:keys [env dir quiet? timeout-ms]} cmd & args])}[& args](let [{:keys [exit out err], :as response} (apply sh* args)]diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex 22cd68bbd96..6ca8321c14e 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -1,4 +1,7 @@(ns mage.core-test+ "Tests for the mage CLI itself.+ Requires every other mage test namespace: `./bin/mage -test` runs `clojure.test/run-all-tests`, which+ only sees namespaces that are already loaded."(:require[babashka.fs :as fs][babashka.tasks :as bt]@@ -9,12 +12,14 @@[mage.bot.autobot-test][mage.bot.dev-env-core-test][mage.bot.env-test]+ [mage.bot.pr-env-test][mage.bot.prompt-test]+ [mage.bot.repl-eval-test][mage.doctor-test][mage.fix-unused-requires-test][mage.kondo-ratchet-test][mage.merge-kondo-ratchets-test]- [mage.merge-yaml-migrations-test :as merge-yaml-migrations-test]+ [mage.merge-yaml-migrations-test][mage.modules-test][mage.project-tests-test][mage.shell-test]@@ -22,19 +27,6 @@[mage.util :as u][mage.util-test]))-(comment- ;; Load test namespaces to ensure code coverage- mage.doctor-test/keep-me- mage.fix-unused-requires-test/keep-me- mage.kondo-ratchet-test/keep-me- mage.merge-kondo-ratchets-test/keep-me- mage.util-test/keep-me- mage.modules-test/keep-me- mage.project-tests-test/keep-me- mage.shell-test/keep-me- merge-yaml-migrations-test/keep-me- token-scan-test/keep-me)-(set! *warn-on-reflection* true)(deftest bin-mage-has-help-testdiff --git a/mage/test/mage/shell_test.clj b/mage/test/mage/shell_test.cljindex 34a88d72740..2b54b07cb59 100644--- a/mage/test/mage/shell_test.clj+++ b/mage/test/mage/shell_test.clj@@ -6,20 +6,20 @@(:import(java.io File)))-;; Referenced by core_test.clj to ensure namespace is loaded+;; core_test.clj references this var to ensure this namespace is loaded.(def keep-me :loaded)(set! *warn-on-reflection* true)(def ^:private ns-per-ms 1000000)-;; How long a killed process gets to disappear from `ps` before we call it a survivor.-(def ^:private gone-timeout-ms (* 2 1000)) ; 2 seconds+;; The test waits this long for a killed process to disappear from `ps`.+(def ^:private gone-timeout-ms (* 2 1000))(def ^:private gone-poll-interval-ms 50)(defn- gone?- "Whether the process `pid` has exited within [[gone-timeout-ms]], once its parent has reaped it."+ "Returns true if `pid` exits or becomes a zombie within [[gone-timeout-ms]]."[pid](let [deadline (+ (System/nanoTime) (* gone-timeout-ms ns-per-ms))](loop []@@ -29,21 +29,22 @@(< (System/nanoTime) deadline) (do (Thread/sleep gone-poll-interval-ms) (recur)):else false)))))-;; Short, so the test spends no longer than it has to waiting for the timeout to fire.-(def ^:private command-timeout-ms 200)+;; The timeout keeps the test quick while still giving a loaded runner time to fork twice and record+;; both pids before the kill.+(def ^:private command-timeout-ms 1000)-;; Long enough that a run which merely waits the command out cannot come in under [[kill-bound-ms]].+;; The grandchild lives long enough that waiting it out cannot finish within [[kill-bound-ms]].(def ^:private grandchild-sleep-seconds 30)-;; The timeout plus both of `kill-process!`'s grace periods, with slack for a loaded machine.-(def ^:private kill-bound-ms (* 15 1000)) ; 15 seconds+;; The command's timeout plus both grace periods, with room to spare on a loaded machine.+(def ^:private kill-bound-ms (* 15 1000))(deftest timeout-kills-the-command-and-its-descendants-test(let [pids (File/createTempFile "mage-shell-timeout" ".pids")started (System/nanoTime)](try- ;; The sleep is a grandchild that ignores SIGTERM, so it only dies when the kill is aimed at it- ;; directly, after its parent shell has already gone.+ ;; The sleep is a grandchild that inherits its parent shell's decision to ignore SIGTERM.+ ;; Its parent may exit before the forced-kill pass, so the kill must be aimed directly at the sleep.(is (thrown-with-msg? clojure.lang.ExceptionInfo (re-pattern (str "Timed out after " command-timeout-ms " ms"))(shell/sh* {:quiet? true, :timeout-ms command-timeout-ms}"sh" "-c"@@ -58,3 +59,32 @@(is (gone? child) "the shell's descendant was left running"))(finally(.delete pids)))))++;; The orphan outlives the command by design.+;; Once it is reparented, the command's process handle can no longer reach it, so the test records its+;; pid for cleanup.+(def ^:private orphan-sleep-seconds 20)++(def ^:private orphan-timeout-ms 500)++;; The bound sits above the drain floor but far below the orphan's lifetime.+;; If `sh*` exceeds it, it waited for the orphan instead of closing the pipe beneath the blocked read.+(def ^:private orphan-bound-ms (* 8 1000))++(deftest orphan-holding-the-output-pipe-does-not-hold-the-caller-test+ (let [pid-file (File/createTempFile "mage-shell-orphan" ".pid")+ started (System/nanoTime)]+ (try+ ;; The command exits at once but leaves an orphan holding stdout open, so the pipe never reaches EOF.+ ;; Closing the reader would wait out the orphan's full 20 seconds, far past [[orphan-bound-ms]].+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"holding its output pipe open"+ (shell/sh* {:quiet? true, :timeout-ms orphan-timeout-ms}+ "sh" "-c"+ (str "sleep " orphan-sleep-seconds " & echo $! > \"$0\"; exit 0")+ (str pid-file))))+ (is (< (quot (- (System/nanoTime) started) ns-per-ms) orphan-bound-ms)+ "sh* waited out the orphan instead of closing the pipe")+ (finally+ (when-let [pid (parse-long (str/trim (slurp pid-file)))]+ (shell/sh* {:quiet? true} "kill" "-9" (str pid)))+ (.delete pid-file)))))