Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

`updated-files` returns all-files + Mage cleanup (#66332)

bryan1591300b492d
diff --git a/.github/workflows/mage.yml b/.github/workflows/mage.yml
index 6deefd0c635..7847a965155 100644
--- a/.github/workflows/mage.yml
+++ b/.github/workflows/mage.yml
@@ -2,7 +2,9 @@ name: Mage Tests
on:
pull_request:
- paths: ["mage/**"]
+ paths:
+ - "mage/**"
+ - "bb.edn"
jobs:
# run mage tests even if the rest of be isnt needed
@@ -13,13 +15,7 @@ jobs:
- uses: actions/checkout@v4
- name: Prepare back-end environment
uses: ./.github/actions/prepare-backend
- - name: Setup Babashka
- uses: turtlequeue/[email protected]
- with:
- babashka-version: 1.12.197
- - name: Check bb runs
- run: bb --version
- - name: bb task list
- run: bb tasks
+ - name: Check mage runs
+ run: ./bin/mage -h
- name: Run mage tests
- run: bb ./mage/test/mage/runtests.clj
+ run: ./bin/mage -test
diff --git a/bb.edn b/bb.edn
index 6a50f28aaf6..047a1bc522e 100644
--- a/bb.edn
+++ b/bb.edn
@@ -179,6 +179,8 @@
:requires [[mage.be-dev :refer [nrepl-eval]]]
:options [["-c" "--comment-id ID" "Notify about this comment id"]
["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]
+ :examples [["./bin/mage notify -c 1234" "Notify about comment with id 1234"]
+ ["./bin/mage notify --comment-id 5678 --port 59498" "Notify about comment with id 5678 on nrepl port 59498"]]
:task (let [opts (:options (cli/parse! (current-task)))
;; note: no quoting -> expect it to be int
code (format "(notify-comment-id! %d)" (parse-long (:comment-id opts)))]
@@ -309,14 +311,15 @@
:data {:a [:b :c] :b [:d]}}
-test {:doc "run all mage tests"
- :requires [[mage.core-test]
- [clojure.test :refer [run-tests]]]
- :task (run-tests 'mage.core-test)}
-
- -test-examples
- {:doc "Runs every example and checks that it exits with 0"
- :requires [[mage.examples-test :as examples-test]]
- :task (System/exit (if (= :ok (examples-test/run-tests)) 0 1))}
+ :requires [[mage.core-test] ;; loads the actual tests
+ [mage.util :as u]
+ [clojure.test]]
+ :task (let [results (clojure.test/run-all-tests (re-pattern "^mage\\..*"))]
+ (if (zero? (+ (:fail results) (:error results)))
+ (println (c/green "All mage tests passed!"))
+ (do
+ (println (c/red "Some mage tests failed."))
+ (u/exit 1))))}
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Personal Tasks:
diff --git a/bin/mage.bb b/bin/mage.bb
index 425f114c012..0701fd5b66a 100755
--- a/bin/mage.bb
+++ b/bin/mage.bb
@@ -6,21 +6,14 @@
(ns mage
(:require
[babashka.tasks :as bt]
+ [bling.banner :refer [banner]]
+ [bling.fonts.ansi-shadow :as ansi-shadow]
[clojure.string :as str]
[mage.color :as c]
[mage.util :as u]))
(set! *warn-on-reflection* true)
-(defn- lolcat [f]
- (if (try (u/sh "command -v lolcat")
- (catch Exception _ false))
- (bt/shell (str "lolcat " f))
- (do
- (println (slurp f))
- (when (> (rand) 0.8)
- (println (c/yellow "Tip: install lolcat for a better experience!"))))))
-
(defn- invalid-task? []
(let [task-name (first *command-line-args*)]
(when-not (contains? (set (u/all-bb-tasks-list)) task-name)
@@ -28,26 +21,30 @@
true)))
(defn- zen []
- (mapv (comp #(str "Zen of Metabase: " %) #(apply str %) #(drop 2 %))
+ (mapv (comp #(str (c/bold "Zen") " of Metabase: " %) #(apply str %) #(drop 2 %))
(filter (fn [x] (str/starts-with? x "-"))
(str/split-lines (slurp "zen.md")))))
(defn- tip-o-day []
(rand-nth
- (concat ["Did you know? You can use `./bin/mage <task> --help` to get more information about a specific task."
- "Pro tip: Use `./bin/mage <task>` to run a specific task."
- "Remember: You can always check available tasks with `./bin/mage`"
+ (concat ["You can use `./bin/mage <task> --help` to get more information? It works with every task."
+ "Remember: You can check private tasks with `./bin/mage ls`"
"Fun fact: The word 'mage' comes from the Latin 'magus', meaning 'wise'"
- "Pro tip: You can setup autocomplete for mage to speed up your workflow with mage setup-autocomplete."
- "Tip: we reccomend aliasing mage like: `alias mage='cd $MB_DIR && ./bin/mage'`"]
+ "Pro tip: You can setup autocomplete for mage to speed up your workflow with " (c/cyan "./bin/mage alias.")]
(zen))))
(defn- print-help []
- (lolcat "./mage/resources/splash.txt")
+ (println
+ (banner {:font ansi-shadow/ansi-shadow
+ :text (str (when (> (rand-int 100) 98) (str (u/sh "whoami") "'s ")) "Mage")
+ :gradient-direction (rand-nth [:to-top :to-bottom :to-right :to-left])
+ :margin-left 3
+ :margin-top 1
+ :gradient-colors (rand-nth [[:green :blue] [:red :magenta] [:orange :purple] [:cool :warm]])}))
(flush)
(println (c/bold " ✨ Metabase Automation Genius Engine ✨"))
(println "")
- (println (u/sh "./bin/bb tasks"))
+ (println (u/sh (str u/project-root-directory "/bin/bb tasks")))
(println (tip-o-day)))
(defn- summarize-exception [^Exception e]
@@ -62,10 +59,6 @@
(.getCause e)
(assoc :cause (summarize-exception (.getCause e)))))
-(def ^:private ex-message-label (c/red (c/reverse-color "ex-message : ")))
-(def ^:private ex-data-label (c/yellow (c/reverse-color "ex-data : ")))
-(def ^:private mage-error-label (c/blue (c/reverse-color "mage/error : ")))
-
(defn -main [& _]
(cond
;; help
@@ -78,8 +71,7 @@
;; errors
(invalid-task?)
(do (print-help)
- #_{:clj-kondo/ignore [:discouraged-java-method]}
- (System/exit 1))
+ (throw (ex-info "" {:babashka/exit 1})))
:else
;; at this point, we always have a valid task, and we are running in bb, so
@@ -96,23 +88,27 @@
;; - The ex-message will be printed
;; - The ex-data will be printed
;; - the (optional) `:babashka/exit` ex-data key will be used to determine the exit code
- ;; - the (optional) `:mage/error` ex-data key will suppress printing the entire exception,
- ;; use this when you know what the problem is and want to avoid noise
+ ;; - the (optional) `:mage/quiet` ex-data key will suppress all error output,
+ ;; - the (optional) `:mage/error` ex-data key will suppress printing the entire exception:
+ ;; use this when you know what the problem is and want to avoid noise
;;
- ;; Avoid using `System/exit` in your tasks, as it will hurt repl-ability of tasks: it will close the repl!
+ ;; Avoid using `System/exit` in your tasks, as it will hurt repl-ability of tasks:
+ ;; it will close your repl, prefer `mage.util/exit`
(let [message (ex-message e)
data (ex-data e)]
- (when (and e (not (:mage/error data)))
- (println (c/yellow "\nException:\n")
- (cond-> e
- (not (u/env "MAGE_DEBUG" (constantly nil)))
- summarize-exception)))
- (when (not (str/blank? message))
- (println ex-message-label message))
- (when (not-empty (dissoc data :mage/error))
- (println ex-data-label (u/pp (dissoc data :mage/error))))
- (when (:mage/error data)
- (println mage-error-label (:mage/error data)))
+ (when-not (:mage/quiet data)
+ (when (and e (not (:mage/error data)))
+ (println (c/yellow "\nException:\n")
+ (cond-> e
+ (not (u/env "MAGE_DEBUG" (constantly nil)))
+ summarize-exception)))
+ (when (and message (not (str/blank? message)))
+ (println (c/red (c/reverse-color "ex-message : ")) message))
+ (when data
+ (println (c/yellow (c/reverse-color "ex-data : ")) (pr-str
+ (dissoc data :mage/error))))
+ (when (:mage/error data)
+ (println (c/blue (c/reverse-color "mage/error : ")) (:mage/error data))))
#_{:clj-kondo/ignore [:discouraged-java-method]}
(System/exit (:babashka/exit data 1))))))))))
diff --git a/mage/resources/splash.txt b/mage/resources/splash.txt
deleted file mode 100644
index b9536f674c5..00000000000
--- a/mage/resources/splash.txt
+++ /dev/null
@@ -1,6 +0,0 @@
- ███╗ ███╗ █████╗ ██████╗ ███████╗
- ████╗ ████║██╔══██╗██╔════╝ ██╔════╝
- ██╔████╔██║███████║██║ ███╗█████╗
- ██║╚██╔╝██║██╔══██║██║ ██║██╔══╝
- ██║ ╚═╝ ██║██║ ██║╚██████╔╝███████╗
- ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝
diff --git a/mage/resources/token_scanner/token_whitelist.txt b/mage/resources/token_scanner/token_whitelist.txt
index 10c975ab6e6..b4afa553120 100644
--- a/mage/resources/token_scanner/token_whitelist.txt
+++ b/mage/resources/token_scanner/token_whitelist.txt
@@ -12,3 +12,7 @@ eyJuYW1lIjoiTmV3IE9yZGVycyBvdmVyIHRpbWUiLCJkZXNjcmlwdGlvbiI6bnVsbCwiZGF0YXNldF9x
# Airgap token:
airgap_eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhDQkMtSFMyNTYifQ.rHUDSPzcF-Kth1pIXbnlLlJiEgwx2rABSyj9PU98ubYGa7vUFwxfR9f4-UhF6EYrjvbtOx0QG9rFOSlfDmgG8jhCpIyuXpfliXdeeFpYt8Pludf-bmB85gU6hQEpYaTq_hihrPwCjpkUaNBaQzxN-klk1uqgDG2gBTspuqBXBln_Km6kLkyeF6BLO2nDJiiqT5pGPINSz42CCB7hGxZOFOnB443PfCas8QkfvmSX5X_Uqe9NNjhF_QKyElUWGCCO-Im48Pgd-HytwdaoaDzhe25fFDnYzDYgmEEQOxFncie0ZITCr7vjBRueK7KNnwXhO1297Xaku3moWnxRGZkcqA.ETtpjR3NQIceA-b77k537g.KHUIHYecBLByiKNm-_oQ7aNd9mJCVfhX12qfmWq6Zg1JBKicY-p7F3i_sk_ci0tfyfxT0PObi-s7BZ8pD7Fjw0wx-fmuQ0351gfA_1F6eKKZ0HwPtDBRN11Ip_FBy51YdRDa_bIRUXZJSEfLZSictQ1wMf-aUDRM6xz1CuzPNMXN3yi6vrkVNwAguHPwgs_pTCvga9P79vmjFyl05cBRzyDEhFOy72HzTd2R4EEXVKngZSNFvyJmILGQ_gcLrGr9.VuP_E_SwQMCeRUe2PNPpFg
+
+# a url that looks like a token
+eyJkYXRhc2V0X3F1ZXJ5Ijp7ImxpYi90eXBlIjoibWJxbC9xdWVyeSIsInN0YWdlcyI6W3sic291cmNlLXRhYmxlIjozLCJsaWIvdHlwZSI6Im1icWwuc3RhZ2UvbWJxbCIsImV4cHJlc3Npb25zIjpbWyJjb25jYXQiLHsibGliL3V1aWQiOiJjZDRlNDM5Yy00OTYwLTRlOWMtOGFkMy1iODQ2Mjk1ZjVmOGYiLCJlZmZlY3RpdmUtdHlwZSI6InR5cGUvVGV4dCIsImxpYi9leHByZXNzaW9uLW5hbWUiOiJHb29nbGUgaXQifSwiaHR0cHM6Ly93d3cuZ29vZ2xlLmNvbS9zZWFyY2g_cT1FQU4lMjAiLFsiZmllbGQiLHsibGliL3V1aWQiOiI2ODkxODJkOS0zNGMzLTQ5NGYtODk4NC1lNmUxODNmNzlmZDAiLCJlZmZlY3RpdmUtdHlwZSI6InR5cGUvVGV4dCIsImJhc2UtdHlwZSI6InR5cGUvVGV4dCJ9LDE1XV0sWyJjb25jYXQiLHsibGliL3V1aWQiOiJhNzMzZjNkNS1mZmNkLTQyMGYtOTMwNy1kMDlhOGM0MjBlMjgiLCJsaWIvZXhwcmVzc2lvbi1uYW1lIjoiQmluZyBpdCJ9LCJodHRwczovL3d3dy5iaW5nLmNvbS9zZWFyY2g_cT1FQU4lMjAiLFsiZmllbGQiLHsiZWZmZWN0aXZlLXR5cGUiOiJ0eXBlL1RleHQiLCJsaWIvdXVpZCI6ImI3ZTc2MGY3LTI3ZTktNDZiZS04NGVhLWZhMDIwOTRlMzUzMCIsImJhc2UtdHlwZSI6InR5cGUvVGV4dCJ9LDE1XV1dfV0sImRhdGFiYXNlIjoxfSwiZGlzcGxheSI6InRhYmxlIiwidmlzdWFsaXphdGlvbl9zZXR0aW5ncyI6e30sInR5cGUiOiJxdWVzdGlvbiJ9
+
diff --git a/mage/src/mage/be_dev.clj b/mage/src/mage/be_dev.clj
index f7316ea850a..d7153b962e7 100644
--- a/mage/src/mage/be_dev.clj
+++ b/mage/src/mage/be_dev.clj
@@ -1,6 +1,7 @@
(ns mage.be-dev
(:require
[bencode.core :as bencode]
+ [clojure.edn :as edn]
^:clj-kondo/ignore
[clojure.pprint :as pp]
[clojure.string :as str]
@@ -60,15 +61,21 @@
(defn eval-in-ns
"This insane code evals the code in the proper namespace.
- It's basically a repl inside a repl."
+ It's basically a repl inside a repl. The code here will be
+ executed in the context of the connected repl."
[nns code]
^:clj-kondo/ignore ;; ignore `eval`
`(let [ns# (symbol ~nns)]
(require ns# :reload)
(in-ns ns#)
- (eval (read-string ~(if (str/blank? code)
- "::loaded"
- code)))))
+ (eval (read-string
+ {:read-cond :allow}
+ ~(if (str/blank? code) "::loaded" code)))))
+
+(def ^{:private true
+ :dynamic true
+ :doc "Set this to true to suppress stdout output from nrepl-eval."}
+ *quiet-nrepl-eval* false)
(defn nrepl-eval
"Evaluate Clojure code in a running nREPL server. With one arg, reads port from .nrepl-port file.
@@ -81,16 +88,16 @@
code-str (str (eval-in-ns nns code))
_ (u/debug "Code:\n-----\n" code-str "\n-----")
_ (bencode/write-bencode out {:op "eval" :code code-str})
- final-value (atom nil)]
+ final-value (atom nil)
+ safe-print (fn [& msg] (when-not *quiet-nrepl-eval* (apply print msg) (flush)))]
(loop []
- (let [response (->> (bencode/read-bencode in)
- (walk/postwalk consume))]
+ (let [response (->> (bencode/read-bencode in) (walk/postwalk consume))]
(u/debug "Response:\n-----\n" (with-out-str (pp/pprint response)) "\n-----")
- (println)
+ (safe-print "\n")
(doseq [[k v] response]
(case k
- "out" (print v)
- "err" (binding [*out* *err*] (print v))
+ "out" (safe-print v)
+ "err" (binding [*out* *err*] (safe-print v))
"value" (reset! final-value v)
nil)) ; Ignore other keys like session, id, status
@@ -99,29 +106,41 @@
(if (some #{"done"} (get response "status"))
(some->> @final-value
- (println "\n=> "))
+ (safe-print "\n=> \n"))
(recur))))
@final-value))
+(nrepl-eval "user" "#?(:bb :bb :clj :clj)")
+
(defn nrepl-open?
"Checks if an nREPL server is running on the given port (or the port in .nrepl-port if none given)."
- [& port]
- (try
- (let [port (nrepl-port port)
- s (java.net.Socket. "localhost" port)]
- (.close s)
- true)
- (catch Exception _ false)))
-
-(defn nrepl-type [& port]
- (try
- (let [port (nrepl-port port)
- type (nrepl-eval "user" "#?(:bb :bb :cljs :cljs :clj :clj)" port)]
- (keyword type))
- (catch Exception _ nil)))
+ ([] (nrepl-open? nil))
+ ([port]
+ (try
+ (let [port (or port (nrepl-port port))
+ o (binding [*quiet-nrepl-eval* true]
+ (nrepl-eval "user" "(+ 1 1)" port))]
+ (= "2" o))
+ (catch Exception _ false))))
+
+(defn nrepl-type
+ "Returns :bb :cljs or :clj to indicate what type of nrepl server is running on the given port (or the port in .nrepl-port if none given)."
+ ([] (nrepl-type nil))
+ ([port]
+ (try
+ (let [port (or port (nrepl-port port))
+ [repl-clj-ver cond-read] (edn/read-string
+ (binding [*quiet-nrepl-eval* true]
+ (nrepl-eval "user" "[*clojure-version* #?(:clj :clj :cljs :cljs)]" port)))]
+ (if (= "SCI" (:qualifier repl-clj-ver))
+ :bb
+ cond-read))
+ (catch Exception _ nil))))
(comment
+ (nrepl-type)
+
(nrepl-eval "metabase.logger.core-test" "(do (println :!! 'hi) 'hi)" 7888)
(nrepl-eval "user" "*ns*" 7888)
diff --git a/mage/src/mage/color.clj b/mage/src/mage/color.clj
index f5a2d5e303e..a65800780ab 100644
--- a/mage/src/mage/color.clj
+++ b/mage/src/mage/color.clj
@@ -2,9 +2,11 @@
(:require
[clojure.string :as str]))
+(set! *warn-on-reflection* true)
+
(def ^:dynamic *disable-colors*
"If set to true, all color functions will return the input string as is."
- false)
+ (if (System/getenv "NO_COLOR") true false))
(def ^:private reset
(str "\033[" 0 "m"))
diff --git a/mage/src/mage/format.clj b/mage/src/mage/format.clj
index bf96ec2520e..33d8a28e83d 100644
--- a/mage/src/mage/format.clj
+++ b/mage/src/mage/format.clj
@@ -26,7 +26,7 @@
"Formats or checks all updated clojure files with cljfmt."
[parsed]
(let [target-branch (or (first (:arguments parsed)) "HEAD")
- updated-files (u/updated-files target-branch)]
+ updated-files (u/updated-clojure-files target-branch)]
(println (str "Checking for updated files against " (c/green target-branch)))
(if (seq updated-files)
(files (assoc parsed :arguments updated-files))
diff --git a/mage/src/mage/jar_download.clj b/mage/src/mage/jar_download.clj
index 97a2811ab73..26c75562d09 100644
--- a/mage/src/mage/jar_download.clj
+++ b/mage/src/mage/jar_download.clj
@@ -255,7 +255,7 @@
dir (some-> (or dir (u/env "JARS" (fn [] (println "JARS not set in env, defaulting to"
(str u/project-root-directory "/jars"))))
(str u/project-root-directory "/jars"))
- u/without-slash)
+ (str/replace #"/$" ""))
;; latest-version could be a version string like 1.50.35 or a map with :branch key
{:keys [latest-version jar-path]} (download-jar! version dir delete?)]
(when run?
diff --git a/mage/src/mage/kondo.clj b/mage/src/mage/kondo.clj
index 55b02f39032..af79e012bf0 100644
--- a/mage/src/mage/kondo.clj
+++ b/mage/src/mage/kondo.clj
@@ -73,7 +73,7 @@
(defn- kondo-updated* [diff-target]
(let [diff-target (or diff-target "HEAD")
- updated-files (u/updated-files diff-target)]
+ updated-files (u/updated-clojure-files diff-target)]
(when (empty? updated-files)
(println "No updated Clojure source files.")
(System/exit 0))
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index d0430c28054..b678b5c4f39 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -3,6 +3,7 @@
[clojure.edn :as edn]
[clojure.set :as set]
[clojure.string :as str]
+ [mage.color :as c]
[mage.util :as u]))
(set! *warn-on-reflection* true)
@@ -104,9 +105,9 @@
(println "(By unaffected, this means these modules do not have a direct or indirect dependency on the modules that have been changed.)")
(println)
(println)
- (println "Driver tests" (if (skip-driver-tests? deps updated)
- "CAN be skipped."
- "MUST be run."))))
+ (println (if (skip-driver-tests? deps updated)
+ (c/green "Driver tests " (c/bold "CAN be skipped") "")
+ (c/red "Driver tests " (c/bold "MUST be run") ".")))))
(defn cli-print-affected-modules
[[git-ref, :as _command-line-args]]
@@ -120,8 +121,8 @@
(println)
(println)
(printf "clojure -X :dev:ee:ee-dev:test :only '%s'\n" (pr-str (mapv module->test-directory affected)))
- (flush))
- (System/exit 0))
+ (flush)
+ (u/exit 0)))
(defn- changes-important-file-for-drivers?
"Whether we should always run driver tests if we have changes relative to `git-ref` to something important like
@@ -133,7 +134,7 @@
(printf "Running driver tests because %s was changed\n" (pr-str filename))
(flush)
filename))
- (u/updated-files (or git-ref "master"))))
+ (u/updated-clojure-files (or git-ref "master"))))
(defn- remove-non-driver-test-namespaces [files]
(into []
@@ -154,7 +155,7 @@
Invoke this from the CLI with
./bin/mage can-skip-driver-tests [git-ref]"
- [[git-ref, :as _command-line-args]]
+ [[git-ref, :as _arguments]]
(let [deps (dependencies)
git-ref (or git-ref "master")
updated-files (remove-non-driver-test-namespaces (u/updated-files git-ref))
@@ -162,7 +163,7 @@
skip-tests? (skip-driver-tests? deps updated)]
;; Not strictly necessary, but people looking at CI will appreciate having this extra info.
(print-updated-and-unaffected-modules deps updated)
- (System/exit ^Long (cond
- (not skip-tests?) 1
- (changes-important-file-for-drivers? git-ref) 1
- :else 0))))
+ (u/exit (cond
+ (not skip-tests?) 1
+ (changes-important-file-for-drivers? git-ref) 1
+ :else 0))))
diff --git a/mage/src/mage/quick_test_runner.clj b/mage/src/mage/quick_test_runner.clj
index fa87039d6d9..054f025cc65 100644
--- a/mage/src/mage/quick_test_runner.clj
+++ b/mage/src/mage/quick_test_runner.clj
@@ -38,7 +38,7 @@
(concat files dirs)))
(defn- gather-tests [selecting]
- (prn ["selecting" selecting])
+ (u/debug ["selecting" selecting])
(case selecting
"file" (gather-file-tests)
"dir" (gather-dir-tests)
@@ -50,16 +50,20 @@
(c/green "file") ", " (c/green "dir") ", or " (c/green "all") ".")
:babashka/exit 1}))))
-(defn- quotify [xs]
- (str/join " " (map #(str "\"" % "\"") xs)))
+(defn- feedback-bar
+ "Returns a glancable feedback bar of test results as string, with one emoji per test."
+ [out-data]
+ (str (str/join (repeat (:pass out-data) "✅"))
+ (str/join (repeat (:fail out-data) "❌"))
+ (str/join (repeat (:error out-data) "⚠️"))))
-(defn- run-tests-over-nrepl [test-dirs] ;; todo add dirs
+(defn- run-tests-over-nrepl [test-dirs options]
(let [start (u/start-timer)
the-ns "mb.hawk.core"
the-cmd (str "(do (require (quote metabase.test-runner)) "
"((requiring-resolve 'dev.reload/reload!)) "
"(metabase.test-runner/find-and-run-tests-repl "
- "{:only [" (quotify test-dirs) "]}))")]
+ "{:only " (pr-str test-dirs) "}))")]
(println "Running Code over nrepl:" (c/bold the-cmd))
(bling/callout
{:type :info
@@ -79,27 +83,40 @@
(bling/callout {:type :positive
:theme :minimal
:label-theme :marquee
- :label "To Rerun Directly"}
- (c/cyan "mage run-tests " (str/join " " test-dirs))))))
+ :label "Rerun This Directly With"}
+ (c/cyan "mage run-tests "
+ (when-let [p (:port options)] (format "--port %s " p))
+ (str/join " " test-dirs)))
+
+ (let [out-data (try (edn/read-string out)
+ (catch Exception _
+ (println (c/red "Problem parsing output, raw output follows:"))
+ #_:clj-kondo/ignore
+ (prn out)))
+ exit-code (if (zero? (+ (:fail out-data) (:error out-data))) 0 1)]
+ (println (feedback-bar out-data))
+ (u/exit exit-code)))))
;; namespaces will be converted to their file paths, so this check will work.
(defn- check-arg [arg]
(or (str/includes? arg ".clj") (str/includes? arg "/")))
-(defn add-cljy-suffix-or-throw [partial-file-path maybe-ns]
+(defn- add-cljy-suffix-or-throw
+ "Given a namespace, we can't tell if it's a clj or cljc file without looking for the file itself."
+ [partial-file-path maybe-ns]
(or (first
(keep (fn [suffix]
(let [file-path (str partial-file-path suffix)]
-
- (cond (str/starts-with? maybe-ns "metabase.")
- ;; src/metabase/...
- (let [path (str "src/" file-path)]
- (and (fs/exists? (str u/project-root-directory "/" path)) path))
-
- (str/starts-with? maybe-ns "metabase-enterprise.")
- ;; enterprise/backend/src/metabase_enterprise/...
- (let [path (str "enterprise/backend/" file-path)]
- (and (fs/exists? (str u/project-root-directory "/" path)) path)))))
+ (cond
+ ;; src/metabase/...
+ (str/starts-with? maybe-ns "metabase.")
+ (let [path (str "src/" file-path)]
+ (and (fs/exists? (str u/project-root-directory "/" path)) path))
+
+ ;; enterprise/backend/src/metabase_enterprise/...
+ (str/starts-with? maybe-ns "metabase-enterprise.")
+ (let [path (str "enterprise/backend/" file-path)]
+ (and (fs/exists? (str u/project-root-directory "/" path)) path)))))
[".clj" ".cljc" ".bb"]))
(throw
(ex-info "" {:mage/error (str "Could not find a file for namespace: "
@@ -107,7 +124,7 @@
". Tried appending .clj, .cljs, .cljc, and .bb -- is that a real namespace?")
:babashka/exit 1}))))
-(defn maybe-convert-ns-to-filename [maybe-ns]
+(defn- maybe-convert-ns-to-path [maybe-ns]
(if (or (str/starts-with? maybe-ns "metabase.")
(str/starts-with? maybe-ns "metabase-enterprise."))
(-> maybe-ns
@@ -123,7 +140,7 @@
(let [tests (if (seq arguments)
arguments
(-> (gather-tests selecting)
- (u/fzf-select
+ (u/fzf-select!
(str/join " " ["--multi"
"--ansi"
"--marker" "'✓ '"
@@ -133,7 +150,7 @@
"--header-border" "rounded"
"--preview" (str "'" u/project-root-directory "/mage/cmd/fzf_preview.clj {}'")]))
str/split-lines))
- test-dir-or-nss (mapv maybe-convert-ns-to-filename tests)]
+ test-dir-or-nss (mapv maybe-convert-ns-to-path tests)]
(when-not (every? check-arg test-dir-or-nss)
(throw (ex-info "" {:mage/error (str
"When providing arguments, they must be file paths or directories, got: "
@@ -143,21 +160,20 @@
test-dir-or-nss))
(defn- run-tests-cli [test-dirs]
- (let [cmd (str "clj -X:dev:ee:ee-dev:test :only '[" (quotify test-dirs) "]'")]
+ (let [cmd (str "clj -X:dev:ee:ee-dev:test :only '" (pr-str test-dirs) "'")]
(bling/callout {:label "Running Command Line"} (c/bold cmd))
- (shell/sh* "clojure" "-X:dev:dev-ee:ee:test" ":only" (str "[" (quotify test-dirs) "]"))))
+ (shell/sh* "clojure" "-X:dev:dev-ee:ee:test" ":only" (pr-str test-dirs))))
(defn go
"Interactively select directories to run tests against."
[{:keys [arguments options] :as _parsed}]
(let [tests (setup-test-files arguments options)]
- (prn ["TESTS" tests])
(if (and (backend/nrepl-open?)
- ;; No testing against the mage nrepl! (probably noone will hit this)
+ ;; No testing against the mage nrepl!
(not= :bb (backend/nrepl-type)))
(do
(println "Running tests via ⏩🏎️✨" (c/green (c/bold "THE REPL")) "✨🏎️⏪.")
- (run-tests-over-nrepl tests))
+ (run-tests-over-nrepl tests options))
(do
(println "Running via " (c/bold (c/magenta "the command line")) "."
(c/red " This is " (c/bold "SLOW") " and " (c/bold "NOT RECCOMENDED!! "))
@@ -168,6 +184,3 @@
:gradient-direction :to-top
:gradient-colors [:magenta :red]}))
(run-tests-cli tests)))))
-
-
-
diff --git a/mage/src/mage/util.clj b/mage/src/mage/util.clj
index c07d4222937..3d283d0d392 100644
--- a/mage/src/mage/util.clj
+++ b/mage/src/mage/util.clj
@@ -82,7 +82,7 @@
(defn public-bb-tasks-list
"Returns all public bb tasks as a vector of strings."
[]
- (->> "bb tasks"
+ (->> (str project-root-directory "/bin/bb tasks")
shl
(drop 2)
(map (comp first #(str/split % #"\s+")))
@@ -102,6 +102,19 @@
(defn updated-files
"Sequence of filenames that have changes in Git relative to `diff-target`."
([] (updated-files "HEAD"))
+ ([diff-target]
+ (->> (shell {:out :string :dir project-root-directory}
+ "git" "diff" "--name-only" diff-target)
+ :out
+ (str/split-lines)
+ ;; filter out any files that have been deleted/moved
+ (remove #{""})
+ (filter (fn [filename]
+ (fs/exists? (str project-root-directory "/" filename)))))))
+
+(defn updated-clojure-files
+ "Sequence of filenames that have changes in Git relative to `diff-target`."
+ ([] (updated-clojure-files "HEAD"))
([diff-target]
(->> (shell {:out :string :dir project-root-directory}
"git" "diff" "--name-only" diff-target
@@ -200,7 +213,7 @@
:mage/error (str "You don't have " (c/yellow cmd) " installed. Please install it to use this task.")
:babashka/exit 1}))))
-(defn fzf-select
+(defn fzf-select!
"Uses fzf to offer interactive selections.
If the user doesn't have fzf installed, explains instructions
@@ -238,3 +251,13 @@
(defn since-ms
"Called on the return value of start-timer, returns the elapsed time in milliseconds."
[timer] (/ (- (System/nanoTime) timer) 1e6))
+
+(defn exit
+ "When invoked from a babashka namespace spawned from mage, exits with the given exit code.
+ Will not crash your repl. Prefer this to System/exit!"
+ ([exit-code]
+ (throw (ex-info "" {:mage/quiet true
+ :babashka/exit exit-code})))
+ ([message exit-code]
+ (println message)
+ (throw (ex-info "" {:mage/quiet true :babashka/exit exit-code}))))
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index eb32356b579..8c058c447b9 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -3,52 +3,38 @@
[babashka.tasks :as bt]
[clojure.string :as str]
[clojure.test :as t :refer [deftest is testing]]
- [mage.token-scan-test :as token-scan-test]
- [mage.util :as u]))
-
-(comment
- token-scan-test/keep-me)
+ [mage.token-scan-test]
+ [mage.util :as u]
+ [mage.util-test]))
(set! *warn-on-reflection* true)
-(defn bin-mage-has-help? []
- (doseq [help-cmds [[] [" "] [" "]
- ["-h"] ["--help"]
- [" -h"] [" --help"]
- [" -h"] [" --help"]]
- :let [cmd (str "./bin/mage " (str/join " " help-cmds))]]
- (testing (format "'%s'" (pr-str cmd))
+(comment
+ mage.token-scan-test/keep-me
+ mage.util-test/keep-me)
+
+(deftest bin-mage-has-help-test
+ (doseq [help-cmds [[] [" "] [" "] ["-h"] ["--help"] [" -h"] [" --help"] [" -h"] [" --help"]]
+ :let [cmd (str "./bin/mage " (str/join " " help-cmds))
+ title (format "'%s' returns help information" (pr-str cmd))]]
+ (println title)
+ (testing title
(let [out (u/sh cmd)]
- (when-not (str/includes? out "The following tasks are available:")
- (System/exit 1))))))
+ (is (str/includes? out "The following tasks are available:"))))))
-(defn- bb-task-has-example? [task-name]
- (doseq [cmd [(str "./bin/mage " task-name " -h")
+(deftest bb-task-has-example-test
+ (doseq [task-name (u/public-bb-tasks-list)
+ cmd [(str "./bin/mage " task-name " -h")
(str "./bin/mage " task-name " --help")]]
- (testing (format "'%s'" cmd)
+ (println "Checking examples for command:" cmd)
+ (testing (format "%s has examples with: '%s'" task-name cmd)
(is (str/includes? (u/sh cmd) "Examples:")))))
-(defn invalid-task-names-print-help-test []
+(deftest invalid-task-names-print-help-test
(doseq [task-name ["foo" "bar" "baz"]]
- (let [cmd (str "./bin/mage " task-name)]
- (testing (format "'%s'" cmd)
- (let [result (try (bt/shell {:err :string :out :string} "./bin/mage foo")
+ (let [cmd (str "./bin/mage " task-name)
+ title (format "invalid task names like: '%s' print help" cmd)]
+ (testing title
+ (let [result (try (bt/shell {:err :string :out :string} (str "./bin/mage " task-name))
(catch Exception e (:out (:proc (ex-data e)))))]
(is (str/includes? result "The following tasks are available:")))))))
-
-(deftest mage-tests
- (println "Running mage tests")
-
- (println " bb tasks have examples")
- (testing "bb tasks have examples"
- (mapv bb-task-has-example? (u/public-bb-tasks-list)))
-
- (println " bb tasks have help")
- (testing "mage has help"
- (bin-mage-has-help?))
-
- (println " invalid task names print help")
- (testing "Invalid task name prints help"
- (invalid-task-names-print-help-test))
-
- (t/run-tests 'mage.token-scan-test))
diff --git a/mage/test/mage/examples_test.clj b/mage/test/mage/examples_test.clj
deleted file mode 100644
index 7e237ff4cbd..00000000000
--- a/mage/test/mage/examples_test.clj
+++ /dev/null
@@ -1,31 +0,0 @@
-(ns mage.examples-test
- (:require
- [clojure.edn :as edn]
- [mage.util :as u]))
-
-(set! *warn-on-reflection* true)
-
-(defn- task->task-maps []
- (dissoc (:tasks (edn/read-string (slurp "bb.edn"))) :requires))
-
-(defn- task->examples []
- (update-vals (task->task-maps) :examples))
-
-(def ^:private excluded-public-task-examples #{"nrepl"})
-
-(defn run-tests
- "This takes a while to run, so it is not run by default. To run it, use the `./bin/mage -test-examples` command."
- []
- (let [public-tasks (set (u/public-bb-tasks-list))]
- (println "Running example tests for:" public-tasks "\n")
- (doseq [[task-name examples] (task->examples)
- :when (contains? public-tasks (str task-name))
- :when (not (excluded-public-task-examples (str task-name)))
- [cmd explaination] examples]
- (println task-name)
- (println "Testing that running ['" cmd "' -> '" explaination "'] returns a 0 exit-code...")
- (try (u/sh cmd)
- (catch Exception _ (println "Failed" task-name ":" cmd))
- (println "OK"))))
- (println "Done.")
- :ok)
diff --git a/mage/test/mage/token_scan_test.clj b/mage/test/mage/token_scan_test.clj
index a2c12f3ef3d..a72bbafcf8e 100644
--- a/mage/test/mage/token_scan_test.clj
+++ b/mage/test/mage/token_scan_test.clj
@@ -12,7 +12,7 @@
(def token-like-comment (str ";;" (apply str (repeat 100 \a)) "\n"))
(def sample-file-path
- (str u/project-root-directory "/test/mage/token_scan_test_example.clj"))
+ (str u/project-root-directory "/" "mage/test/mage/token_scan_test_example.clj"))
(deftest token-scan-finds-error
(try
diff --git a/mage/test/mage/util_test.clj b/mage/test/mage/util_test.clj
new file mode 100644
index 00000000000..6b7bdeb081a
--- /dev/null
+++ b/mage/test/mage/util_test.clj
@@ -0,0 +1,30 @@
+(ns mage.util-test
+ (:require
+ [babashka.fs :as fs]
+ [clojure.string :as str]
+ [clojure.test :as t :refer [deftest is]]
+ [mage.color :as c]
+ [mage.util :as u]))
+
+(def ^:private new-content "MAGE_UTIL_TEST_STRING")
+
+(defn- add-content [file-path]
+ (spit file-path new-content :append true))
+
+(defn- remove-content [file-path]
+ (spit file-path
+ (-> (slurp file-path)
+ (str/replace (re-pattern (str new-content "$")) ""))))
+
+(deftest updated-files-returns-all-kinds-of-files
+ (let [extension->path (-> (group-by fs/extension (fs/glob u/project-root-directory "**.*"))
+ (update-vals (comp str first))
+ (select-keys ["clj" "cljc" "cljs" "edn" "js" "ts" "jsx" "json" "css" "html" "md"]))]
+ (doseq [[_ path] extension->path]
+ (println
+ (str "Testing that " (c/green "u/updated-files") " picks up changes to file:" (c/cyan path)))
+ (try (add-content path)
+ (is (contains? (set (u/updated-files)) (str/replace path (str u/project-root-directory "/") ""))
+ (str "updated-files should include modified file: " path))
+ (finally (remove-content path))))))
+