The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
`updated-files` returns all-files + Mage cleanup (#66332)
bryan1591300b492d
diff --git a/.github/workflows/mage.yml b/.github/workflows/mage.ymlindex 6deefd0c635..7847a965155 100644--- a/.github/workflows/mage.yml+++ b/.github/workflows/mage.yml@@ -2,7 +2,9 @@ name: Mage Testson:pull_request:- paths: ["mage/**"]+ paths:+ - "mage/**"+ - "bb.edn"jobs:# run mage tests even if the rest of be isnt needed@@ -13,13 +15,7 @@ jobs:- uses: actions/checkout@v4- name: Prepare back-end environmentuses: ./.github/actions/prepare-backend- - name: Setup Babashka- uses: turtlequeue/[email protected]- with:- babashka-version: 1.12.197- - name: Check bb runs- run: bb --version- - name: bb task list- run: bb tasks+ - name: Check mage runs+ run: ./bin/mage -h- name: Run mage tests- run: bb ./mage/test/mage/runtests.clj+ run: ./bin/mage -testdiff --git a/bb.edn b/bb.ednindex 6a50f28aaf6..047a1bc522e 100644--- a/bb.edn+++ b/bb.edn@@ -179,6 +179,8 @@:requires [[mage.be-dev :refer [nrepl-eval]]]:options [["-c" "--comment-id ID" "Notify about this comment id"]["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]+ :examples [["./bin/mage notify -c 1234" "Notify about comment with id 1234"]+ ["./bin/mage notify --comment-id 5678 --port 59498" "Notify about comment with id 5678 on nrepl port 59498"]]:task (let [opts (:options (cli/parse! (current-task)));; note: no quoting -> expect it to be intcode (format "(notify-comment-id! %d)" (parse-long (:comment-id opts)))]@@ -309,14 +311,15 @@:data {:a [:b :c] :b [:d]}}-test {:doc "run all mage tests"- :requires [[mage.core-test]- [clojure.test :refer [run-tests]]]- :task (run-tests 'mage.core-test)}-- -test-examples- {:doc "Runs every example and checks that it exits with 0"- :requires [[mage.examples-test :as examples-test]]- :task (System/exit (if (= :ok (examples-test/run-tests)) 0 1))}+ :requires [[mage.core-test] ;; loads the actual tests+ [mage.util :as u]+ [clojure.test]]+ :task (let [results (clojure.test/run-all-tests (re-pattern "^mage\\..*"))]+ (if (zero? (+ (:fail results) (:error results)))+ (println (c/green "All mage tests passed!"))+ (do+ (println (c/red "Some mage tests failed."))+ (u/exit 1))))};;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Personal Tasks:diff --git a/bin/mage.bb b/bin/mage.bbindex 425f114c012..0701fd5b66a 100755--- a/bin/mage.bb+++ b/bin/mage.bb@@ -6,21 +6,14 @@(ns mage(:require[babashka.tasks :as bt]+ [bling.banner :refer [banner]]+ [bling.fonts.ansi-shadow :as ansi-shadow][clojure.string :as str][mage.color :as c][mage.util :as u]))(set! *warn-on-reflection* true)-(defn- lolcat [f]- (if (try (u/sh "command -v lolcat")- (catch Exception _ false))- (bt/shell (str "lolcat " f))- (do- (println (slurp f))- (when (> (rand) 0.8)- (println (c/yellow "Tip: install lolcat for a better experience!"))))))-(defn- invalid-task? [](let [task-name (first *command-line-args*)](when-not (contains? (set (u/all-bb-tasks-list)) task-name)@@ -28,26 +21,30 @@true)))(defn- zen []- (mapv (comp #(str "Zen of Metabase: " %) #(apply str %) #(drop 2 %))+ (mapv (comp #(str (c/bold "Zen") " of Metabase: " %) #(apply str %) #(drop 2 %))(filter (fn [x] (str/starts-with? x "-"))(str/split-lines (slurp "zen.md")))))(defn- tip-o-day [](rand-nth- (concat ["Did you know? You can use `./bin/mage <task> --help` to get more information about a specific task."- "Pro tip: Use `./bin/mage <task>` to run a specific task."- "Remember: You can always check available tasks with `./bin/mage`"+ (concat ["You can use `./bin/mage <task> --help` to get more information? It works with every task."+ "Remember: You can check private tasks with `./bin/mage ls`""Fun fact: The word 'mage' comes from the Latin 'magus', meaning 'wise'"- "Pro tip: You can setup autocomplete for mage to speed up your workflow with mage setup-autocomplete."- "Tip: we reccomend aliasing mage like: `alias mage='cd $MB_DIR && ./bin/mage'`"]+ "Pro tip: You can setup autocomplete for mage to speed up your workflow with " (c/cyan "./bin/mage alias.")](zen))))(defn- print-help []- (lolcat "./mage/resources/splash.txt")+ (println+ (banner {:font ansi-shadow/ansi-shadow+ :text (str (when (> (rand-int 100) 98) (str (u/sh "whoami") "'s ")) "Mage")+ :gradient-direction (rand-nth [:to-top :to-bottom :to-right :to-left])+ :margin-left 3+ :margin-top 1+ :gradient-colors (rand-nth [[:green :blue] [:red :magenta] [:orange :purple] [:cool :warm]])}))(flush)(println (c/bold " ✨ Metabase Automation Genius Engine ✨"))(println "")- (println (u/sh "./bin/bb tasks"))+ (println (u/sh (str u/project-root-directory "/bin/bb tasks")))(println (tip-o-day)))(defn- summarize-exception [^Exception e]@@ -62,10 +59,6 @@(.getCause e)(assoc :cause (summarize-exception (.getCause e)))))-(def ^:private ex-message-label (c/red (c/reverse-color "ex-message : ")))-(def ^:private ex-data-label (c/yellow (c/reverse-color "ex-data : ")))-(def ^:private mage-error-label (c/blue (c/reverse-color "mage/error : ")))-(defn -main [& _](cond;; help@@ -78,8 +71,7 @@;; errors(invalid-task?)(do (print-help)- #_{:clj-kondo/ignore [:discouraged-java-method]}- (System/exit 1))+ (throw (ex-info "" {:babashka/exit 1}))):else;; at this point, we always have a valid task, and we are running in bb, so@@ -96,23 +88,27 @@;; - The ex-message will be printed;; - The ex-data will be printed;; - the (optional) `:babashka/exit` ex-data key will be used to determine the exit code- ;; - the (optional) `:mage/error` ex-data key will suppress printing the entire exception,- ;; use this when you know what the problem is and want to avoid noise+ ;; - the (optional) `:mage/quiet` ex-data key will suppress all error output,+ ;; - the (optional) `:mage/error` ex-data key will suppress printing the entire exception:+ ;; use this when you know what the problem is and want to avoid noise;;- ;; Avoid using `System/exit` in your tasks, as it will hurt repl-ability of tasks: it will close the repl!+ ;; Avoid using `System/exit` in your tasks, as it will hurt repl-ability of tasks:+ ;; it will close your repl, prefer `mage.util/exit`(let [message (ex-message e)data (ex-data e)]- (when (and e (not (:mage/error data)))- (println (c/yellow "\nException:\n")- (cond-> e- (not (u/env "MAGE_DEBUG" (constantly nil)))- summarize-exception)))- (when (not (str/blank? message))- (println ex-message-label message))- (when (not-empty (dissoc data :mage/error))- (println ex-data-label (u/pp (dissoc data :mage/error))))- (when (:mage/error data)- (println mage-error-label (:mage/error data)))+ (when-not (:mage/quiet data)+ (when (and e (not (:mage/error data)))+ (println (c/yellow "\nException:\n")+ (cond-> e+ (not (u/env "MAGE_DEBUG" (constantly nil)))+ summarize-exception)))+ (when (and message (not (str/blank? message)))+ (println (c/red (c/reverse-color "ex-message : ")) message))+ (when data+ (println (c/yellow (c/reverse-color "ex-data : ")) (pr-str+ (dissoc data :mage/error))))+ (when (:mage/error data)+ (println (c/blue (c/reverse-color "mage/error : ")) (:mage/error data))))#_{:clj-kondo/ignore [:discouraged-java-method]}(System/exit (:babashka/exit data 1))))))))))diff --git a/mage/resources/splash.txt b/mage/resources/splash.txtdeleted file mode 100644index b9536f674c5..00000000000--- a/mage/resources/splash.txt+++ /dev/null@@ -1,6 +0,0 @@- ███╗ ███╗ █████╗ ██████╗ ███████╗- ████╗ ████║██╔══██╗██╔════╝ ██╔════╝- ██╔████╔██║███████║██║ ███╗█████╗- ██║╚██╔╝██║██╔══██║██║ ██║██╔══╝- ██║ ╚═╝ ██║██║ ██║╚██████╔╝███████╗- ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝diff --git a/mage/resources/token_scanner/token_whitelist.txt b/mage/resources/token_scanner/token_whitelist.txtindex 10c975ab6e6..b4afa553120 100644--- a/mage/resources/token_scanner/token_whitelist.txt+++ b/mage/resources/token_scanner/token_whitelist.txt@@ -12,3 +12,7 @@ eyJuYW1lIjoiTmV3IE9yZGVycyBvdmVyIHRpbWUiLCJkZXNjcmlwdGlvbiI6bnVsbCwiZGF0YXNldF9x# Airgap token:airgap_eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhDQkMtSFMyNTYifQ.rHUDSPzcF-Kth1pIXbnlLlJiEgwx2rABSyj9PU98ubYGa7vUFwxfR9f4-UhF6EYrjvbtOx0QG9rFOSlfDmgG8jhCpIyuXpfliXdeeFpYt8Pludf-bmB85gU6hQEpYaTq_hihrPwCjpkUaNBaQzxN-klk1uqgDG2gBTspuqBXBln_Km6kLkyeF6BLO2nDJiiqT5pGPINSz42CCB7hGxZOFOnB443PfCas8QkfvmSX5X_Uqe9NNjhF_QKyElUWGCCO-Im48Pgd-HytwdaoaDzhe25fFDnYzDYgmEEQOxFncie0ZITCr7vjBRueK7KNnwXhO1297Xaku3moWnxRGZkcqA.ETtpjR3NQIceA-b77k537g.KHUIHYecBLByiKNm-_oQ7aNd9mJCVfhX12qfmWq6Zg1JBKicY-p7F3i_sk_ci0tfyfxT0PObi-s7BZ8pD7Fjw0wx-fmuQ0351gfA_1F6eKKZ0HwPtDBRN11Ip_FBy51YdRDa_bIRUXZJSEfLZSictQ1wMf-aUDRM6xz1CuzPNMXN3yi6vrkVNwAguHPwgs_pTCvga9P79vmjFyl05cBRzyDEhFOy72HzTd2R4EEXVKngZSNFvyJmILGQ_gcLrGr9.VuP_E_SwQMCeRUe2PNPpFg++# a url that looks like a token+eyJkYXRhc2V0X3F1ZXJ5Ijp7ImxpYi90eXBlIjoibWJxbC9xdWVyeSIsInN0YWdlcyI6W3sic291cmNlLXRhYmxlIjozLCJsaWIvdHlwZSI6Im1icWwuc3RhZ2UvbWJxbCIsImV4cHJlc3Npb25zIjpbWyJjb25jYXQiLHsibGliL3V1aWQiOiJjZDRlNDM5Yy00OTYwLTRlOWMtOGFkMy1iODQ2Mjk1ZjVmOGYiLCJlZmZlY3RpdmUtdHlwZSI6InR5cGUvVGV4dCIsImxpYi9leHByZXNzaW9uLW5hbWUiOiJHb29nbGUgaXQifSwiaHR0cHM6Ly93d3cuZ29vZ2xlLmNvbS9zZWFyY2g_cT1FQU4lMjAiLFsiZmllbGQiLHsibGliL3V1aWQiOiI2ODkxODJkOS0zNGMzLTQ5NGYtODk4NC1lNmUxODNmNzlmZDAiLCJlZmZlY3RpdmUtdHlwZSI6InR5cGUvVGV4dCIsImJhc2UtdHlwZSI6InR5cGUvVGV4dCJ9LDE1XV0sWyJjb25jYXQiLHsibGliL3V1aWQiOiJhNzMzZjNkNS1mZmNkLTQyMGYtOTMwNy1kMDlhOGM0MjBlMjgiLCJsaWIvZXhwcmVzc2lvbi1uYW1lIjoiQmluZyBpdCJ9LCJodHRwczovL3d3dy5iaW5nLmNvbS9zZWFyY2g_cT1FQU4lMjAiLFsiZmllbGQiLHsiZWZmZWN0aXZlLXR5cGUiOiJ0eXBlL1RleHQiLCJsaWIvdXVpZCI6ImI3ZTc2MGY3LTI3ZTktNDZiZS04NGVhLWZhMDIwOTRlMzUzMCIsImJhc2UtdHlwZSI6InR5cGUvVGV4dCJ9LDE1XV1dfV0sImRhdGFiYXNlIjoxfSwiZGlzcGxheSI6InRhYmxlIiwidmlzdWFsaXphdGlvbl9zZXR0aW5ncyI6e30sInR5cGUiOiJxdWVzdGlvbiJ9+diff --git a/mage/src/mage/be_dev.clj b/mage/src/mage/be_dev.cljindex f7316ea850a..d7153b962e7 100644--- a/mage/src/mage/be_dev.clj+++ b/mage/src/mage/be_dev.clj@@ -1,6 +1,7 @@(ns mage.be-dev(:require[bencode.core :as bencode]+ [clojure.edn :as edn]^:clj-kondo/ignore[clojure.pprint :as pp][clojure.string :as str]@@ -60,15 +61,21 @@(defn eval-in-ns"This insane code evals the code in the proper namespace.- It's basically a repl inside a repl."+ It's basically a repl inside a repl. The code here will be+ executed in the context of the connected repl."[nns code]^:clj-kondo/ignore ;; ignore `eval``(let [ns# (symbol ~nns)](require ns# :reload)(in-ns ns#)- (eval (read-string ~(if (str/blank? code)- "::loaded"- code)))))+ (eval (read-string+ {:read-cond :allow}+ ~(if (str/blank? code) "::loaded" code)))))++(def ^{:private true+ :dynamic true+ :doc "Set this to true to suppress stdout output from nrepl-eval."}+ *quiet-nrepl-eval* false)(defn nrepl-eval"Evaluate Clojure code in a running nREPL server. With one arg, reads port from .nrepl-port file.@@ -81,16 +88,16 @@code-str (str (eval-in-ns nns code))_ (u/debug "Code:\n-----\n" code-str "\n-----")_ (bencode/write-bencode out {:op "eval" :code code-str})- final-value (atom nil)]+ final-value (atom nil)+ safe-print (fn [& msg] (when-not *quiet-nrepl-eval* (apply print msg) (flush)))](loop []- (let [response (->> (bencode/read-bencode in)- (walk/postwalk consume))]+ (let [response (->> (bencode/read-bencode in) (walk/postwalk consume))](u/debug "Response:\n-----\n" (with-out-str (pp/pprint response)) "\n-----")- (println)+ (safe-print "\n")(doseq [[k v] response](case k- "out" (print v)- "err" (binding [*out* *err*] (print v))+ "out" (safe-print v)+ "err" (binding [*out* *err*] (safe-print v))"value" (reset! final-value v)nil)) ; Ignore other keys like session, id, status@@ -99,29 +106,41 @@(if (some #{"done"} (get response "status"))(some->> @final-value- (println "\n=> "))+ (safe-print "\n=> \n"))(recur))))@final-value))+(nrepl-eval "user" "#?(:bb :bb :clj :clj)")+(defn nrepl-open?"Checks if an nREPL server is running on the given port (or the port in .nrepl-port if none given)."- [& port]- (try- (let [port (nrepl-port port)- s (java.net.Socket. "localhost" port)]- (.close s)- true)- (catch Exception _ false)))--(defn nrepl-type [& port]- (try- (let [port (nrepl-port port)- type (nrepl-eval "user" "#?(:bb :bb :cljs :cljs :clj :clj)" port)]- (keyword type))- (catch Exception _ nil)))+ ([] (nrepl-open? nil))+ ([port]+ (try+ (let [port (or port (nrepl-port port))+ o (binding [*quiet-nrepl-eval* true]+ (nrepl-eval "user" "(+ 1 1)" port))]+ (= "2" o))+ (catch Exception _ false))))++(defn nrepl-type+ "Returns :bb :cljs or :clj to indicate what type of nrepl server is running on the given port (or the port in .nrepl-port if none given)."+ ([] (nrepl-type nil))+ ([port]+ (try+ (let [port (or port (nrepl-port port))+ [repl-clj-ver cond-read] (edn/read-string+ (binding [*quiet-nrepl-eval* true]+ (nrepl-eval "user" "[*clojure-version* #?(:clj :clj :cljs :cljs)]" port)))]+ (if (= "SCI" (:qualifier repl-clj-ver))+ :bb+ cond-read))+ (catch Exception _ nil))))(comment+ (nrepl-type)+(nrepl-eval "metabase.logger.core-test" "(do (println :!! 'hi) 'hi)" 7888)(nrepl-eval "user" "*ns*" 7888)diff --git a/mage/src/mage/color.clj b/mage/src/mage/color.cljindex f5a2d5e303e..a65800780ab 100644--- a/mage/src/mage/color.clj+++ b/mage/src/mage/color.clj@@ -2,9 +2,11 @@(:require[clojure.string :as str]))+(set! *warn-on-reflection* true)+(def ^:dynamic *disable-colors*"If set to true, all color functions will return the input string as is."- false)+ (if (System/getenv "NO_COLOR") true false))(def ^:private reset(str "\033[" 0 "m"))diff --git a/mage/src/mage/format.clj b/mage/src/mage/format.cljindex bf96ec2520e..33d8a28e83d 100644--- a/mage/src/mage/format.clj+++ b/mage/src/mage/format.clj@@ -26,7 +26,7 @@"Formats or checks all updated clojure files with cljfmt."[parsed](let [target-branch (or (first (:arguments parsed)) "HEAD")- updated-files (u/updated-files target-branch)]+ updated-files (u/updated-clojure-files target-branch)](println (str "Checking for updated files against " (c/green target-branch)))(if (seq updated-files)(files (assoc parsed :arguments updated-files))diff --git a/mage/src/mage/jar_download.clj b/mage/src/mage/jar_download.cljindex 97a2811ab73..26c75562d09 100644--- a/mage/src/mage/jar_download.clj+++ b/mage/src/mage/jar_download.clj@@ -255,7 +255,7 @@dir (some-> (or dir (u/env "JARS" (fn [] (println "JARS not set in env, defaulting to"(str u/project-root-directory "/jars"))))(str u/project-root-directory "/jars"))- u/without-slash)+ (str/replace #"/$" ""));; latest-version could be a version string like 1.50.35 or a map with :branch key{:keys [latest-version jar-path]} (download-jar! version dir delete?)](when run?diff --git a/mage/src/mage/kondo.clj b/mage/src/mage/kondo.cljindex 55b02f39032..af79e012bf0 100644--- a/mage/src/mage/kondo.clj+++ b/mage/src/mage/kondo.clj@@ -73,7 +73,7 @@(defn- kondo-updated* [diff-target](let [diff-target (or diff-target "HEAD")- updated-files (u/updated-files diff-target)]+ updated-files (u/updated-clojure-files diff-target)](when (empty? updated-files)(println "No updated Clojure source files.")(System/exit 0))diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex d0430c28054..b678b5c4f39 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -3,6 +3,7 @@[clojure.edn :as edn][clojure.set :as set][clojure.string :as str]+ [mage.color :as c][mage.util :as u]))(set! *warn-on-reflection* true)@@ -104,9 +105,9 @@(println "(By unaffected, this means these modules do not have a direct or indirect dependency on the modules that have been changed.)")(println)(println)- (println "Driver tests" (if (skip-driver-tests? deps updated)- "CAN be skipped."- "MUST be run."))))+ (println (if (skip-driver-tests? deps updated)+ (c/green "Driver tests " (c/bold "CAN be skipped") "")+ (c/red "Driver tests " (c/bold "MUST be run") ".")))))(defn cli-print-affected-modules[[git-ref, :as _command-line-args]]@@ -120,8 +121,8 @@(println)(println)(printf "clojure -X :dev:ee:ee-dev:test :only '%s'\n" (pr-str (mapv module->test-directory affected)))- (flush))- (System/exit 0))+ (flush)+ (u/exit 0)))(defn- changes-important-file-for-drivers?"Whether we should always run driver tests if we have changes relative to `git-ref` to something important like@@ -133,7 +134,7 @@(printf "Running driver tests because %s was changed\n" (pr-str filename))(flush)filename))- (u/updated-files (or git-ref "master"))))+ (u/updated-clojure-files (or git-ref "master"))))(defn- remove-non-driver-test-namespaces [files](into []@@ -154,7 +155,7 @@Invoke this from the CLI with./bin/mage can-skip-driver-tests [git-ref]"- [[git-ref, :as _command-line-args]]+ [[git-ref, :as _arguments]](let [deps (dependencies)git-ref (or git-ref "master")updated-files (remove-non-driver-test-namespaces (u/updated-files git-ref))@@ -162,7 +163,7 @@skip-tests? (skip-driver-tests? deps updated)];; Not strictly necessary, but people looking at CI will appreciate having this extra info.(print-updated-and-unaffected-modules deps updated)- (System/exit ^Long (cond- (not skip-tests?) 1- (changes-important-file-for-drivers? git-ref) 1- :else 0))))+ (u/exit (cond+ (not skip-tests?) 1+ (changes-important-file-for-drivers? git-ref) 1+ :else 0))))diff --git a/mage/src/mage/quick_test_runner.clj b/mage/src/mage/quick_test_runner.cljindex fa87039d6d9..054f025cc65 100644--- a/mage/src/mage/quick_test_runner.clj+++ b/mage/src/mage/quick_test_runner.clj@@ -38,7 +38,7 @@(concat files dirs)))(defn- gather-tests [selecting]- (prn ["selecting" selecting])+ (u/debug ["selecting" selecting])(case selecting"file" (gather-file-tests)"dir" (gather-dir-tests)@@ -50,16 +50,20 @@(c/green "file") ", " (c/green "dir") ", or " (c/green "all") "."):babashka/exit 1}))))-(defn- quotify [xs]- (str/join " " (map #(str "\"" % "\"") xs)))+(defn- feedback-bar+ "Returns a glancable feedback bar of test results as string, with one emoji per test."+ [out-data]+ (str (str/join (repeat (:pass out-data) "✅"))+ (str/join (repeat (:fail out-data) "❌"))+ (str/join (repeat (:error out-data) "⚠️"))))-(defn- run-tests-over-nrepl [test-dirs] ;; todo add dirs+(defn- run-tests-over-nrepl [test-dirs options](let [start (u/start-timer)the-ns "mb.hawk.core"the-cmd (str "(do (require (quote metabase.test-runner)) ""((requiring-resolve 'dev.reload/reload!)) ""(metabase.test-runner/find-and-run-tests-repl "- "{:only [" (quotify test-dirs) "]}))")]+ "{:only " (pr-str test-dirs) "}))")](println "Running Code over nrepl:" (c/bold the-cmd))(bling/callout{:type :info@@ -79,27 +83,40 @@(bling/callout {:type :positive:theme :minimal:label-theme :marquee- :label "To Rerun Directly"}- (c/cyan "mage run-tests " (str/join " " test-dirs))))))+ :label "Rerun This Directly With"}+ (c/cyan "mage run-tests "+ (when-let [p (:port options)] (format "--port %s " p))+ (str/join " " test-dirs)))++ (let [out-data (try (edn/read-string out)+ (catch Exception _+ (println (c/red "Problem parsing output, raw output follows:"))+ #_:clj-kondo/ignore+ (prn out)))+ exit-code (if (zero? (+ (:fail out-data) (:error out-data))) 0 1)]+ (println (feedback-bar out-data))+ (u/exit exit-code)))));; namespaces will be converted to their file paths, so this check will work.(defn- check-arg [arg](or (str/includes? arg ".clj") (str/includes? arg "/")))-(defn add-cljy-suffix-or-throw [partial-file-path maybe-ns]+(defn- add-cljy-suffix-or-throw+ "Given a namespace, we can't tell if it's a clj or cljc file without looking for the file itself."+ [partial-file-path maybe-ns](or (first(keep (fn [suffix](let [file-path (str partial-file-path suffix)]-- (cond (str/starts-with? maybe-ns "metabase.")- ;; src/metabase/...- (let [path (str "src/" file-path)]- (and (fs/exists? (str u/project-root-directory "/" path)) path))-- (str/starts-with? maybe-ns "metabase-enterprise.")- ;; enterprise/backend/src/metabase_enterprise/...- (let [path (str "enterprise/backend/" file-path)]- (and (fs/exists? (str u/project-root-directory "/" path)) path)))))+ (cond+ ;; src/metabase/...+ (str/starts-with? maybe-ns "metabase.")+ (let [path (str "src/" file-path)]+ (and (fs/exists? (str u/project-root-directory "/" path)) path))++ ;; enterprise/backend/src/metabase_enterprise/...+ (str/starts-with? maybe-ns "metabase-enterprise.")+ (let [path (str "enterprise/backend/" file-path)]+ (and (fs/exists? (str u/project-root-directory "/" path)) path)))))[".clj" ".cljc" ".bb"]))(throw(ex-info "" {:mage/error (str "Could not find a file for namespace: "@@ -107,7 +124,7 @@". Tried appending .clj, .cljs, .cljc, and .bb -- is that a real namespace?"):babashka/exit 1}))))-(defn maybe-convert-ns-to-filename [maybe-ns]+(defn- maybe-convert-ns-to-path [maybe-ns](if (or (str/starts-with? maybe-ns "metabase.")(str/starts-with? maybe-ns "metabase-enterprise."))(-> maybe-ns@@ -123,7 +140,7 @@(let [tests (if (seq arguments)arguments(-> (gather-tests selecting)- (u/fzf-select+ (u/fzf-select!(str/join " " ["--multi""--ansi""--marker" "'✓ '"@@ -133,7 +150,7 @@"--header-border" "rounded""--preview" (str "'" u/project-root-directory "/mage/cmd/fzf_preview.clj {}'")]))str/split-lines))- test-dir-or-nss (mapv maybe-convert-ns-to-filename tests)]+ test-dir-or-nss (mapv maybe-convert-ns-to-path tests)](when-not (every? check-arg test-dir-or-nss)(throw (ex-info "" {:mage/error (str"When providing arguments, they must be file paths or directories, got: "@@ -143,21 +160,20 @@test-dir-or-nss))(defn- run-tests-cli [test-dirs]- (let [cmd (str "clj -X:dev:ee:ee-dev:test :only '[" (quotify test-dirs) "]'")]+ (let [cmd (str "clj -X:dev:ee:ee-dev:test :only '" (pr-str test-dirs) "'")](bling/callout {:label "Running Command Line"} (c/bold cmd))- (shell/sh* "clojure" "-X:dev:dev-ee:ee:test" ":only" (str "[" (quotify test-dirs) "]"))))+ (shell/sh* "clojure" "-X:dev:dev-ee:ee:test" ":only" (pr-str test-dirs))))(defn go"Interactively select directories to run tests against."[{:keys [arguments options] :as _parsed}](let [tests (setup-test-files arguments options)]- (prn ["TESTS" tests])(if (and (backend/nrepl-open?)- ;; No testing against the mage nrepl! (probably noone will hit this)+ ;; No testing against the mage nrepl!(not= :bb (backend/nrepl-type)))(do(println "Running tests via ⏩🏎️✨" (c/green (c/bold "THE REPL")) "✨🏎️⏪.")- (run-tests-over-nrepl tests))+ (run-tests-over-nrepl tests options))(do(println "Running via " (c/bold (c/magenta "the command line")) "."(c/red " This is " (c/bold "SLOW") " and " (c/bold "NOT RECCOMENDED!! "))@@ -168,6 +184,3 @@:gradient-direction :to-top:gradient-colors [:magenta :red]}))(run-tests-cli tests)))))---diff --git a/mage/src/mage/util.clj b/mage/src/mage/util.cljindex c07d4222937..3d283d0d392 100644--- a/mage/src/mage/util.clj+++ b/mage/src/mage/util.clj@@ -82,7 +82,7 @@(defn public-bb-tasks-list"Returns all public bb tasks as a vector of strings."[]- (->> "bb tasks"+ (->> (str project-root-directory "/bin/bb tasks")shl(drop 2)(map (comp first #(str/split % #"\s+")))@@ -102,6 +102,19 @@(defn updated-files"Sequence of filenames that have changes in Git relative to `diff-target`."([] (updated-files "HEAD"))+ ([diff-target]+ (->> (shell {:out :string :dir project-root-directory}+ "git" "diff" "--name-only" diff-target)+ :out+ (str/split-lines)+ ;; filter out any files that have been deleted/moved+ (remove #{""})+ (filter (fn [filename]+ (fs/exists? (str project-root-directory "/" filename)))))))++(defn updated-clojure-files+ "Sequence of filenames that have changes in Git relative to `diff-target`."+ ([] (updated-clojure-files "HEAD"))([diff-target](->> (shell {:out :string :dir project-root-directory}"git" "diff" "--name-only" diff-target@@ -200,7 +213,7 @@:mage/error (str "You don't have " (c/yellow cmd) " installed. Please install it to use this task."):babashka/exit 1}))))-(defn fzf-select+(defn fzf-select!"Uses fzf to offer interactive selections.If the user doesn't have fzf installed, explains instructions@@ -238,3 +251,13 @@(defn since-ms"Called on the return value of start-timer, returns the elapsed time in milliseconds."[timer] (/ (- (System/nanoTime) timer) 1e6))++(defn exit+ "When invoked from a babashka namespace spawned from mage, exits with the given exit code.+ Will not crash your repl. Prefer this to System/exit!"+ ([exit-code]+ (throw (ex-info "" {:mage/quiet true+ :babashka/exit exit-code})))+ ([message exit-code]+ (println message)+ (throw (ex-info "" {:mage/quiet true :babashka/exit exit-code}))))diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex eb32356b579..8c058c447b9 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -3,52 +3,38 @@[babashka.tasks :as bt][clojure.string :as str][clojure.test :as t :refer [deftest is testing]]- [mage.token-scan-test :as token-scan-test]- [mage.util :as u]))--(comment- token-scan-test/keep-me)+ [mage.token-scan-test]+ [mage.util :as u]+ [mage.util-test]))(set! *warn-on-reflection* true)-(defn bin-mage-has-help? []- (doseq [help-cmds [[] [" "] [" "]- ["-h"] ["--help"]- [" -h"] [" --help"]- [" -h"] [" --help"]]- :let [cmd (str "./bin/mage " (str/join " " help-cmds))]]- (testing (format "'%s'" (pr-str cmd))+(comment+ mage.token-scan-test/keep-me+ mage.util-test/keep-me)++(deftest bin-mage-has-help-test+ (doseq [help-cmds [[] [" "] [" "] ["-h"] ["--help"] [" -h"] [" --help"] [" -h"] [" --help"]]+ :let [cmd (str "./bin/mage " (str/join " " help-cmds))+ title (format "'%s' returns help information" (pr-str cmd))]]+ (println title)+ (testing title(let [out (u/sh cmd)]- (when-not (str/includes? out "The following tasks are available:")- (System/exit 1))))))+ (is (str/includes? out "The following tasks are available:"))))))-(defn- bb-task-has-example? [task-name]- (doseq [cmd [(str "./bin/mage " task-name " -h")+(deftest bb-task-has-example-test+ (doseq [task-name (u/public-bb-tasks-list)+ cmd [(str "./bin/mage " task-name " -h")(str "./bin/mage " task-name " --help")]]- (testing (format "'%s'" cmd)+ (println "Checking examples for command:" cmd)+ (testing (format "%s has examples with: '%s'" task-name cmd)(is (str/includes? (u/sh cmd) "Examples:")))))-(defn invalid-task-names-print-help-test []+(deftest invalid-task-names-print-help-test(doseq [task-name ["foo" "bar" "baz"]]- (let [cmd (str "./bin/mage " task-name)]- (testing (format "'%s'" cmd)- (let [result (try (bt/shell {:err :string :out :string} "./bin/mage foo")+ (let [cmd (str "./bin/mage " task-name)+ title (format "invalid task names like: '%s' print help" cmd)]+ (testing title+ (let [result (try (bt/shell {:err :string :out :string} (str "./bin/mage " task-name))(catch Exception e (:out (:proc (ex-data e)))))](is (str/includes? result "The following tasks are available:")))))))--(deftest mage-tests- (println "Running mage tests")-- (println " bb tasks have examples")- (testing "bb tasks have examples"- (mapv bb-task-has-example? (u/public-bb-tasks-list)))-- (println " bb tasks have help")- (testing "mage has help"- (bin-mage-has-help?))-- (println " invalid task names print help")- (testing "Invalid task name prints help"- (invalid-task-names-print-help-test))-- (t/run-tests 'mage.token-scan-test))diff --git a/mage/test/mage/examples_test.clj b/mage/test/mage/examples_test.cljdeleted file mode 100644index 7e237ff4cbd..00000000000--- a/mage/test/mage/examples_test.clj+++ /dev/null@@ -1,31 +0,0 @@-(ns mage.examples-test- (:require- [clojure.edn :as edn]- [mage.util :as u]))--(set! *warn-on-reflection* true)--(defn- task->task-maps []- (dissoc (:tasks (edn/read-string (slurp "bb.edn"))) :requires))--(defn- task->examples []- (update-vals (task->task-maps) :examples))--(def ^:private excluded-public-task-examples #{"nrepl"})--(defn run-tests- "This takes a while to run, so it is not run by default. To run it, use the `./bin/mage -test-examples` command."- []- (let [public-tasks (set (u/public-bb-tasks-list))]- (println "Running example tests for:" public-tasks "\n")- (doseq [[task-name examples] (task->examples)- :when (contains? public-tasks (str task-name))- :when (not (excluded-public-task-examples (str task-name)))- [cmd explaination] examples]- (println task-name)- (println "Testing that running ['" cmd "' -> '" explaination "'] returns a 0 exit-code...")- (try (u/sh cmd)- (catch Exception _ (println "Failed" task-name ":" cmd))- (println "OK"))))- (println "Done.")- :ok)diff --git a/mage/test/mage/token_scan_test.clj b/mage/test/mage/token_scan_test.cljindex a2c12f3ef3d..a72bbafcf8e 100644--- a/mage/test/mage/token_scan_test.clj+++ b/mage/test/mage/token_scan_test.clj@@ -12,7 +12,7 @@(def token-like-comment (str ";;" (apply str (repeat 100 \a)) "\n"))(def sample-file-path- (str u/project-root-directory "/test/mage/token_scan_test_example.clj"))+ (str u/project-root-directory "/" "mage/test/mage/token_scan_test_example.clj"))(deftest token-scan-finds-error(trydiff --git a/mage/test/mage/util_test.clj b/mage/test/mage/util_test.cljnew file mode 100644index 00000000000..6b7bdeb081a--- /dev/null+++ b/mage/test/mage/util_test.clj@@ -0,0 +1,30 @@+(ns mage.util-test+ (:require+ [babashka.fs :as fs]+ [clojure.string :as str]+ [clojure.test :as t :refer [deftest is]]+ [mage.color :as c]+ [mage.util :as u]))++(def ^:private new-content "MAGE_UTIL_TEST_STRING")++(defn- add-content [file-path]+ (spit file-path new-content :append true))++(defn- remove-content [file-path]+ (spit file-path+ (-> (slurp file-path)+ (str/replace (re-pattern (str new-content "$")) ""))))++(deftest updated-files-returns-all-kinds-of-files+ (let [extension->path (-> (group-by fs/extension (fs/glob u/project-root-directory "**.*"))+ (update-vals (comp str first))+ (select-keys ["clj" "cljc" "cljs" "edn" "js" "ts" "jsx" "json" "css" "html" "md"]))]+ (doseq [[_ path] extension->path]+ (println+ (str "Testing that " (c/green "u/updated-files") " picks up changes to file:" (c/cyan path)))+ (try (add-content path)+ (is (contains? (set (u/updated-files)) (str/replace path (str u/project-root-directory "/") ""))+ (str "updated-files should include modified file: " path))+ (finally (remove-content path))))))+