The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Mage run tests via nrepl (#64236)
Oleksandr Solovyov899d6edb3c86
diff --git a/CLAUDE.md b/CLAUDE.mdindex e15da3fe685..717013c7d59 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -29,7 +29,8 @@- Use the linter as a way to know that you are adhering to conventions in place in the codebase- **Lint Changes:** `./bin/mage kondo-updated HEAD`- **Format:** `./bin/mage cljfmt-files [path]`-- **Test file:** `clojure -X:dev:test :only namespace/test-name`+- **Run a test:** `./bin-mage run-tests namespace/test-name`+- **Run all tests in a namespace:** `./bin-mage run-tests namespace`- **Check Code Readability** `./bin/mage -check-readable` with optional line-number- Run this after every change to Clojure code, only accept readable code- **Evaluating Clojure Code** `./bin/mage -repl '<code>'`diff --git a/bb.edn b/bb.ednindex 394e69b57f8..d40f98372ae 100644--- a/bb.edn+++ b/bb.edn@@ -1,4 +1,4 @@-{:min-bb-version "1.12.197"+{:min-bb-version "1.12.208";; we put path as bin, and everything is in the ./mage subdirectory,;; so the namespaces are mage.cli, mage.format, etc.:paths ["mage" "test" "bin/lint-migrations-file/src"]@@ -161,7 +161,7 @@:task (stats-repl/connect (cli/parse! (current-task)))}notify- {:doc "Send notification for a comment (that already exists"+ {:doc "Send notification for a comment (that already exists)":requires [[mage.be-dev :refer [nrepl-eval]]]:options [["-c" "--comment-id ID" "Notify about this comment id"]["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]@@ -172,6 +172,25 @@(nrepl-eval "metabase-enterprise.comments.core" code (:port opts))(nrepl-eval "metabase-enterprise.comments.core" code)))}+ run-tests+ {:doc "Run a tests (or a single test) against nrepl server"+ :requires [[clojure.string :as str]+ [mage.be-dev :refer [nrepl-eval]]]+ :arg-schema [:tuple [:string {:name "NS/TEST" :desc "or just NS"}]]+ :options [["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]+ :task (let [{:keys [options]+ [test] :arguments} (cli/parse! (current-task))+ test (if (or (str/starts-with? test "test/")+ (str/starts-with? test "enterprise/"))+ ;; paths should come in as strings+ (format "\"%s\"" test)+ ;; ns or var should be a symbol+ (str "'" test))+ code (format+ "(do ((requiring-resolve 'dev.reload/reload!)) (find-and-run-tests-repl {:only %s}))"+ test)]+ (nrepl-eval "mb.hawk.core" (str code) (:port options)))}+ls {:doc "List all mage public and private tasks":examples [["./bin/mage ls" "List all tasks"]]:requires [[mage.util :as u]]diff --git a/dev/src/dev/reload.clj b/dev/src/dev/reload.cljnew file mode 100644index 00000000000..6dfed0fd03b--- /dev/null+++ b/dev/src/dev/reload.clj@@ -0,0 +1,151 @@+(ns dev.reload+ (:require+ [clojure.java.shell :as shell]+ [clojure.string :as str]+ [metabase.util :as u])+ (:import+ (java.io File)+ (java.nio.file Files LinkOption Paths)))++(set! *warn-on-reflection* true)++#_:clj-kondo/ignore+(defonce *reload-timestamps (atom {}))++(defn system-classpath+ "Returns a sequence of File paths from the 'java.class.path' system+ property."+ []+ (into [] (comp (filter #(not (str/ends-with? % ".jar")))+ (map #(File. ^String %))+ (filter #(.isDirectory ^File %))+ (map #(.getCanonicalPath ^File %)))+ (.split (System/getProperty "java.class.path")+ (System/getProperty "path.separator"))))++(defn find-classpath-root+ "Finds the classpath directory that contains the given file path.+ Returns the portion of the path to strip, or nil if not found."+ [classpath file-path]+ (let [canonical-file (try+ (.getCanonicalPath (File. ^String file-path))+ (catch Exception _ file-path))]+ (->> classpath+ (filter #(str/starts-with? canonical-file %))+ (sort-by count >) ; Longest match first+ first)))++(defn git-changed-files+ "Returns a set of changed/added/untracked files from git status."+ []+ (let [{:keys [exit out]} (shell/sh "git" "status" "--porcelain")]+ (if (and (zero? exit)+ (not (str/blank? out)))+ (->> (str/split-lines out)+ (map #(str/trim (subs % 3))) ; Remove status prefix (e.g., "M ", "A ", "??")+ (filter #(str/ends-with? % ".clj"))+ set)+ #{})))++(defn file-last-modified+ "Returns the last modified time of a file in milliseconds."+ [file-path]+ (try+ (-> (Paths/get file-path (into-array String []))+ (Files/getLastModifiedTime (into-array LinkOption []))+ (.toMillis))+ (catch Exception _ nil)))++(defn relpath->ns+ "foo/bar_baz.clj -> foo.bar-baz"+ [relpath]+ (-> relpath+ (str/replace #"\.clj$" "")+ (str/replace #"/" ".")+ (str/replace #"_" "-")+ symbol))++(defn file-path->namespace+ "Converts a file path to a namespace symbol using classpath information.+ Example: enterprise/backend/src/foo/bar/baz.clj -> foo.bar.baz"+ [file-path classpath]+ (when (and file-path (re-find #".cljc?$" file-path))+ (when-let [root (find-classpath-root classpath file-path)]+ (let [canonical-file (try+ (.getCanonicalPath (File. ^String file-path))+ (catch Exception _ file-path))+ relative-path (subs canonical-file (inc (count root)))]+ (relpath->ns relative-path)))))++(defn loaded-namespaces+ "Returns a set of all currently loaded namespace symbols."+ []+ (set (map ns-name (all-ns))))++(defn needs-reload?+ "Checks if a file needs reloading based on its last modified time."+ [file-path ns-sym]+ (let [last-reload (get @*reload-timestamps ns-sym)+ last-modified (file-last-modified file-path)]+ (or (nil? last-reload)+ (nil? last-modified)+ (> last-modified last-reload))))++(defn changed-namespaces []+ (let [changed-files (git-changed-files)+ loaded-ns (loaded-namespaces)+ classpath-dirs (system-classpath)+ candidates (->> changed-files+ (map (fn [f] {:file f :ns (file-path->namespace f classpath-dirs)}))+ (filter :ns)+ (filter #(loaded-ns (:ns %))))+ to-reload (filter #(needs-reload? (:file %) (:ns %)) candidates)+ skipped (remove #(needs-reload? (:file %) (:ns %)) candidates)]+ {:to-reload to-reload+ :skipped skipped}))++(defn reload-namespaces!+ [to-reload]+ (->> (for [{:keys [_file ns]} to-reload]+ (try+ (require ns :reload)+ (swap! *reload-timestamps assoc ns (System/currentTimeMillis))+ [:reloaded ns]+ (catch Exception e+ [:failed {:ns ns :error (.getMessage e)}])))+ (u/group-by first second)))++(defn reload!+ "Reloads all changed files from git that are already loaded as namespaces+ and have been modified since last reload.+ Returns a map with :reloaded, :skipped, and :failed keys."+ []+ (let [{:keys [to-reload+ skipped]} (changed-namespaces)+ _ (println (format "Reloading %s namespaces (%s skipped):" (count to-reload) (count skipped)))+ {:keys [reloaded+ failed]+ :as res} (reload-namespaces! to-reload)]+ (doseq [ns reloaded]+ (println " ✓" ns))+ (doseq [{:keys [ns error]} failed]+ (println " ✗" ns ":" error))+ (assoc res :skipped skipped)))++(defn clear-reload-history!+ "Clears the reload timestamp history, forcing all files to reload next time."+ []+ (reset! *reload-timestamps {})+ (println "Reload history cleared."))++(defn show-classpath+ "Shows all classpath directories for debugging."+ []+ (println "\n=== Classpath Directories ===")+ (run! #(println (.getCanonicalPath ^File %)) (sort (system-classpath))))++(comment+ (show-classpath) ; to see what's in your classpath+ (clear-reload-history!) ; if you want to force reload everything+ (changed-namespaces)+ (reload!))diff --git a/mage/mage/be_dev.clj b/mage/mage/be_dev.cljindex 5986f2d9208..e63608336e5 100644--- a/mage/mage/be_dev.clj+++ b/mage/mage/be_dev.clj@@ -4,8 +4,11 @@^:clj-kondo/ignore[clojure.pprint :as pp][clojure.string :as str]+ [clojure.walk :as walk][mage.color :as c]- [mage.util :as u]))+ [mage.util :as u])+ (:import+ [java.io InputStream]))(set! *warn-on-reflection* true)@@ -13,87 +16,88 @@"Parse a string into an integer. Returns nil if parsing fails. Returns the input unchanged if not a string."[s-or-i](if (string? s-or-i)- (let [s (str/trim s-or-i)]- (try- (Integer/parseInt s)- (catch Exception _ nil)))+ (try+ (parse-long (str/trim s-or-i))+ (catch Exception _ nil))s-or-i))-(defn- print-capture-code- "Capture output and return it as strings along with the value from the orignal code."- [code-string]- (str "-(let [o# (new java.io.StringWriter)- e# (new java.io.StringWriter)]- (binding [*out* o#- *err* e#]- {:value (do" code-string ")- :stdout (str o#)- :stderr (str e#)}))"))+(defn ^:private nrepl-port [port]+ (safe-parse-int+ (or port+ (try (slurp ".nrepl-port")+ (catch Exception _+ (throw (ex-info+ (str "Unable to find .nrepl-port, is the server's repl running?"+ " See: the :dev-start alias in deps.edn")+ {})))))))++(defn ^:private socket!+ "Acquire a Java socket or die."+ [host port]+ (try+ (java.net.Socket. host port)+ (catch java.net.ConnectException _+ (println+ (str "Could not connect to the REPL server on port: " (c/red port) " (found port number in .nrepl-port).\n"+ "Is the Metabase backend running?\n\n"+ "To start it, run:\n"+ (c/green " clj -M:dev:ee:ee-dev:drivers:drivers-dev:dev-start\n\n")+ "If you prefer a different way to start the backend, please use that instead."))+ (System/exit 1))+ (catch Exception e+ (if (= "No matching ctor found for class java.net.Socket" (ex-message e))+ (do+ (println (format "There seems to be an error specifying port: '%s'."+ (c/yellow (pr-str port))))+ (System/exit 1))+ (throw e)))))++(defn ^:private consume [v]+ (cond+ (bytes? v) (String. ^bytes v)+ (instance? InputStream v) (slurp v)+ :else v))(defn eval-in-ns"This insane code evals the code in the proper namespace.It's basically a repl inside a repl."[nns code]- (str "- (let [ns-sym (symbol \"" nns "\")]- (require ns-sym :reload)- (in-ns ns-sym)- (eval (read-string " (pr-str (or code "::loaded")) ")))"))+ ^:clj-kondo/ignore ;; ignore `eval`+ `(let [ns# (symbol ~nns)]+ (require ns# :reload)+ (in-ns ns#)+ (eval (read-string ~(or code "::loaded")))))(defn nrepl-eval"Evaluate Clojure code in a running nREPL server. With one arg, reads port from .nrepl-port file.With two args, uses the provided port number. Returns and formats the evaluation results."- ([nns code]- (nrepl-eval (or nns "user") code (try (slurp ".nrepl-port")- (catch Exception _- (throw (ex-info- (str- "Unable to find .nrepl-port, is the server's repl running?"- " See: the :dev-start alias in deps.edn") {}))))))- ([nns code port]- (try (let [port (safe-parse-int port)- s (java.net.Socket. "localhost" port)- out (.getOutputStream s)- in (java.io.PushbackInputStream. (.getInputStream s))- code-str (->> code- (eval-in-ns nns)- print-capture-code)- _ (u/debug "Code: ----- \n" code-str "\n -----")- _ (bencode/write-bencode out {"op" "eval" "code" code-str})- return (update-vals (bencode/read-bencode in) slurp)]- (doseq [[k v] return]- (if (= k "value")- (if-let [v (try (read-string v)- ;; try to read v, which is a map but comes back as a string:- (catch Exception _ nil))]- (do- (println "value: " (pr-str (:value v)))- (when-not (str/blank? (:stdout v))- (println "stdout: ")- (doseq [out (str/split-lines (:stdout v))]- (println " " out)))- (when-not (str/blank? (:stderr v))- (println "stderr: ")- (doseq [err (str/split-lines (:stderr v))]- (println " " err))))- (println "value: " v))- (println (str k ":") v))))- (catch java.net.ConnectException _- (println (str "Could not connect to the REPL server on port: " (c/red port) " (found port number in .nrepl-port).\n"- "Is the Metabase backend running?\n\n"- "To start it, run:\n"- (c/green " clj -M:dev:ee:ee-dev:drivers:drivers-dev:dev-start\n\n")- "If you prefer a different way to start the backend, please use that instead.")))- (catch Exception e- (if (= "No matching ctor found for class java.net.Socket" (ex-message e))- (println (str "Unable to connect to nREPL server. Is it running on port " (c/yellow port) "?"))- (do- (println "message: " (ex-message e))- (println "data: " (pr-str (ex-data e)))- {:exception true- :message (ex-message e)- :data (ex-data e)}))))))+ [nns code & [port]]+ (let [port (nrepl-port port)+ s (socket! "localhost" port)+ out (.getOutputStream s)+ in (java.io.PushbackInputStream. (.getInputStream s))+ code-str (str (eval-in-ns nns code))+ _ (u/debug "Code:\n-----\n" code-str "\n-----")+ _ (bencode/write-bencode out {:op "eval" :code code-str})+ final-value (atom nil)]+ (loop []+ (let [response (->> (bencode/read-bencode in)+ (walk/postwalk consume))]+ (doseq [[k v] response]+ (case k+ "out" (print v)+ "err" (binding [*out* *err*]+ (print v))+ "value" (reset! final-value v)+ nil)) ; Ignore other keys like session, id, status++ ;; Flush to ensure output appears immediately+ (flush)++ (if (some #{"done"} (get response "status"))+ (some->> @final-value+ (println "\n=> "))+ (recur))))))(comment@@ -104,6 +108,6 @@(nrepl-eval "dev.migrate" "(do (rollback! :count 1) ::rollback-done)")(nrepl-eval "dev.migrate" "(do (migrate! :up) ::migrate-up-done)")- (nrepl-eval "")+ (nrepl-eval "dev" "")(println code-str))diff --git a/mage/mage/cli.clj b/mage/mage/cli.cljindex e624b124532..6ba681a8574 100644--- a/mage/mage/cli.clj+++ b/mage/mage/cli.clj@@ -14,11 +14,10 @@(arg-name nil arg))([idx arg];; TODO: support `:maybe` as an optional argument?- (if-let [name (:arg/name (mc/properties arg))]- name- (cond-> "arg"- ;; Don't zero index generated argument names- idx (str (inc idx))))))+ (or (:name (mc/properties arg))+ (cond-> "arg"+ ;; Don't zero index generated argument names+ idx (str (inc idx))))))(defn- seq-arg-name[arg]diff --git a/mage/mage/util.clj b/mage/mage/util.cljindex 73b32d91ffa..442564748a4 100644--- a/mage/mage/util.clj+++ b/mage/mage/util.clj@@ -65,7 +65,7 @@(filter (fn [[k _]] (re-find (re-pattern (str "(?i).*" match ".*")) k)))(sort-by first))](println (c/underline- (str "Environemnt Variables" (when (not= ".*" match) (str " containing '" match "'")) " :")))+ (str "Environment Variables" (when (not= ".*" match) (str " containing '" match "'")) " :")))(doseq [[setting value] important-env](print (c/yellow setting))(print (c/white "="))