Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

test: agent-specific test runner (#70821)

Edward Paget647861b62452
diff --git a/.claude/skills/_shared/clojure-commands.md b/.claude/skills/_shared/clojure-commands.md
index 06ad2586d65..9d9fc0754ea 100644
--- a/.claude/skills/_shared/clojure-commands.md
+++ b/.claude/skills/_shared/clojure-commands.md
@@ -12,14 +12,7 @@
## Testing
-- **Run a test:** `./bin/mage run-tests namespace/test-name`
-- **Run all tests in a namespace:** `./bin/mage run-tests namespace`
-- **Run all tests for a module:** `./bin/mage run-tests test/metabase/notification` Because the module lives in that directory.
-
-Note: the `./bin/mage run-tests` command accepts multiple args, so you can pass
-`./bin/mage run-tests namespace/test-name namespace/other-test namespace/third-test`
-to run 3 tests, or
-`./bin/mage run-tests test/metabase/module1 test/metabase/module2` to run 2 modules.
+Use `./bin/test-agent` to run backend tests. See the top-level CLAUDE.md for usage.
## Code Readability
diff --git a/CLAUDE.md b/CLAUDE.md
index 11defa25357..9f93c456dd3 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -25,6 +25,18 @@ For detailed guidance on writing and reviewing code and documentation, see the s
**Important**: When working with frontend code, read [frontend/CLAUDE.md](frontend/CLAUDE.md) for project-specific guidelines on component preferences, styling, TypeScript migration, testing requirements, and available scripts.
+## Running Backend Tests
+
+Use `./bin/test-agent` to run Clojure tests. It produces clean, plain-text output with no progress bars or ANSI codes.
+
+```bash
+./bin/test-agent :only '[metabase.foo-test]' # run a namespace
+./bin/test-agent :only '[metabase.foo-test/some-test]' # run a single test
+./bin/test-agent :only '[metabase.foo-test metabase.bar-test]' # multiple namespaces
+```
+
+Do not use `clj -X:dev:test` directly — its progress-bar output is hard to parse.
+
## Tool Preferences
If `clojure-mcp` tools are available, prefer them over shell-based alternatives for Clojure development.
diff --git a/bin/test-agent b/bin/test-agent
new file mode 100755
index 00000000000..f444616446b
--- /dev/null
+++ b/bin/test-agent
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Agent-friendly test runner for Metabase backend (Clojure) tests.
+#
+# Differences from a normal `clj -X:dev:test` run:
+# - Uses HAWK_MODE=cli/ci so eftest.report.pretty is used instead of the
+# progress-bar reporter. Failures print inline as they happen.
+# - Strips ANSI escape codes from output so results are plain text.
+# - Filters Maven download noise and reflection warnings.
+#
+# Usage:
+# bin/test-agent :only '[metabase.foo-test]'
+# bin/test-agent :only '[metabase.foo-test/some-test]'
+# bin/test-agent # runs all tests (rarely what you want)
+#
+# Any extra arguments are forwarded directly to `clojure -X:dev:ee:ee-dev:test`.
+# The :test alias uses port 0 (OS-assigned random port) so parallel runs never collide.
+
+set -euo pipefail
+
+export HAWK_MODE=cli/ci
+
+# Filter noise and strip ANSI escape sequences
+clean_output() {
+ grep -v '^Downloading: ' \
+ | grep -v '^Reflection warning,' \
+ | grep -v '^Excluding directory' \
+ | grep -v '^Looking for test namespaces in directory' \
+ | sed $'s/\033[\[(][0-9;]*[A-Za-z]//g; s/\033[^[]*//g'
+}
+
+exec clojure -X:dev:ee:ee-dev:test "$@" 2>&1 | clean_output
diff --git a/deps.edn b/deps.edn
index c79389008c0..7bc5b758525 100644
--- a/deps.edn
+++ b/deps.edn
@@ -332,9 +332,9 @@
"-Dmb.jetty.join=false"
"-Dmb.field.filter.operators.enabled=true"
"-Dmb.api.key=test-api-key"
- ;; Different port from normal `:dev` so you can run tests on a different server.
- ;; TODO -- figure out how to do a random port like in the old project.clj?
- "-Dmb.jetty.port=3001"]}
+ ;; Port 0 tells the OS to assign a random free port, allowing parallel test runs.
+ ;; The actual port is read from the running Jetty server via server.instance/server-port.
+ "-Dmb.jetty.port=0"]}
;; run the dev server with
;; clojure -M:run
diff --git a/enterprise/backend/test/metabase_enterprise/sso/api/sso_test.clj b/enterprise/backend/test/metabase_enterprise/sso/api/sso_test.clj
index 36d0f6945cc..f83f7c1455f 100644
--- a/enterprise/backend/test/metabase_enterprise/sso/api/sso_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sso/api/sso_test.clj
@@ -3,8 +3,8 @@
[buddy.sign.jwt :as jwt]
[clojure.string :as str]
[clojure.test :refer :all]
- [metabase.config.core :as config]
[metabase.request.core :as request]
+ [metabase.server.instance :as server.instance]
[metabase.session.core :as session]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -62,7 +62,7 @@
[jwt-enabled true
jwt-identity-provider-uri "http://test.idp.metabase.com"
jwt-shared-secret default-jwt-secret
- site-url (format "http://localhost:%s" (config/config-str :mb-jetty-port))]
+ site-url (format "http://localhost:%s" (server.instance/server-port))]
(f))))
(defn- create-valid-jwt-token
diff --git a/enterprise/backend/test/metabase_enterprise/sso/integrations/oidc_test.clj b/enterprise/backend/test/metabase_enterprise/sso/integrations/oidc_test.clj
index b78a824a3e7..7f5b4973dd2 100644
--- a/enterprise/backend/test/metabase_enterprise/sso/integrations/oidc_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sso/integrations/oidc_test.clj
@@ -4,7 +4,7 @@
[clojure.test :refer :all]
[metabase-enterprise.sso.test-setup :as sso.test-setup]
[metabase.auth-identity.core :as auth-identity]
- [metabase.config.core :as config]
+ [metabase.server.instance :as server.instance]
[metabase.sso.oidc.state :as oidc.state]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -86,7 +86,7 @@
(fn []
(mt/with-temporary-setting-values
[oidc-providers [test-provider]
- site-url (format "http://localhost:%s" (config/config-str :mb-jetty-port))]
+ site-url (format "http://localhost:%s" (server.instance/server-port))]
~@body)))))))))
;;; -------------------------------------------------- Prerequisites Tests --------------------------------------------------
@@ -106,7 +106,7 @@
(mt/with-additional-premium-features #{:sso-oidc}
(mt/with-temporary-setting-values
[oidc-providers [(assoc test-provider :enabled false)]
- site-url (format "http://localhost:%s" (config/config-str :mb-jetty-port))]
+ site-url (format "http://localhost:%s" (server.instance/server-port))]
(with-ensure-encryption!
(with-successful-oidc!
(let [response (mt/client-full-response :get 400 "/auth/sso/test-idp"
diff --git a/enterprise/backend/test/metabase_enterprise/sso/test_setup.clj b/enterprise/backend/test/metabase_enterprise/sso/test_setup.clj
index 72268bb6696..949825668e2 100644
--- a/enterprise/backend/test/metabase_enterprise/sso/test_setup.clj
+++ b/enterprise/backend/test/metabase_enterprise/sso/test_setup.clj
@@ -5,9 +5,9 @@
to avoid circular dependencies between test namespaces."
(:require
[clojure.string :as str]
- [metabase.config.core :as config]
[metabase.premium-features.token-check :as token-check]
[metabase.request.core :as request]
+ [metabase.server.instance :as server.instance]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.random :as u.random]
@@ -100,7 +100,7 @@
[jwt-enabled true
jwt-identity-provider-uri default-jwt-idp-uri
jwt-shared-secret default-jwt-secret
- site-url (format "http://localhost:%s" (config/config-str :mb-jetty-port))]
+ site-url (format "http://localhost:%s" (server.instance/server-port))]
(mt/with-premium-features current-features
(f))))))
@@ -136,7 +136,7 @@
slack-connect-client-secret default-slack-client-secret
slack-connect-authentication-mode "sso"
slack-connect-user-provisioning-enabled true
- site-url (format "http://localhost:%s" (config/config-str :mb-jetty-port))]
+ site-url (format "http://localhost:%s" (server.instance/server-port))]
(mt/with-premium-features current-features
(f))))))
@@ -158,7 +158,7 @@
(mt/with-additional-premium-features #{:sso-oidc}
(mt/with-temporary-setting-values
[oidc-providers [default-oidc-provider]
- site-url (format "http://localhost:%s" (config/config-str :mb-jetty-port))]
+ site-url (format "http://localhost:%s" (server.instance/server-port))]
(mt/with-premium-features current-features
(f))))))
diff --git a/src/metabase/server/core.clj b/src/metabase/server/core.clj
index 5ef84ce99ba..7020be9500c 100644
--- a/src/metabase/server/core.clj
+++ b/src/metabase/server/core.clj
@@ -23,6 +23,7 @@
make-handler]
[metabase.server.instance
instance
+ server-port
start-web-server!
stop-web-server!]
;; TODO -- I think all of this stuff probably belongs in [[metabase.request.*]]
diff --git a/src/metabase/server/instance.clj b/src/metabase/server/instance.clj
index 6543c9a163f..611fd37d6c0 100644
--- a/src/metabase/server/instance.clj
+++ b/src/metabase/server/instance.clj
@@ -65,6 +65,13 @@
^Server []
@instance*)
+(defn server-port
+ "Return the actual port the running Jetty server is listening on. Useful when the server was started with port 0
+ (OS-assigned random port). Returns `nil` if no server is running."
+ []
+ (when-let [^Server server (instance)]
+ (.. server getURI getPort)))
+
(defn- async-proxy-handler ^ServletHandler [handler timeout]
(proxy [ServletHandler] []
(doHandle [_ ^Request base-request ^HttpServletRequest request ^HttpServletResponse response]
diff --git a/test/metabase/embedding_rest/api/embed_test.clj b/test/metabase/embedding_rest/api/embed_test.clj
index 41eb44b42df..4ab15f67f10 100644
--- a/test/metabase/embedding_rest/api/embed_test.clj
+++ b/test/metabase/embedding_rest/api/embed_test.clj
@@ -10,7 +10,6 @@
[dk.ative.docjure.spreadsheet :as spreadsheet]
[java-time.api :as t]
[metabase.api.common :as api]
- [metabase.config.core :as config]
[metabase.dashboards-rest.api-test :as api.dashboard-test]
[metabase.embedding-rest.api.common :as api.embed.common]
[metabase.parameters.chain-filter-test :as chain-filer-test]
@@ -21,6 +20,7 @@
[metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]
[metabase.query-processor.pivot.test-util :as api.pivots]
[metabase.query-processor.test-util :as qp.test-util]
+ [metabase.server.instance :as server.instance]
[metabase.test :as mt]
[metabase.test.http-client :as client]
[metabase.tiles.api-test :as tiles.api-test]
@@ -557,7 +557,7 @@
(mt/with-temp [:model/Card card (card-with-date-field-filter)]
;; make sure the URL doesn't include /api/ at the beginning like it normally would
(binding [client/*url-prefix* ""]
- (mt/with-temporary-setting-values [site-url (str "http://localhost:" (config/config-str :mb-jetty-port) client/*url-prefix*)]
+ (mt/with-temporary-setting-values [site-url (str "http://localhost:" (server.instance/server-port) client/*url-prefix*)]
(is (= "count\n107\n"
(client/real-client :get 200 (str "embed/question/" (card-token card) ".csv?date=Q1-2014"))))))))))
diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.clj
index 8141534b7f8..baa3cb0e495 100644
--- a/test/metabase/public_sharing_rest/api_test.clj
+++ b/test/metabase/public_sharing_rest/api_test.clj
@@ -8,7 +8,6 @@
[dk.ative.docjure.spreadsheet :as spreadsheet]
[metabase.analytics.snowplow-test :as snowplow-test]
[metabase.analytics.stats :as stats]
- [metabase.config.core :as config]
[metabase.dashboards-rest.api-test :as api.dashboard-test]
[metabase.parameters.chain-filter-test :as chain-filter-test]
[metabase.parameters.custom-values :as custom-values]
@@ -19,6 +18,7 @@
[metabase.query-processor.card-test :as qp.card-test]
[metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]
[metabase.query-processor.pivot.test-util :as api.pivots]
+ [metabase.server.instance :as server.instance]
[metabase.test :as mt]
[metabase.test.http-client :as client]
[metabase.test.util :as tu]
@@ -537,7 +537,7 @@
(mt/with-temp [:model/Card {uuid :public_uuid} (card-with-date-field-filter)]
;; make sure the URL doesn't include /api/ at the beginning like it normally would
(binding [client/*url-prefix* ""]
- (mt/with-temporary-setting-values [site-url (str "http://localhost:" (config/config-str :mb-jetty-port) client/*url-prefix*)]
+ (mt/with-temporary-setting-values [site-url (str "http://localhost:" (server.instance/server-port) client/*url-prefix*)]
(is (= "count\n107\n"
(client/real-client :get 200 (str "public/question/" uuid ".csv")
:parameters (json/encode [{:id "_DATE_"
diff --git a/test/metabase/server/middleware/security_test.clj b/test/metabase/server/middleware/security_test.clj
index f3ea83bfb0e..c4f20a9d9da 100644
--- a/test/metabase/server/middleware/security_test.clj
+++ b/test/metabase/server/middleware/security_test.clj
@@ -6,6 +6,7 @@
[environ.core :as env]
[metabase.config.core :as config]
[metabase.embedding.settings :as embed.settings]
+ [metabase.server.instance :as server.instance]
[metabase.server.middleware.security :as mw.security]
[metabase.server.settings :as server.settings]
[metabase.test :as mt]
@@ -90,7 +91,7 @@
;; should be the script tags for the webpack bundles
(assert (= path "frontend_client/index.html"))
(render-file "frontend_client/index_template.html" variables))]
- (let [response (http/get (str "http://localhost:" (config/config-str :mb-jetty-port)))
+ (let [response (http/get (str "http://localhost:" (server.instance/server-port)))
nonce (json/decode @nonceJSON)
csp (get-in response [:headers "Content-Security-Policy"])
style-src (->> (str/split csp #"; *")
diff --git a/test/metabase/test/http_client.clj b/test/metabase/test/http_client.clj
index 7999b4a2c31..e8f3e9950db 100644
--- a/test/metabase/test/http_client.clj
+++ b/test/metabase/test/http_client.clj
@@ -11,6 +11,7 @@
[malli.core :as mc]
[medley.core :as m]
[metabase.config.core :as config]
+ [metabase.server.instance :as server.instance]
[metabase.server.middleware.session :as mw.session]
[metabase.server.streaming-response :as streaming-response]
[metabase.server.test-handler :as server.test-handler]
@@ -58,14 +59,17 @@
[(encode-key-value k value-or-values)]))))))
(defn build-url
- "Build an API URL for `localhost` and `MB_JETTY_PORT` with `query-parameters`.
+ "Build an API URL for `localhost` with `query-parameters`. Uses the actual port from the running Jetty server
+ (supporting port 0 / OS-assigned random ports), falling back to `MB_JETTY_PORT` config if the server isn't started.
(build-url \"db/1\" {:x true}) -> \"http://localhost:3000/api/db/1?x=true\""
[url query-parameters]
{:pre [(string? url) (u/maybe? map? query-parameters)]}
- (let [url (if (= (first url) \/) url (str "/" url))]
+ (let [url (if (= (first url) \/) url (str "/" url))
+ port (or (server.instance/server-port)
+ (config/config-str :mb-jetty-port))]
(str "http://localhost:"
- (config/config-str :mb-jetty-port)
+ port
*url-prefix*
url
(when (seq query-parameters)
diff --git a/test/metabase/test/http_client_test.clj b/test/metabase/test/http_client_test.clj
index ba60f4ca85a..e00fabf204b 100644
--- a/test/metabase/test/http_client_test.clj
+++ b/test/metabase/test/http_client_test.clj
@@ -1,18 +1,17 @@
(ns metabase.test.http-client-test
(:require
[clojure.test :refer :all]
- [metabase.config.core :as config]
[metabase.test.http-client :as client]))
(deftest ^:parallel build-url-test
(binding [client/*url-prefix* "/api"]
(testing "correctly encode all data types"
- (is (= (format "http://localhost:%s/api/database/1?int=1&float=1.23&string=a&keyword=b&seq=1&seq=2&seq=3" (config/config-str :mb-jetty-port))
- (client/build-url "database/1" {:int 1
- :float 1.23
- :string "a"
- :keyword :b
- :seq [1 2 3]}))))))
+ (is (re-matches #"http://localhost:\d+/api/database/1\?int=1&float=1\.23&string=a&keyword=b&seq=1&seq=2&seq=3"
+ (client/build-url "database/1" {:int 1
+ :float 1.23
+ :string "a"
+ :keyword :b
+ :seq [1 2 3]}))))))
(deftest ^:parallel parse-http-client-args-test
(testing "parse correctly"
diff --git a/test/metabase/test/initialize/web_server.clj b/test/metabase/test/initialize/web_server.clj
index 72902cfa7e4..13bc156bfc9 100644
--- a/test/metabase/test/initialize/web_server.clj
+++ b/test/metabase/test/initialize/web_server.clj
@@ -12,7 +12,7 @@
(defn init! []
(try
(server/start-web-server! (server.test-handler/test-handler))
- (log/infof "Started test server on port %d" (config/config-int :mb-jetty-port))
+ (log/infof "Started test server on port %d" (server/server-port))
(catch Throwable e
(log/fatal e "Web server failed to start")
(when config/is-test?