Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Record CLI usage analytics from the REST API middleware (#79152)

Sébastien2627e335f310
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index fa2615b5881..7ce8fc34192 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2081,7 +2081,8 @@
metabase.server.middleware.session
metabase.server.settings
metabase.server.streaming-response} ; TODO -- too many API namespaces
- :uses #{analytics
+ :uses #{agent-api
+ analytics
analytics-interface
api
api-keys
diff --git a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj
index 94c38e5b742..676ff96ed2c 100644
--- a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj
@@ -1,11 +1,13 @@
(ns metabase-enterprise.agent-api.api-test
- "Integration tests for the Agent API usage instrumentation in `metabase.agent-api.api` — driving
- the real `routes` wrapper so the recording happens on the same synchronous thread, where
- identity, status, duration, and PII are all in scope. Recording runs on every EE instance
- (`:feature :none`); PII is gated by `analytics-pii-retention-enabled` (itself `:audit-app`-gated)."
+ "Integration tests for CLI usage recording via the `wrap-record-cli-usage` middleware in
+ `metabase.agent-api.usage`. Drives the middleware directly so recording fires on the same
+ synchronous thread, where identity, status, duration, and PII are all in scope. Recording
+ runs on every EE instance (`:feature :none`); PII is gated by `analytics-pii-retention-enabled`
+ (itself `:audit-app`-gated)."
(:require
[clojure.test :refer [deftest is testing use-fixtures]]
- [metabase.agent-api.api :as agent-api.api]
+ [metabase.agent-api.usage :as agent-api.usage]
+ [metabase.api.common :as api]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[toucan2.core :as t2]))
@@ -15,63 +17,117 @@
(use-fixtures :once (fixtures/initialize :db :test-users))
(defn- cleanup-by-ip!
- "Isolate + clean up a test's rows by the unique fake ip_address it recorded (PII, so present only
- when retention is on)."
[& conditions]
(apply t2/delete! :model/AgentApiCallLog conditions))
-(defn- random-ip
- "A random, well-formed IPv4 address for test-row isolation. Well-formed so it survives
- `request/ip-address`'s non-IP-character stripping unchanged, and thus round-trips into
- `ip_address` for a precise lookup/cleanup key."
- []
+(defn- random-ip []
(format "10.%d.%d.%d" (rand-int 250) (rand-int 250) (rand-int 250)))
-(defn- invoke-routes
- "Drive `agent-api.api/routes` synchronously and return the response. `extra` is merged onto a
- minimal authenticated GET /api/agent/v1/ping request. `:uri` is the full public path (what the
- recorder stores as `operation`); `:path-info` is the context-stripped path the router matches on."
- [extra]
- (let [p (promise)]
- (agent-api.api/routes
- (merge {:request-method :get
- :uri "/api/agent/v1/ping"
- :path-info "/v1/ping"
- :metabase-user-id (mt/user->id :rasta)}
- extra)
- (fn [resp] (deliver p resp))
- (fn [e] (deliver p e)))
- (deref p 5000 :timeout)))
+(defn- invoke-middleware
+ "Drive `wrap-record-cli-usage` synchronously and return the response. Wraps a stub handler that
+ returns the given `handler-response` (default 200 OK). The middleware's respond callback fires
+ on the same thread so identity/PII/duration are in scope."
+ ([request]
+ (invoke-middleware request {:status 200, :body "ok"}))
+ ([request handler-response]
+ (let [p (promise)
+ handler (agent-api.usage/wrap-record-cli-usage
+ (fn [_req respond _raise]
+ (respond handler-response)))]
+ (binding [api/*current-user-id* (:metabase-user-id request)]
+ (handler request
+ (fn [resp] (deliver p resp))
+ (fn [e] (deliver p e))))
+ (deref p 5000 :timeout))))
-(deftest routes-records-direct-http-call-test
- (testing "a direct Agent API call is recorded with client classified from the User-Agent"
+(deftest middleware-records-cli-request-test
+ (testing "a CLI REST API call is recorded with client classified from the User-Agent"
(mt/with-premium-features #{:audit-app}
(mt/with-temporary-setting-values [analytics-pii-retention-enabled true]
(let [ip (random-ip)]
(try
- (let [resp (invoke-routes {:headers {"user-agent" "metabase-cli/4.5.6"}
- :remote-addr ip})]
+ (let [resp (invoke-middleware {:request-method :get
+ :uri "/api/card/42"
+ :headers {"user-agent" "metabase-cli/4.5.6"}
+ :remote-addr ip
+ :metabase-user-id (mt/user->id :rasta)})]
(is (= 200 (:status resp))))
(let [row (t2/select-one :model/AgentApiCallLog :ip_address ip)]
- (is (some? row) "a row is recorded for the direct HTTP call")
+ (is (some? row) "a row is recorded for the CLI call")
(is (= "metabase-cli" (:client_name row)))
- (is (= "GET /api/agent/v1/ping" (:operation row)))
+ (is (= "GET /api/card/:id" (:operation row)) "numeric segments are templatized")
(is (= "success" (:status row)))
(is (= (mt/user->id :rasta) (:user_id row)))
(is (nat-int? (:duration_ms row))))
(finally (cleanup-by-ip! :ip_address ip))))))))
-(deftest routes-skips-synthetic-internal-request-test
- (testing "MCP's synthetic in-process dispatch (:agent-api-internal-request? true) is NOT recorded,
- so it never double-counts against the row already written to mcp_tool_call_log"
+(deftest middleware-skips-non-cli-request-test
+ (testing "a non-CLI request does not produce a row"
(mt/with-premium-features #{:audit-app}
(mt/with-temporary-setting-values [analytics-pii-retention-enabled true]
(let [ip (random-ip)]
(try
- (let [resp (invoke-routes {:headers {"user-agent" "metabase-cli/4.5.6"}
- :remote-addr ip
- :agent-api-internal-request? true})]
+ (let [resp (invoke-middleware {:request-method :get
+ :uri "/api/card"
+ :headers {"user-agent" "Mozilla/5.0"}
+ :remote-addr ip
+ :metabase-user-id (mt/user->id :rasta)})]
(is (= 200 (:status resp))))
(is (nil? (t2/select-one :model/AgentApiCallLog :ip_address ip))
- "no agent_api_call_log row is written for the synthetic internal request")
+ "no row is written for a non-CLI request")
(finally (cleanup-by-ip! :ip_address ip))))))))
+
+(deftest middleware-skips-non-api-request-test
+ (testing "a CLI request to a non-/api/ path does not produce a row"
+ (mt/with-premium-features #{:audit-app}
+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled true]
+ (let [ip (random-ip)]
+ (try
+ (let [resp (invoke-middleware {:request-method :get
+ :uri "/health"
+ :headers {"user-agent" "metabase-cli/4.5.6"}
+ :remote-addr ip
+ :metabase-user-id (mt/user->id :rasta)})]
+ (is (= 200 (:status resp))))
+ (is (nil? (t2/select-one :model/AgentApiCallLog :ip_address ip))
+ "no row is written for a non-/api/ path")
+ (finally (cleanup-by-ip! :ip_address ip))))))))
+
+(deftest middleware-records-error-response-test
+ (testing "an error response is recorded with status=error and error_message (when PII is on)"
+ (mt/with-premium-features #{:audit-app}
+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled true]
+ (let [ip (random-ip)]
+ (try
+ (let [resp (invoke-middleware {:request-method :get
+ :uri "/api/card/999999"
+ :headers {"user-agent" "metabase-cli/4.5.6"}
+ :remote-addr ip
+ :metabase-user-id (mt/user->id :rasta)}
+ {:status 404, :body "Not found."})]
+ (is (= 404 (:status resp))))
+ (let [row (t2/select-one :model/AgentApiCallLog :ip_address ip)]
+ (is (some? row) "a row is recorded for the error call")
+ (is (= "error" (:status row)))
+ (is (= "Not found." (:error_message row)) "error_message is captured from the body")
+ (is (= "GET /api/card/:id" (:operation row))))
+ (finally (cleanup-by-ip! :ip_address ip))))))))
+
+(deftest middleware-pii-gating-test
+ (testing "ip_address and error_message are nil when PII retention is off"
+ (mt/with-premium-features #{}
+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled false]
+ (let [ip (random-ip)
+ count-before (t2/count :model/AgentApiCallLog)]
+ (invoke-middleware {:request-method :get
+ :uri "/api/card/999999"
+ :headers {"user-agent" "metabase-cli/4.5.6"}
+ :remote-addr ip
+ :metabase-user-id (mt/user->id :rasta)}
+ {:status 404, :body "Not found."})
+ ;; Can't look up by ip (it's nil when PII is off), so check by count
+ (let [count-after (t2/count :model/AgentApiCallLog)]
+ (when (is (= (inc count-before) count-after) "a row was still recorded")
+ (let [row (t2/select-one :model/AgentApiCallLog {:order-by [[:id :desc]]})]
+ (is (nil? (:ip_address row)) "ip_address is nil when PII is off")
+ (is (nil? (:error_message row)) "error_message is nil when PII is off")))))))))
diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.clj
index ed769f6338f..c2fd976ee3c 100644
--- a/src/metabase/agent_api/api.clj
+++ b/src/metabase/agent_api/api.clj
@@ -4,7 +4,6 @@
(:require
[clojure.string :as str]
[metabase.agent-api.settings :as agent-api.settings]
- [metabase.agent-api.usage :as agent-api.usage]
[metabase.agent-api.validation :as agent-api.validation]
[metabase.ai-tracing.core :as ait]
[metabase.api.common :as api]
@@ -1810,35 +1809,6 @@
(def ^:private base-routes
(api.macros/ns-handler *ns* +auth))
-(defn- error-message-from-response
- "Best-effort human-readable error string from an agent-api error `response`, for the gated
- `error_message` column. Keyword lookups are nil-safe, so a streaming/opaque body just yields nil."
- [response]
- (let [body (:body response)]
- (or (:message body) (:error body))))
-
-(defn- record-agent-api-usage!
- "Record one `agent_api_call_log` row for a completed direct Agent API HTTP call (EE-only).
- Best-effort: the EE writer wraps the insert in try/catch, so a failure to record is logged and
- swallowed — analytics never fails the request. Skips the synthetic in-process requests MCP
- dispatches through here — those are already counted in `mcp_tool_call_log`, so recording them
- again would double-count. `status`, `duration_ms`, IP, and User-Agent are all in scope here, and
- `+auth` has bound `*current-user-id*` by the time this respond fires so direct HTTP callers get
- attributed."
- [request response timer]
- (when-not (:agent-api-internal-request? request)
- (let [status-code (:status response)
- error? (or (nil? status-code) (>= status-code 400))]
- (agent-api.usage/record-agent-api-call!
- {:user-id (or (:metabase-user-id request) api/*current-user-id*)
- :tenant-id (some-> api/*current-user* deref :tenant_id)
- :user-agent (get-in request [:headers "user-agent"])
- :operation (str (some-> (:request-method request) name u/upper-case-en) " " (:uri request))
- :status (if error? "error" "success")
- :duration-ms (long (u/since-ms timer))
- :ip-address (request/ip-address request)
- :error-message (when error? (error-message-from-response response))}))))
-
(def ^{:arglists '([request respond raise])} routes
"`/api/agent/` routes."
;; Wrapped in `handler-with-open-api-spec` so the handler still implements `OpenAPISpec` for
@@ -1850,37 +1820,33 @@
;; Agent-API endpoints are synchronous, so `respond` fires inside the span and the span
;; closes after the handler returns.
(fn [request respond raise]
- (let [timer (u/start-timer)]
- (ait/with-eval-session nil
- (ait/eval-span (str "agent-api." (some-> (:request-method request) name) " " (:uri request))
- {:http/method (some-> (:request-method request) name)
- :http/uri (:uri request)
- :http/request (:body request)
- :http/user-id (or (:metabase-user-id request) api/*current-user-id*)}
- (base-routes request
- ;; Relies on `respond` firing synchronously on this thread (see
- ;; above): if an endpoint ever responds async, `*parent*` is
- ;; unbound there and this `record!` no-ops, so the span captures
- ;; no status/response. Both fail soft; the trace is just incomplete
- ;; for async agent-api responses.
- (fn eval-traced-respond [response]
- (when (ait/capture-active?)
- ;; `+auth` binds `*current-user-id*` inside `base-routes`, so it
- ;; is unbound when the span opened above but set by the time this
- ;; respond fires — record the user id here so direct HTTP callers
- ;; (not just the MCP path, which carries `:metabase-user-id`) get it.
- (ait/record! {:http/status (:status response)
- ;; Only record a plain data body. A streaming/opaque
- ;; body (not a coll) would otherwise be stringified
- ;; by the log sink into a useless `#object[…]`, so
- ;; skip it — the trace just omits the response there.
- :http/response (when (coll? (:body response))
- (:body response))
- :http/user-id (or (:metabase-user-id request)
- api/*current-user-id*)}))
- ;; CLI usage analytics: one lean row per direct HTTP call, on the
- ;; same synchronous thread so identity/PII/duration are all in scope.
- (record-agent-api-usage! request response timer)
- (respond response))
- raise)))))
+ (ait/with-eval-session nil
+ (ait/eval-span (str "agent-api." (some-> (:request-method request) name) " " (:uri request))
+ {:http/method (some-> (:request-method request) name)
+ :http/uri (:uri request)
+ :http/request (:body request)
+ :http/user-id (or (:metabase-user-id request) api/*current-user-id*)}
+ (base-routes request
+ ;; Relies on `respond` firing synchronously on this thread (see
+ ;; above): if an endpoint ever responds async, `*parent*` is
+ ;; unbound there and this `record!` no-ops, so the span captures
+ ;; no status/response. Both fail soft; the trace is just incomplete
+ ;; for async agent-api responses.
+ (fn eval-traced-respond [response]
+ (when (ait/capture-active?)
+ ;; `+auth` binds `*current-user-id*` inside `base-routes`, so it
+ ;; is unbound when the span opened above but set by the time this
+ ;; respond fires — record the user id here so direct HTTP callers
+ ;; (not just the MCP path, which carries `:metabase-user-id`) get it.
+ (ait/record! {:http/status (:status response)
+ ;; Only record a plain data body. A streaming/opaque
+ ;; body (not a coll) would otherwise be stringified
+ ;; by the log sink into a useless `#object[…]`, so
+ ;; skip it — the trace just omits the response there.
+ :http/response (when (coll? (:body response))
+ (:body response))
+ :http/user-id (or (:metabase-user-id request)
+ api/*current-user-id*)}))
+ (respond response))
+ raise))))
(fn [prefix] (open-api/open-api-spec base-routes prefix))))
diff --git a/src/metabase/agent_api/usage.clj b/src/metabase/agent_api/usage.clj
index e418e6f14e0..fb1d940926d 100644
--- a/src/metabase/agent_api/usage.clj
+++ b/src/metabase/agent_api/usage.clj
@@ -1,9 +1,12 @@
(ns metabase.agent-api.usage
"Agent API (CLI) usage logging.
- One write point feeds the analytics table: a lean `agent_api_call_log` row per direct Agent
- API HTTP call, recorded from the `routes` respond callback in `metabase.agent-api.api`. The
- write is a `defenterprise` no-op in OSS — an OSS instance records nothing — with the real
+ [[wrap-record-cli-usage]] is the single write point: a Ring middleware in the main handler
+ stack that records one `agent_api_call_log` row for every `/api/*` HTTP call whose
+ `User-Agent` identifies the Metabase CLI. MCP's synthetic in-process dispatch bypasses the
+ Ring stack entirely, so it's never double-counted (those calls land in `mcp_tool_call_log`).
+
+ The write is a `defenterprise` no-op in OSS — an OSS instance records nothing — with the real
insert in `metabase-enterprise.agent-api.usage`. Like `ai_usage_log`, collection runs on every
EE instance (`:feature :none`); the `:audit-app` feature gates the surfaces that read these
rows (the audit view + page), not the writing.
@@ -14,8 +17,11 @@
the caller's self-reported `User-Agent` (analytics only — never used to gate access)."
(:require
[clojure.string :as str]
+ [metabase.api.common :as api]
[metabase.premium-features.core :refer [defenterprise]]
- [metabase.util :as u]))
+ [metabase.request.core :as request]
+ [metabase.util :as u]
+ [metabase.util.json :as json]))
(def supported-client-keys
"Canonical client keys [[detect-client]] classifies callers into for analytics. Keep in sync
@@ -43,3 +49,55 @@
metabase-enterprise.agent-api.usage
[_call-info]
nil)
+
+(defn templatize-uri
+ "Replace variable path segments in `uri` with `:id` (numeric) or `:uuid` (UUID v4) to reduce
+ cardinality for analytics. `/api/card/134` → `/api/card/:id`,
+ `/api/card/abc-def-…/query` → `/api/card/:uuid/query`."
+ [^String uri]
+ (-> uri
+ (str/replace #"(?<=/)\d+(?=/|$)" ":id")
+ (str/replace u/uuid-regex ":uuid")))
+
+(defn- error-message-from-response
+ "Best-effort human-readable error string from a response, for the gated `error_message` column.
+ The body may be a Clojure map (inside the handler), a JSON string, or a bare string (after
+ serialization). Handles all three; returns nil for streaming/opaque bodies."
+ [{:keys [body]}]
+ (cond
+ (map? body) (or (:message body) (:error body))
+ (string? body) (or (try
+ (let [parsed (json/decode body keyword)]
+ (or (:message parsed) (:error parsed)))
+ (catch Exception _ nil))
+ body)
+ :else nil))
+
+(defn wrap-record-cli-usage
+ "Ring middleware that records one `agent_api_call_log` row for every `/api/*` call whose
+ `User-Agent` identifies the Metabase CLI. MCP's synthetic in-process dispatch bypasses the
+ Ring middleware stack entirely, so it's never double-counted (those calls are already in
+ `mcp_tool_call_log`). Non-CLI callers pay zero overhead — a single header check."
+ [handler]
+ (fn [request respond raise]
+ (let [user-agent (get-in request [:headers "user-agent"])
+ ^String uri (:uri request)]
+ (if-not (and (= "metabase-cli" (detect-client user-agent))
+ (str/starts-with? uri "/api/"))
+ (handler request respond raise)
+ (let [timer (u/start-timer)]
+ (handler request
+ (fn [{:keys [status] :as response}]
+ (let [error? (or (nil? status) (>= status 400))]
+ (record-agent-api-call!
+ {:user-id (or (:metabase-user-id request) api/*current-user-id*)
+ :tenant-id (some-> api/*current-user* deref :tenant_id)
+ :user-agent user-agent
+ :operation (str (some-> (:request-method request) name u/upper-case-en)
+ " " (templatize-uri uri))
+ :status (if error? "error" "success")
+ :duration-ms (long (u/since-ms timer))
+ :ip-address (request/ip-address request)
+ :error-message (when error? (error-message-from-response response))}))
+ (respond response))
+ raise))))))
diff --git a/src/metabase/server/handler.clj b/src/metabase/server/handler.clj
index 1093a9f3422..813c0608688 100644
--- a/src/metabase/server/handler.clj
+++ b/src/metabase/server/handler.clj
@@ -1,6 +1,7 @@
(ns metabase.server.handler
"Top-level Metabase Ring handler."
(:require
+ [metabase.agent-api.usage :as agent-api.usage]
[metabase.analytics.core :as analytics]
[metabase.api.macros :as api.macros]
[metabase.config.core :as config]
@@ -83,6 +84,7 @@
#'mw.exceptions/catch-uncaught-exceptions ; catch any Exceptions that weren't passed to `raise`
#'mw.exceptions/catch-api-exceptions ; catch exceptions and return them in our expected format
#'mw.log/log-api-call ; log info about the request, db call counts etc.
+ #'agent-api.usage/wrap-record-cli-usage ; record CLI usage analytics for metabase-cli REST API calls
#'mw.browser-cookie/ensure-browser-id-cookie ; add cookie to identify browser; add `:browser-id` to the request
#'mw.security/add-security-headers ; Add HTTP headers to API responses to prevent them from being cached
#'mw.json/wrap-json-body ; extracts json POST/PUT body and makes it available on request
diff --git a/test/metabase/agent_api/usage_test.clj b/test/metabase/agent_api/usage_test.clj
new file mode 100644
index 00000000000..c2ee1dce996
--- /dev/null
+++ b/test/metabase/agent_api/usage_test.clj
@@ -0,0 +1,20 @@
+(ns metabase.agent-api.usage-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.agent-api.usage :as agent-api.usage]))
+
+(deftest templatize-uri-test
+ (testing "numeric path segments are replaced with :id"
+ (is (= "/api/card/:id" (agent-api.usage/templatize-uri "/api/card/134")))
+ (is (= "/api/card/:id/query" (agent-api.usage/templatize-uri "/api/card/134/query")))
+ (is (= "/api/database/:id" (agent-api.usage/templatize-uri "/api/database/2")))
+ (is (= "/api/table/:id/fields" (agent-api.usage/templatize-uri "/api/table/7/fields"))))
+ (testing "multiple numeric segments"
+ (is (= "/api/dashboard/:id/card/:id" (agent-api.usage/templatize-uri "/api/dashboard/5/card/42"))))
+ (testing "UUID path segments are replaced with :uuid"
+ (is (= "/api/card/:uuid/query"
+ (agent-api.usage/templatize-uri "/api/card/abc12345-1234-1234-1234-123456789abc/query"))))
+ (testing "non-numeric, non-UUID segments are left alone"
+ (is (= "/api/search" (agent-api.usage/templatize-uri "/api/search")))
+ (is (= "/api/collection/root/items" (agent-api.usage/templatize-uri "/api/collection/root/items")))
+ (is (= "/api/setting/site-name" (agent-api.usage/templatize-uri "/api/setting/site-name")))))