The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Record CLI usage analytics from the REST API middleware (#79152)
Sébastien2627e335f310
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex fa2615b5881..7ce8fc34192 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2081,7 +2081,8 @@metabase.server.middleware.sessionmetabase.server.settingsmetabase.server.streaming-response} ; TODO -- too many API namespaces- :uses #{analytics+ :uses #{agent-api+ analyticsanalytics-interfaceapiapi-keysdiff --git a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.cljindex 94c38e5b742..676ff96ed2c 100644--- a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj@@ -1,11 +1,13 @@(ns metabase-enterprise.agent-api.api-test- "Integration tests for the Agent API usage instrumentation in `metabase.agent-api.api` — driving- the real `routes` wrapper so the recording happens on the same synchronous thread, where- identity, status, duration, and PII are all in scope. Recording runs on every EE instance- (`:feature :none`); PII is gated by `analytics-pii-retention-enabled` (itself `:audit-app`-gated)."+ "Integration tests for CLI usage recording via the `wrap-record-cli-usage` middleware in+ `metabase.agent-api.usage`. Drives the middleware directly so recording fires on the same+ synchronous thread, where identity, status, duration, and PII are all in scope. Recording+ runs on every EE instance (`:feature :none`); PII is gated by `analytics-pii-retention-enabled`+ (itself `:audit-app`-gated)."(:require[clojure.test :refer [deftest is testing use-fixtures]]- [metabase.agent-api.api :as agent-api.api]+ [metabase.agent-api.usage :as agent-api.usage]+ [metabase.api.common :as api][metabase.test :as mt][metabase.test.fixtures :as fixtures][toucan2.core :as t2]))@@ -15,63 +17,117 @@(use-fixtures :once (fixtures/initialize :db :test-users))(defn- cleanup-by-ip!- "Isolate + clean up a test's rows by the unique fake ip_address it recorded (PII, so present only- when retention is on)."[& conditions](apply t2/delete! :model/AgentApiCallLog conditions))-(defn- random-ip- "A random, well-formed IPv4 address for test-row isolation. Well-formed so it survives- `request/ip-address`'s non-IP-character stripping unchanged, and thus round-trips into- `ip_address` for a precise lookup/cleanup key."- []+(defn- random-ip [](format "10.%d.%d.%d" (rand-int 250) (rand-int 250) (rand-int 250)))-(defn- invoke-routes- "Drive `agent-api.api/routes` synchronously and return the response. `extra` is merged onto a- minimal authenticated GET /api/agent/v1/ping request. `:uri` is the full public path (what the- recorder stores as `operation`); `:path-info` is the context-stripped path the router matches on."- [extra]- (let [p (promise)]- (agent-api.api/routes- (merge {:request-method :get- :uri "/api/agent/v1/ping"- :path-info "/v1/ping"- :metabase-user-id (mt/user->id :rasta)}- extra)- (fn [resp] (deliver p resp))- (fn [e] (deliver p e)))- (deref p 5000 :timeout)))+(defn- invoke-middleware+ "Drive `wrap-record-cli-usage` synchronously and return the response. Wraps a stub handler that+ returns the given `handler-response` (default 200 OK). The middleware's respond callback fires+ on the same thread so identity/PII/duration are in scope."+ ([request]+ (invoke-middleware request {:status 200, :body "ok"}))+ ([request handler-response]+ (let [p (promise)+ handler (agent-api.usage/wrap-record-cli-usage+ (fn [_req respond _raise]+ (respond handler-response)))]+ (binding [api/*current-user-id* (:metabase-user-id request)]+ (handler request+ (fn [resp] (deliver p resp))+ (fn [e] (deliver p e))))+ (deref p 5000 :timeout))))-(deftest routes-records-direct-http-call-test- (testing "a direct Agent API call is recorded with client classified from the User-Agent"+(deftest middleware-records-cli-request-test+ (testing "a CLI REST API call is recorded with client classified from the User-Agent"(mt/with-premium-features #{:audit-app}(mt/with-temporary-setting-values [analytics-pii-retention-enabled true](let [ip (random-ip)](try- (let [resp (invoke-routes {:headers {"user-agent" "metabase-cli/4.5.6"}- :remote-addr ip})]+ (let [resp (invoke-middleware {:request-method :get+ :uri "/api/card/42"+ :headers {"user-agent" "metabase-cli/4.5.6"}+ :remote-addr ip+ :metabase-user-id (mt/user->id :rasta)})](is (= 200 (:status resp))))(let [row (t2/select-one :model/AgentApiCallLog :ip_address ip)]- (is (some? row) "a row is recorded for the direct HTTP call")+ (is (some? row) "a row is recorded for the CLI call")(is (= "metabase-cli" (:client_name row)))- (is (= "GET /api/agent/v1/ping" (:operation row)))+ (is (= "GET /api/card/:id" (:operation row)) "numeric segments are templatized")(is (= "success" (:status row)))(is (= (mt/user->id :rasta) (:user_id row)))(is (nat-int? (:duration_ms row))))(finally (cleanup-by-ip! :ip_address ip))))))))-(deftest routes-skips-synthetic-internal-request-test- (testing "MCP's synthetic in-process dispatch (:agent-api-internal-request? true) is NOT recorded,- so it never double-counts against the row already written to mcp_tool_call_log"+(deftest middleware-skips-non-cli-request-test+ (testing "a non-CLI request does not produce a row"(mt/with-premium-features #{:audit-app}(mt/with-temporary-setting-values [analytics-pii-retention-enabled true](let [ip (random-ip)](try- (let [resp (invoke-routes {:headers {"user-agent" "metabase-cli/4.5.6"}- :remote-addr ip- :agent-api-internal-request? true})]+ (let [resp (invoke-middleware {:request-method :get+ :uri "/api/card"+ :headers {"user-agent" "Mozilla/5.0"}+ :remote-addr ip+ :metabase-user-id (mt/user->id :rasta)})](is (= 200 (:status resp))))(is (nil? (t2/select-one :model/AgentApiCallLog :ip_address ip))- "no agent_api_call_log row is written for the synthetic internal request")+ "no row is written for a non-CLI request")(finally (cleanup-by-ip! :ip_address ip))))))))++(deftest middleware-skips-non-api-request-test+ (testing "a CLI request to a non-/api/ path does not produce a row"+ (mt/with-premium-features #{:audit-app}+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled true]+ (let [ip (random-ip)]+ (try+ (let [resp (invoke-middleware {:request-method :get+ :uri "/health"+ :headers {"user-agent" "metabase-cli/4.5.6"}+ :remote-addr ip+ :metabase-user-id (mt/user->id :rasta)})]+ (is (= 200 (:status resp))))+ (is (nil? (t2/select-one :model/AgentApiCallLog :ip_address ip))+ "no row is written for a non-/api/ path")+ (finally (cleanup-by-ip! :ip_address ip))))))))++(deftest middleware-records-error-response-test+ (testing "an error response is recorded with status=error and error_message (when PII is on)"+ (mt/with-premium-features #{:audit-app}+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled true]+ (let [ip (random-ip)]+ (try+ (let [resp (invoke-middleware {:request-method :get+ :uri "/api/card/999999"+ :headers {"user-agent" "metabase-cli/4.5.6"}+ :remote-addr ip+ :metabase-user-id (mt/user->id :rasta)}+ {:status 404, :body "Not found."})]+ (is (= 404 (:status resp))))+ (let [row (t2/select-one :model/AgentApiCallLog :ip_address ip)]+ (is (some? row) "a row is recorded for the error call")+ (is (= "error" (:status row)))+ (is (= "Not found." (:error_message row)) "error_message is captured from the body")+ (is (= "GET /api/card/:id" (:operation row))))+ (finally (cleanup-by-ip! :ip_address ip))))))))++(deftest middleware-pii-gating-test+ (testing "ip_address and error_message are nil when PII retention is off"+ (mt/with-premium-features #{}+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled false]+ (let [ip (random-ip)+ count-before (t2/count :model/AgentApiCallLog)]+ (invoke-middleware {:request-method :get+ :uri "/api/card/999999"+ :headers {"user-agent" "metabase-cli/4.5.6"}+ :remote-addr ip+ :metabase-user-id (mt/user->id :rasta)}+ {:status 404, :body "Not found."})+ ;; Can't look up by ip (it's nil when PII is off), so check by count+ (let [count-after (t2/count :model/AgentApiCallLog)]+ (when (is (= (inc count-before) count-after) "a row was still recorded")+ (let [row (t2/select-one :model/AgentApiCallLog {:order-by [[:id :desc]]})]+ (is (nil? (:ip_address row)) "ip_address is nil when PII is off")+ (is (nil? (:error_message row)) "error_message is nil when PII is off")))))))))diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.cljindex ed769f6338f..c2fd976ee3c 100644--- a/src/metabase/agent_api/api.clj+++ b/src/metabase/agent_api/api.clj@@ -4,7 +4,6 @@(:require[clojure.string :as str][metabase.agent-api.settings :as agent-api.settings]- [metabase.agent-api.usage :as agent-api.usage][metabase.agent-api.validation :as agent-api.validation][metabase.ai-tracing.core :as ait][metabase.api.common :as api]@@ -1810,35 +1809,6 @@(def ^:private base-routes(api.macros/ns-handler *ns* +auth))-(defn- error-message-from-response- "Best-effort human-readable error string from an agent-api error `response`, for the gated- `error_message` column. Keyword lookups are nil-safe, so a streaming/opaque body just yields nil."- [response]- (let [body (:body response)]- (or (:message body) (:error body))))--(defn- record-agent-api-usage!- "Record one `agent_api_call_log` row for a completed direct Agent API HTTP call (EE-only).- Best-effort: the EE writer wraps the insert in try/catch, so a failure to record is logged and- swallowed — analytics never fails the request. Skips the synthetic in-process requests MCP- dispatches through here — those are already counted in `mcp_tool_call_log`, so recording them- again would double-count. `status`, `duration_ms`, IP, and User-Agent are all in scope here, and- `+auth` has bound `*current-user-id*` by the time this respond fires so direct HTTP callers get- attributed."- [request response timer]- (when-not (:agent-api-internal-request? request)- (let [status-code (:status response)- error? (or (nil? status-code) (>= status-code 400))]- (agent-api.usage/record-agent-api-call!- {:user-id (or (:metabase-user-id request) api/*current-user-id*)- :tenant-id (some-> api/*current-user* deref :tenant_id)- :user-agent (get-in request [:headers "user-agent"])- :operation (str (some-> (:request-method request) name u/upper-case-en) " " (:uri request))- :status (if error? "error" "success")- :duration-ms (long (u/since-ms timer))- :ip-address (request/ip-address request)- :error-message (when error? (error-message-from-response response))}))))-(def ^{:arglists '([request respond raise])} routes"`/api/agent/` routes.";; Wrapped in `handler-with-open-api-spec` so the handler still implements `OpenAPISpec` for@@ -1850,37 +1820,33 @@;; Agent-API endpoints are synchronous, so `respond` fires inside the span and the span;; closes after the handler returns.(fn [request respond raise]- (let [timer (u/start-timer)]- (ait/with-eval-session nil- (ait/eval-span (str "agent-api." (some-> (:request-method request) name) " " (:uri request))- {:http/method (some-> (:request-method request) name)- :http/uri (:uri request)- :http/request (:body request)- :http/user-id (or (:metabase-user-id request) api/*current-user-id*)}- (base-routes request- ;; Relies on `respond` firing synchronously on this thread (see- ;; above): if an endpoint ever responds async, `*parent*` is- ;; unbound there and this `record!` no-ops, so the span captures- ;; no status/response. Both fail soft; the trace is just incomplete- ;; for async agent-api responses.- (fn eval-traced-respond [response]- (when (ait/capture-active?)- ;; `+auth` binds `*current-user-id*` inside `base-routes`, so it- ;; is unbound when the span opened above but set by the time this- ;; respond fires — record the user id here so direct HTTP callers- ;; (not just the MCP path, which carries `:metabase-user-id`) get it.- (ait/record! {:http/status (:status response)- ;; Only record a plain data body. A streaming/opaque- ;; body (not a coll) would otherwise be stringified- ;; by the log sink into a useless `#object[…]`, so- ;; skip it — the trace just omits the response there.- :http/response (when (coll? (:body response))- (:body response))- :http/user-id (or (:metabase-user-id request)- api/*current-user-id*)}))- ;; CLI usage analytics: one lean row per direct HTTP call, on the- ;; same synchronous thread so identity/PII/duration are all in scope.- (record-agent-api-usage! request response timer)- (respond response))- raise)))))+ (ait/with-eval-session nil+ (ait/eval-span (str "agent-api." (some-> (:request-method request) name) " " (:uri request))+ {:http/method (some-> (:request-method request) name)+ :http/uri (:uri request)+ :http/request (:body request)+ :http/user-id (or (:metabase-user-id request) api/*current-user-id*)}+ (base-routes request+ ;; Relies on `respond` firing synchronously on this thread (see+ ;; above): if an endpoint ever responds async, `*parent*` is+ ;; unbound there and this `record!` no-ops, so the span captures+ ;; no status/response. Both fail soft; the trace is just incomplete+ ;; for async agent-api responses.+ (fn eval-traced-respond [response]+ (when (ait/capture-active?)+ ;; `+auth` binds `*current-user-id*` inside `base-routes`, so it+ ;; is unbound when the span opened above but set by the time this+ ;; respond fires — record the user id here so direct HTTP callers+ ;; (not just the MCP path, which carries `:metabase-user-id`) get it.+ (ait/record! {:http/status (:status response)+ ;; Only record a plain data body. A streaming/opaque+ ;; body (not a coll) would otherwise be stringified+ ;; by the log sink into a useless `#object[…]`, so+ ;; skip it — the trace just omits the response there.+ :http/response (when (coll? (:body response))+ (:body response))+ :http/user-id (or (:metabase-user-id request)+ api/*current-user-id*)}))+ (respond response))+ raise))))(fn [prefix] (open-api/open-api-spec base-routes prefix))))diff --git a/src/metabase/agent_api/usage.clj b/src/metabase/agent_api/usage.cljindex e418e6f14e0..fb1d940926d 100644--- a/src/metabase/agent_api/usage.clj+++ b/src/metabase/agent_api/usage.clj@@ -1,9 +1,12 @@(ns metabase.agent-api.usage"Agent API (CLI) usage logging.- One write point feeds the analytics table: a lean `agent_api_call_log` row per direct Agent- API HTTP call, recorded from the `routes` respond callback in `metabase.agent-api.api`. The- write is a `defenterprise` no-op in OSS — an OSS instance records nothing — with the real+ [[wrap-record-cli-usage]] is the single write point: a Ring middleware in the main handler+ stack that records one `agent_api_call_log` row for every `/api/*` HTTP call whose+ `User-Agent` identifies the Metabase CLI. MCP's synthetic in-process dispatch bypasses the+ Ring stack entirely, so it's never double-counted (those calls land in `mcp_tool_call_log`).++ The write is a `defenterprise` no-op in OSS — an OSS instance records nothing — with the realinsert in `metabase-enterprise.agent-api.usage`. Like `ai_usage_log`, collection runs on everyEE instance (`:feature :none`); the `:audit-app` feature gates the surfaces that read theserows (the audit view + page), not the writing.@@ -14,8 +17,11 @@the caller's self-reported `User-Agent` (analytics only — never used to gate access)."(:require[clojure.string :as str]+ [metabase.api.common :as api][metabase.premium-features.core :refer [defenterprise]]- [metabase.util :as u]))+ [metabase.request.core :as request]+ [metabase.util :as u]+ [metabase.util.json :as json]))(def supported-client-keys"Canonical client keys [[detect-client]] classifies callers into for analytics. Keep in sync@@ -43,3 +49,55 @@metabase-enterprise.agent-api.usage[_call-info]nil)++(defn templatize-uri+ "Replace variable path segments in `uri` with `:id` (numeric) or `:uuid` (UUID v4) to reduce+ cardinality for analytics. `/api/card/134` → `/api/card/:id`,+ `/api/card/abc-def-…/query` → `/api/card/:uuid/query`."+ [^String uri]+ (-> uri+ (str/replace #"(?<=/)\d+(?=/|$)" ":id")+ (str/replace u/uuid-regex ":uuid")))++(defn- error-message-from-response+ "Best-effort human-readable error string from a response, for the gated `error_message` column.+ The body may be a Clojure map (inside the handler), a JSON string, or a bare string (after+ serialization). Handles all three; returns nil for streaming/opaque bodies."+ [{:keys [body]}]+ (cond+ (map? body) (or (:message body) (:error body))+ (string? body) (or (try+ (let [parsed (json/decode body keyword)]+ (or (:message parsed) (:error parsed)))+ (catch Exception _ nil))+ body)+ :else nil))++(defn wrap-record-cli-usage+ "Ring middleware that records one `agent_api_call_log` row for every `/api/*` call whose+ `User-Agent` identifies the Metabase CLI. MCP's synthetic in-process dispatch bypasses the+ Ring middleware stack entirely, so it's never double-counted (those calls are already in+ `mcp_tool_call_log`). Non-CLI callers pay zero overhead — a single header check."+ [handler]+ (fn [request respond raise]+ (let [user-agent (get-in request [:headers "user-agent"])+ ^String uri (:uri request)]+ (if-not (and (= "metabase-cli" (detect-client user-agent))+ (str/starts-with? uri "/api/"))+ (handler request respond raise)+ (let [timer (u/start-timer)]+ (handler request+ (fn [{:keys [status] :as response}]+ (let [error? (or (nil? status) (>= status 400))]+ (record-agent-api-call!+ {:user-id (or (:metabase-user-id request) api/*current-user-id*)+ :tenant-id (some-> api/*current-user* deref :tenant_id)+ :user-agent user-agent+ :operation (str (some-> (:request-method request) name u/upper-case-en)+ " " (templatize-uri uri))+ :status (if error? "error" "success")+ :duration-ms (long (u/since-ms timer))+ :ip-address (request/ip-address request)+ :error-message (when error? (error-message-from-response response))}))+ (respond response))+ raise))))))diff --git a/src/metabase/server/handler.clj b/src/metabase/server/handler.cljindex 1093a9f3422..813c0608688 100644--- a/src/metabase/server/handler.clj+++ b/src/metabase/server/handler.clj@@ -1,6 +1,7 @@(ns metabase.server.handler"Top-level Metabase Ring handler."(:require+ [metabase.agent-api.usage :as agent-api.usage][metabase.analytics.core :as analytics][metabase.api.macros :as api.macros][metabase.config.core :as config]@@ -83,6 +84,7 @@#'mw.exceptions/catch-uncaught-exceptions ; catch any Exceptions that weren't passed to `raise`#'mw.exceptions/catch-api-exceptions ; catch exceptions and return them in our expected format#'mw.log/log-api-call ; log info about the request, db call counts etc.+ #'agent-api.usage/wrap-record-cli-usage ; record CLI usage analytics for metabase-cli REST API calls#'mw.browser-cookie/ensure-browser-id-cookie ; add cookie to identify browser; add `:browser-id` to the request#'mw.security/add-security-headers ; Add HTTP headers to API responses to prevent them from being cached#'mw.json/wrap-json-body ; extracts json POST/PUT body and makes it available on requestdiff --git a/test/metabase/agent_api/usage_test.clj b/test/metabase/agent_api/usage_test.cljnew file mode 100644index 00000000000..c2ee1dce996--- /dev/null+++ b/test/metabase/agent_api/usage_test.clj@@ -0,0 +1,20 @@+(ns metabase.agent-api.usage-test+ (:require+ [clojure.test :refer :all]+ [metabase.agent-api.usage :as agent-api.usage]))++(deftest templatize-uri-test+ (testing "numeric path segments are replaced with :id"+ (is (= "/api/card/:id" (agent-api.usage/templatize-uri "/api/card/134")))+ (is (= "/api/card/:id/query" (agent-api.usage/templatize-uri "/api/card/134/query")))+ (is (= "/api/database/:id" (agent-api.usage/templatize-uri "/api/database/2")))+ (is (= "/api/table/:id/fields" (agent-api.usage/templatize-uri "/api/table/7/fields"))))+ (testing "multiple numeric segments"+ (is (= "/api/dashboard/:id/card/:id" (agent-api.usage/templatize-uri "/api/dashboard/5/card/42"))))+ (testing "UUID path segments are replaced with :uuid"+ (is (= "/api/card/:uuid/query"+ (agent-api.usage/templatize-uri "/api/card/abc12345-1234-1234-1234-123456789abc/query"))))+ (testing "non-numeric, non-UUID segments are left alone"+ (is (= "/api/search" (agent-api.usage/templatize-uri "/api/search")))+ (is (= "/api/collection/root/items" (agent-api.usage/templatize-uri "/api/collection/root/items")))+ (is (= "/api/setting/site-name" (agent-api.usage/templatize-uri "/api/setting/site-name")))))