The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Bot 1729 eval tracing (#77169)
Mihael Konjević2ca895407f41
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex ba79b8591b2..3f21d9e4066 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -179,6 +179,7 @@clojure.core.async/take!clojure.core.async/to-chan!clojure.core.async/to-chan!!+ metabase.ai-tracing.core/record!metabase.api.macros.defendpoint.tools-manifest/assert-optional-fields-nullable!metabase.channel.core/send!metabase.channel.render.pdf/draw-too-large!diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex e5332e9cb86..84c5f40baf9 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -159,7 +159,8 @@{:team "Metabot":api #{metabase.agent-api.apimetabase.agent-api.init}- :uses #{api+ :uses #{ai-tracing+ apiauth-identitychannelcollections@@ -194,6 +195,15 @@modelsutil}}+ ai-tracing+ {:team "Metabot"+ :api #{metabase.ai-tracing.api+ metabase.ai-tracing.core+ metabase.ai-tracing.init}+ :uses #{api+ settings+ util}}+analytics{:team "UX West":api #{metabase.analytics.api@@ -1254,6 +1264,7 @@:model-exports #{:model/McpSessionLog:model/McpToolCallLog}:uses #{agent-api+ ai-tracingapiapp-dbconfig@@ -1299,6 +1310,7 @@slackbot}:uses #{activity-feedagent-lib+ ai-tracinganalyticsanalytics-interfaceapidiff --git a/resources/log4j2.xml b/resources/log4j2.xmlindex 70e017fdfa9..2cf343f76f2 100644--- a/resources/log4j2.xml+++ b/resources/log4j2.xml@@ -19,6 +19,26 @@</PatternLayout></RollingFile>-->++ <!-- Eval-trace sink (metabase.ai-tracing): one JSONL file per eval session, keyed on the+ mb-eval-session-id ThreadContext value. Inert unless that key is set (i.e. only during a+ capture). NOTE: the `$$` in the Routes pattern is REQUIRED — a single `$` resolves once at+ config-parse time and collapses every session into one file. `%m%n` keeps each line pure+ JSON. No redaction: traces intentionally contain full prompts; keep this off in prod. -->+ <Routing name="EVAL_TRACE_ROUTING">+ <Routes pattern="$${ctx:mb-eval-session-id}">+ <Route>+ <File name="eval-trace-${ctx:mb-eval-session-id}"+ fileName="${sys:logfile.path:-target/log}/eval-traces/${ctx:mb-eval-session-id}.jsonl"+ append="true">+ <PatternLayout pattern="%m%n"/>+ </File>+ </Route>+ <!-- Required default route: fires when the ctx key is absent. -->+ <Route key="$${ctx:mb-eval-session-id}" ref="STDOUT"/>+ </Routes>+ <IdlePurgePolicy timeToLive="15" timeUnit="MINUTES"/>+ </Routing></Appenders><Loggers>@@ -27,6 +47,14 @@<Logger name="metabase" level="INFO"/><Logger name="metabase-enterprise" level="INFO"/><Logger name="metabase.metabot" level="DEBUG"/>+ <!-- Eval-trace sink. The single on/off switch is MB_AI_EVAL_CAPTURE (off by default): this+ logger only ever receives events while a capture is active, so on a normal instance it+ writes nothing even though it sits at INFO. additivity=false keeps these lines out of the+ main app log. Set level="off" here as a hard kill if you ever need to mute it regardless+ of the capture flag. -->+ <Logger name="metabase.ai-tracing.log" level="info" additivity="false">+ <AppenderRef ref="EVAL_TRACE_ROUTING"/>+ </Logger><Logger name="metabase.plugins" level="DEBUG"/><Logger name="metabase.query-processor.async" level="DEBUG"/><Logger name="metabase.server.middleware" level="DEBUG"/>diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.cljindex bf8c9978fb5..d291c3a3f87 100644--- a/src/metabase/agent_api/api.clj+++ b/src/metabase/agent_api/api.clj@@ -5,9 +5,11 @@[clojure.string :as str][metabase.agent-api.settings :as agent-api.settings][metabase.agent-api.validation :as agent-api.validation]+ [metabase.ai-tracing.core :as ait][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.macros.scope :as scope]+ [metabase.api.open-api :as open-api][metabase.api.routes.common :as api.routes.common][metabase.auth-identity.core :as auth-identity][metabase.channel.urls :as channel.urls]@@ -291,6 +293,11 @@_query-params{:keys [prompt] :as body} :- ::construct-query-request](let [query (evaluate-external-query-for-execution body)]+ ;; Record the resolved query for eval tracing (inert unless capturing). The MCP `construct_query`+ ;; tool only returns an opaque handle, so this span attribute is how an eval harness recovers the+ ;; agent's actual query off the trace to grade it. `query` is already serialization-ready (no+ ;; `:lib/metadata`), so it stays small.+ (ait/record! {:ai/query query})(cond-> {:query (-> query json/encode u/encode-base64)}prompt (assoc :prompt prompt))))@@ -330,11 +337,11 @@;; target database so a bogus/inaccessible database_id fails here rather than at save time.(api/read-check :model/Database database_id);; Emit MBQL 5 (via `lib/native-query` + `prepare-for-serialization`, same as `construct_query`)- (let [mp (lib-be/application-database-metadata-provider database_id)]- {:query (-> (lib/native-query mp sql)- lib/prepare-for-serialization- json/encode- u/encode-base64)}))+ (let [mp (lib-be/application-database-metadata-provider database_id)+ query (-> (lib/native-query mp sql) lib/prepare-for-serialization)]+ ;; See /v2/construct-query: record the resolved query so an eval harness can grade it off the trace.+ (ait/record! {:ai/query query})+ {:query (-> query json/encode u/encode-base64)}));;; ------------------------------------------------- Combined Query -------------------------------------------------@@ -1799,6 +1806,47 @@;;; ---------------------------------------------------- Routes ------------------------------------------------------+(def ^:private base-routes+ (api.macros/ns-handler *ns* +auth))+(def ^{:arglists '([request respond raise])} routes"`/api/agent/` routes."- (api.macros/ns-handler *ns* +auth))+ ;; Wrapped in `handler-with-open-api-spec` so the handler still implements `OpenAPISpec` for+ ;; full-API spec generation (openapi.json, endpoint-dox); the spec is delegated to `base-routes`,+ ;; the underlying `ns-handler`, since the eval-tracing wrapper below carries no route metadata.+ (open-api/handler-with-open-api-spec+ ;; Eval tracing (inert unless MB_AI_EVAL_CAPTURE). Direct callers get a fresh session;+ ;; the synthetic in-process call from MCP inherits the MCP session and nests under it.+ ;; Agent-API endpoints are synchronous, so `respond` fires inside the span and the span+ ;; closes after the handler returns.+ (fn [request respond raise]+ (ait/with-eval-session nil+ (ait/eval-span (str "agent-api." (some-> (:request-method request) name) " " (:uri request))+ {:http/method (some-> (:request-method request) name)+ :http/uri (:uri request)+ :http/request (:body request)+ :http/user-id (or (:metabase-user-id request) api/*current-user-id*)}+ (base-routes request+ ;; Relies on `respond` firing synchronously on this thread (see+ ;; above): if an endpoint ever responds async, `*parent*` is+ ;; unbound there and this `record!` no-ops, so the span captures+ ;; no status/response. Both fail soft; the trace is just incomplete+ ;; for async agent-api responses.+ (fn eval-traced-respond [response]+ (when (ait/capture-active?)+ ;; `+auth` binds `*current-user-id*` inside `base-routes`, so it+ ;; is unbound when the span opened above but set by the time this+ ;; respond fires — record the user id here so direct HTTP callers+ ;; (not just the MCP path, which carries `:metabase-user-id`) get it.+ (ait/record! {:http/status (:status response)+ ;; Only record a plain data body. A streaming/opaque+ ;; body (not a coll) would otherwise be stringified+ ;; by the log sink into a useless `#object[…]`, so+ ;; skip it — the trace just omits the response there.+ :http/response (when (coll? (:body response))+ (:body response))+ :http/user-id (or (:metabase-user-id request)+ api/*current-user-id*)}))+ (respond response))+ raise))))+ (fn [prefix] (open-api/open-api-spec base-routes prefix))))diff --git a/src/metabase/ai_tracing/api.clj b/src/metabase/ai_tracing/api.cljnew file mode 100644index 00000000000..8d83aec704e--- /dev/null+++ b/src/metabase/ai_tracing/api.clj@@ -0,0 +1,69 @@+(ns metabase.ai-tracing.api+ "HTTP endpoint for reading captured eval traces by session id.++ Gated two ways: the route is only mounted/served when `MB_AI_EVAL_CAPTURE` is enabled+ (`+eval-capture-enabled`, mirroring `+agent-api-enabled` / `+mcp-enabled`), and each request+ additionally requires a superuser — traces contain full, unredacted prompts and content, so this+ is intended for dedicated eval instances only."+ (:require+ [clojure.java.io :as io]+ [metabase.ai-tracing.core :as ait]+ [metabase.ai-tracing.settings :as ai-tracing.settings]+ [metabase.api.common :as api]+ [metabase.api.macros :as api.macros]+ [metabase.api.routes.common :as routes.common]+ [metabase.util.i18n :refer [tru]])+ (:import+ (java.io File)))++(set! *warn-on-reflection* true)++(defn- trace-dir+ "Directory the eval-trace RoutingAppender writes to — mirrors the log4j2 path+ `${sys:logfile.path:-target/log}/eval-traces`."+ ^File []+ (io/file (or (System/getProperty "logfile.path") "target/log") "eval-traces"))++(defn- trace-file+ "Resolve the JSONL file for `session-id`, or nil if the id is unsafe or would escape the trace dir.+ `ait/safe-session-id-re` is the same pattern enforced at write time when the id is minted."+ ^File [session-id]+ (when (re-matches ait/safe-session-id-re (str session-id))+ (let [dir (.getCanonicalFile (trace-dir))+ f (.getCanonicalFile (io/file dir (str session-id ".jsonl")))]+ ;; defense in depth: the resolved file must sit directly inside the trace dir+ (when (= dir (.getParentFile f))+ f))))++(api.macros/defendpoint :get "/:session-id" :- :any+ "Return the captured eval trace as JSONL (one span per line) for `session-id`. Superuser-only;+ served only when `MB_AI_EVAL_CAPTURE` is enabled."+ ;; Bound the length here too (not only via `trace-file`): `session-id` is a raw user-supplied path+ ;; segment, and `ait/max-session-id-length` is the single cap enforced at both write and read.+ [{:keys [session-id]} :- [:map [:session-id [:string {:min 1, :max ait/max-session-id-length}]]]]+ (api/check-superuser)+ (let [f (trace-file session-id)]+ (api/check (and f (.exists ^File f)) [404 (tru "Eval trace not found")])+ ;; Return the File directly so Ring streams it — a per-session trace can be many MB of full+ ;; prompts/completions, which we don't want to slurp into the heap.+ {:status 200+ :headers {"Content-Type" "application/x-ndjson"}+ :body ^File f}))++(defn- enforce-eval-capture-enabled+ "Ring middleware: 404 when eval capture is disabled. Uses a generic not-found message (rather than+ naming the feature) so the endpoint is indistinguishable from an unmounted route to a caller who+ isn't allowed to know it exists."+ [handler]+ (fn [request respond raise]+ (if (ai-tracing.settings/ai-eval-capture)+ (handler request respond raise)+ (raise (ex-info (tru "Not found.") {:status-code 404})))))++(def ^{:arglists '([handler])} +eval-capture-enabled+ "Wrap routes so they may only be accessed when eval capture is enabled."+ (routes.common/wrap-middleware-for-open-api-spec-generation enforce-eval-capture-enabled))++(def ^{:arglists '([request respond raise])} routes+ "`/api/eval-trace/` routes."+ (api.macros/ns-handler *ns*))diff --git a/src/metabase/ai_tracing/core.clj b/src/metabase/ai_tracing/core.cljnew file mode 100644index 00000000000..03aaff0bd92--- /dev/null+++ b/src/metabase/ai_tracing/core.clj@@ -0,0 +1,293 @@+(ns metabase.ai-tracing.core+ "Eval-time tracing for the AI / Metabot agentic system.++ This layer is **separate from production observability** (`metabase.tracing`). It exists+ to capture rich, complete agent traces *during benchmark/eval runs* so a scorer can read+ the data back — full prompts/completions, tool I/O, and ad-hoc debug spans.++ ## The separate gate (production safety)++ All ops here are gated on [[*capture*]], a per-run dynamic binding established ONLY by the+ eval entrypoints ([[with-eval-session]] / [[capturing]]). It is orthogonal to+ `metabase.tracing`'s `MB_TRACING_ENABLED` + group gates:++ - In production — even with `metabase.tracing` fully enabled — ai-tracing does no work:+ each span's body runs behind a nil-check on [[*capture*]] (the span macros still build+ their `span-name`/`attrs` args, which just reference already-computed values). Eval spans,+ which carry full prompts and verbose internals, can NEVER fire on organic traffic.+ `MB_AI_EVAL_CAPTURE` is off by default.+ - In an eval run, an entrypoint binds [[*capture*]] and the spans fire.++ ## What it captures++ An in-memory span tree (`agent.turn → llm.call → tool.*`, or `mcp.<method> → …`) with full+ attributes, durations, and events. Nesting + the session id are tracked through dynamic+ bindings ([[*parent*]], [[*session-id*]]) that Clojure conveys across the agent's+ virtual-thread tool execution (`bound-fn*`), so concurrent tool spans nest correctly.++ ## Session id (the OTel trace-id analog)++ [[*session-id*]] groups all of a run's spans and names its log file. It is minted at the root+ of a fresh capture, or *supplied* by the caller (e.g. MCP passes its `Mcp-Session-Id`), so a+ multi-request MCP conversation all lands in one file.++ ## Sink++ Each span, on finish, is streamed as one JSON line to the dedicated `metabase.ai-tracing.log`+ logger (off by default), routed per-session to its own JSONL file. See+ `metabase.ai-tracing.log`."+ (:require+ [metabase.ai-tracing.log :as ai-tracing.log]+ [metabase.ai-tracing.settings :as ai-tracing.settings]))++(set! *warn-on-reflection* true)++;;;; ----------------------------------------- Gate (separate axis) -----------------------------------------++(def ^:dynamic *capture*+ "Per-run capture sink: an atom holding the vector of root span maps, or nil.+ Bound ONLY by an entrypoint. nil ⇒ ai-tracing is inert (the production safety gate)."+ nil)++(def ^:dynamic *parent*+ "The current in-flight span node (an atom), or nil at top level. Conveyed across the+ agent's virtual-thread tool execution via `bound-fn*`, so concurrent tool spans nest+ under the correct parent."+ nil)++(def ^:dynamic *session-id*+ "The eval session id (OTel trace-id analog). Minted at the root of a fresh capture, or supplied+ by the caller (e.g. MCP's `Mcp-Session-Id`). Conveyed across virtual threads via `bound-fn*`;+ drives the per-session log file routing."+ nil)++(def ^:dynamic *retain-tree*+ "When true, each finished span is also retained in the in-memory tree ([[*capture*]] / its parent's+ `:children`) so an in-process caller can read the whole trace back as a return value (see+ [[capturing]]). Left false on the file-routed path ([[with-eval-session]]): there every span is+ already streamed to its JSONL file on finish and the in-memory tree is never read, so retaining it+ would only pin heap proportional to the whole run. Off by default."+ false)++(def safe-session-id-re+ "A session id becomes BOTH a log-file name (the RoutingAppender's `${ctx:mb-eval-session-id}.jsonl`)+ and a URL path segment on the read endpoint, so it must be filesystem/URL safe: start alphanumeric+ (rejects `.`/`..`), then only alphanumerics / `.` `_` `-` (no `/` ⇒ no path traversal). This is the+ single source of truth — enforced at the mint/supply boundary ([[checked-session-id]], via+ [[with-eval-session]]) AND at the read boundary (`metabase.ai-tracing.api`).++ Anchored with `\\A`/`\\z` so it's an exact, newline-safe match under BOTH `re-matches` (the mint/read+ checks) and `re-find` (Malli's `:re` schema on the `eval[_-]session[_-]id` API params). Without the+ anchors, `re-find` would accept any string merely CONTAINING a safe run (e.g. `../../etc/passwd`)."+ #"\A[A-Za-z0-9][A-Za-z0-9._-]*\z")++(def ^:const max-session-id-length+ "Cap on a supplied session id. It becomes `<id>.jsonl` on disk, so this keeps the filename well+ under the ~255-byte filesystem limit. Also mirrored as a `:max` on the API schemas (defense in+ depth for non-API callers: MCP, direct invocation)."+ 200)++(defn checked-session-id+ "Return a safe session id: mint a fresh uuid when `supplied` is nil, otherwise validate it against+ [[safe-session-id-re]] and [[max-session-id-length]]. Throws on a supplied id that isn't safe — the+ id names a file written by the RoutingAppender, so an unsafe value (e.g. `../../etc/x`) or an+ over-long one (which would fail filesystem creation and silently drop the trace) must never reach+ it. We throw rather than silently substituting a uuid so a caller that named a trace file gets an+ error instead of a trace it can't find."+ ^String [supplied]+ (if (nil? supplied)+ (str (random-uuid))+ (let [id (str supplied)]+ (when-not (re-matches safe-session-id-re id)+ (throw (ex-info "Invalid eval-session-id: must start alphanumeric and contain only [A-Za-z0-9._-]"+ {:session-id id})))+ (when (> (count id) max-session-id-length)+ (throw (ex-info (str "Invalid eval-session-id: must be at most " max-session-id-length " characters")+ {:session-id-length (count id)})))+ id)))++(defn capture-active?+ "True when an eval capture is in progress on this thread."+ []+ (some? *capture*))++;;;; ----------------------------------------- In-memory span tree ------------------------------------------++(defn- now-ns ^long [] (System/nanoTime))++(defn- now-epoch-ms ^long [] (System/currentTimeMillis))++(defn- new-node [span-type span-name attrs parent-id]+ ;; :start-ns (monotonic) drives duration; :start-epoch-ms (wall clock) drives the log line.+ (atom {:type span-type+ :name span-name+ :id (str (random-uuid))+ :parent-id parent-id+ :attributes (or attrs {})+ :events []+ :children []+ :start-ns (now-ns)+ :start-epoch-ms (now-epoch-ms)}))++(defn- finish-node [node]+ ;; one deref ⇒ a consistent snapshot (duration, start, and :children all from the same state).+ (let [{:keys [start-ns start-epoch-ms] :as snapshot} @node+ dur-ns (- (now-ns) start-ns)]+ (-> snapshot+ (assoc :duration-ms (/ (double dur-ns) 1e6)+ ;; wall-clock end derived from the monotonic duration (no second, possibly+ ;; backward-jumping clock read); ms-granular — sub-ms precision is in :duration-ms.+ :end-epoch-ms (+ (long start-epoch-ms) (Math/round (/ (double dur-ns) 1e6))))+ (dissoc :start-ns))))++(defn- attach! [parent finished]+ ;; Only build the in-memory tree when a caller will read it back ([[capturing]]). On the file-routed+ ;; path emit! has already streamed this span, so retaining it here would pin heap for the whole run.+ ;; swap! is atomic, so concurrent tool spans appending to the same parent are safe.+ (when *retain-tree*+ (if parent+ (swap! parent update :children conj finished)+ (swap! *capture* conj finished))))++(defn eval-span*+ "Functional core behind the span macros. Runs `thunk` inside a captured span of `span-type`+ named `span-name` with `attrs`, recording nesting + duration, attaching the finished node to+ the tree, and streaming it to the log sink. No-op (just the nil-check) unless a capture is+ active. Errors are recorded as an event and re-thrown."+ [span-type span-name attrs thunk]+ (if-not (capture-active?)+ (thunk)+ (let [parent *parent*+ node (new-node span-type span-name attrs (when parent (:id @parent)))]+ (try+ (binding [*parent* node]+ (thunk))+ (catch Throwable t+ ;; Capture class + ex-data, not just the message: agent-validation `ex-info`s carry their+ ;; signal in ex-data and a bare NPE has a nil message. The log sink JSON-safe-coerces+ ;; :data, so a non-encodable value here never breaks emission.+ (swap! node update :events conj {:event :error+ :message (ex-message t)+ :class (.getName (class t))+ :data (ex-data t)})+ (throw t))+ (finally+ (let [finished (finish-node node)]+ (attach! parent finished)+ ;; Stream this span as one JSONL line. Runs on whatever thread the span finished on+ ;; (incl. tool virtual threads) — *session-id* is conveyed there by `bound-fn*`.+ (ai-tracing.log/emit! finished *session-id*)))))))++;;;; ----------------------------------------- Enrichment of current span -----------------------------------++(defn record!+ "Merge `attrs` (namespaced keyword → value) onto the current eval span. Use for+ outputs/results/health signals known mid- or post-body. No-op when inactive or outside a span."+ [attrs]+ (when (and (capture-active?) *parent* (seq attrs))+ (swap! *parent* update :attributes merge attrs))+ nil)++;;;; ----------------------------------------- Span macros --------------------------------------------------++(defmacro spanning+ "Implementation detail of the span macros below — not intended for direct use (it must be public+ only so the public span macros can expand to it from other namespaces).++ Runs `body` directly when no capture is live (the production no-op), else inside a captured span.+ Guarding `capture-active?` HERE — ahead of the `eval-span*` call — means `span-name`/`attrs` are+ only constructed when a capture is active, so organic traffic never builds the span name string or+ the attrs map (it pays only the `*capture*` nil-check). `body` is emitted once, behind a `thunk#`,+ so a large body isn't duplicated into both branches; the disabled path's only residual cost is that+ one small closure."+ {:style/indent 3}+ [span-type span-name attrs & body]+ `(let [thunk# (^:once fn* [] ~@body)]+ (if (capture-active?)+ (eval-span* ~span-type ~span-name ~attrs thunk#)+ (thunk#))))++(defmacro eval-span+ "Generic eval-only span. Usable from any subsystem. `attrs` is a map of namespaced keyword → value."+ {:style/indent 2}+ [span-name attrs & body]+ `(spanning :span ~span-name ~attrs ~@body))++(defmacro with-agent-turn+ "Root span for one agentic turn."+ {:style/indent 1}+ [attrs & body]+ `(spanning :turn "agent.turn" ~attrs ~@body))++(defmacro with-llm-call+ "Span around one LLM round-trip. Record the completion/usage via [[record!]] once known."+ {:style/indent 1}+ [attrs & body]+ `(spanning :llm "llm.call" ~attrs ~@body))++(defmacro with-tool-call+ "Span around one tool invocation. Expects `:ai/tool-name` in `attrs`."+ {:style/indent 1}+ [attrs & body]+ ;; Can't route through `spanning`: the span name derives from `attrs`, and we want `attrs` built+ ;; only when a capture is active AND only once (not twice — for the name and for the span). So we+ ;; inline the gate and bind `attrs#` inside the active branch. Tool calls run on the organic agent+ ;; path, so keeping this build behind `capture-active?` is the point.+ `(let [thunk# (^:once fn* [] ~@body)]+ (if (capture-active?)+ (let [attrs# ~attrs]+ (eval-span* :tool (str "tool." (:ai/tool-name attrs#)) attrs# thunk#))+ (thunk#))))++;;;; ----------------------------------------- Entrypoints --------------------------------------------------++(defmacro with-eval-session+ "Generic eval-session entrypoint. When capture is enabled and not already active, establish a+ FRESH capture: bind [[*capture*]] (the in-memory tree), [[*session-id*]] (a [[checked-session-id]]+ of `supplied-id`, or a fresh uuid), and reset [[*parent*]] to nil. When already [[capture-active?]],+ INHERIT the outer bindings (nesting). Inert (body only) when eval-capture is disabled. Throws if+ `supplied-id` is present but not [[safe-session-id-re]] (it names a file — no path traversal).++ Session-id is minted eagerly here so callers can read [[*session-id*]] before the first span.++ (with-eval-session nil (with-agent-turn {…} …)) ; agent loop — mint+ (with-eval-session mcp-session (eval-span \"mcp.tools/call\" {…} …)) ; MCP — supplied"+ {:style/indent 1}+ [supplied-id & body]+ `(cond+ (capture-active?) (do ~@body) ; inherit / nest+ (ai-tracing.settings/ai-eval-capture) (binding [*capture* (atom [])+ *session-id* (checked-session-id ~supplied-id)+ *parent* nil]+ ~@body)+ :else (do ~@body))) ; inert++(defmacro capturing+ "In-process capture, UNCONDITIONAL (ignores the gate). Establishes a fresh capture that RETAINS the+ span tree ([[*retain-tree*]]) and returns `{:result <body value> :trace <root spans>}`. Used by+ [[capture-reducible]] and tests. The harness must fully realize any lazy result inside `body`.++ This path is RETURN-VALUE-based, not file-routed: the trace comes back as `:trace`. It deliberately+ does NOT mint a session id — [[*session-id*]] is left as-is (nil at top level), so+ [[metabase.ai-tracing.log/emit!]] no-ops and NO `<id>.jsonl` is written; read `:trace` instead. A+ session id supplied to an inner [[with-eval-session]] (e.g. `:eval-session-id` threaded through+ `run-agent-loop`) is IGNORED — capture is already active, so the inner form inherits these bindings+ rather than re-minting. (A capture nested inside a live file-routed session inherits that session+ and still streams to its file.)"+ [& body]+ `(let [sink# (atom [])]+ (binding [*capture* sink#+ *parent* nil+ *retain-tree* true]+ ;; bind result# in a let so the body (which populates sink#) is forced BEFORE @sink#+ (let [result# (do ~@body)]+ {:result result# :trace @sink#}))))++(defn capture-reducible+ "In-process eval entrypoint. Realizes `reducible` (e.g. the value of+ `metabase.metabot.agent.core/run-agent-loop`) with capture active, collecting every reduced item+ into a vector. Returns `{:result [items…] :trace [root-spans]}`.++ (ai-tracing/capture-reducible (run-agent-loop opts))"+ [reducible]+ (capturing (into [] reducible)))diff --git a/src/metabase/ai_tracing/init.clj b/src/metabase/ai_tracing/init.cljnew file mode 100644index 00000000000..115b2e66094--- /dev/null+++ b/src/metabase/ai_tracing/init.clj@@ -0,0 +1,3 @@+(ns metabase.ai-tracing.init+ (:require+ [metabase.ai-tracing.settings]))diff --git a/src/metabase/ai_tracing/log.clj b/src/metabase/ai_tracing/log.cljnew file mode 100644index 00000000000..e8230f74a28--- /dev/null+++ b/src/metabase/ai_tracing/log.clj@@ -0,0 +1,106 @@+(ns metabase.ai-tracing.log+ "File-based JSONL sink for eval traces.++ One JSON object per span is emitted on THIS dedicated logger (`metabase.ai-tracing.log` — the+ logger name equals the namespace), routed per-session to its own `<session-id>.jsonl` file by a+ Log4j2 `RoutingAppender` keyed on the `mb-eval-session-id` ThreadContext value.++ Each line is the message only (the appender uses `%m%n`), so the file is pure JSONL: an external+ script joins lines by `:session` and rebuilds the tree by chaining `:id`/`:parent`.++ `:start-epoch-ms`/`:end-epoch-ms` are wall-clock epoch milliseconds (`currentTimeMillis`), NOT+ `System/nanoTime`; `:dur-ms` carries the precise (sub-ms, monotonic) duration. The end is the start+ plus that duration, so it never reads a second, possibly backward-jumping wall clock.++ The single on/off switch is `MB_AI_EVAL_CAPTURE` (off by default): [[emit!]] is only reached while+ a capture is active, so on a normal instance this logger writes nothing even though it sits at+ INFO. (`resources/log4j2.xml` also lets you set the logger to `off` as a hard kill.) Traces contain+ full, unredacted user content — intended for dedicated eval instances only.++ Retention is the operator's responsibility. Files are append-only with no rollover, no size cap,+ and no expiry — a long session grows unbounded, and re-running the same session id appends to the+ same file. The `RoutingAppender`'s IdlePurgePolicy only reclaims idle *appenders* (open file+ handles), never the files on disk. An eval host should periodically prune+ `${logfile.path}/eval-traces/` (the harness has usually consumed each `<id>.jsonl` by then)."+ (:require+ [metabase.util.json :as json]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(defn- node->entry+ "Pure: a finished span node + `session-id` → the JSONL map. No semantic-convention mapping and no+ size limits — full payloads. The reconstruction script joins on `:session` and chains+ `:id`/`:parent`. `:start-epoch-ms`/`:end-epoch-ms` are wall-clock epoch milliseconds (see ns+ docstring); sub-ms precision lives in `:dur-ms`."+ [{:keys [type name id parent-id attributes events duration-ms start-epoch-ms end-epoch-ms]}+ session-id]+ {:session session-id+ :id id+ :parent parent-id+ :type type+ :name name+ :start-epoch-ms start-epoch-ms+ :end-epoch-ms end-epoch-ms+ :dur-ms duration-ms+ :attributes attributes+ :events events})++(def ^:private json-safe-max-depth+ "Recursion cap for [[json-safe]]. Attribute values are arbitrary request context / agent state, so a+ pathologically deep — or cyclic — value would otherwise overflow the stack. A `StackOverflowError` is+ an `Error`, not an `Exception`, so it would escape [[emit!]]'s `catch Exception` net and break the+ traced run. Past this depth we stringify instead of recursing."+ 200)++(defn- json-safe+ "Best-effort coercion of a value into something the JSON encoder can handle: recurse through plain+ collections, keep scalars, and stringify anything else. Used ONLY as a fallback after a direct+ encode throws, so it never touches values (UUIDs, dates, …) that already encode fine. Bounded by+ [[json-safe-max-depth]] so a deep/cyclic value degrades to a string rather than overflowing the stack."+ ([v] (json-safe v 0))+ ([v depth]+ (if (>= depth json-safe-max-depth)+ (str v)+ (let [d (inc depth)]+ (cond+ ;; Coerce KEYS too, not just values: the JSON encoder rejects a non-scalar map key (a+ ;; vector/map key), so leaving it intact would let the fallback re-encode throw — orphaning+ ;; the very root span this fallback exists to save.+ (map? v) (into {} (map (fn [[k val]]+ [(if (or (string? k) (keyword? k) (number? k)) k (str k))+ (json-safe val d)]))+ v)+ (set? v) (mapv #(json-safe % d) v)+ (sequential? v) (mapv #(json-safe % d) v)+ (or (nil? v) (string? v) (number? v) (boolean? v) (keyword? v) (symbol? v)) v+ :else (str v))))))++(defn emit!+ "Stream one finished span `node` as a single JSONL line, routed to `session-id`'s file. No-op when+ `session-id` is nil.++ `log/with-context` sets `mb-eval-session-id` in the Log4j2 ThreadContext for the duration of the+ single `log/info` call, on whatever thread the span finished on. Because `*session-id*` is conveyed+ across the agent's virtual-thread tool executor by `bound-fn*`, the routing key is always correct,+ and `with-context` restores/removes it afterward so it never leaks onto a pooled thread."+ [node session-id]+ (when session-id+ (try+ (let [entry (node->entry node session-id)+ line (try+ (json/encode entry)+ (catch Exception _+ ;; The attributes carry arbitrary request context / agent state, which could+ ;; contain a non-encodable value. Dropping the line here would lose the span —+ ;; and for a root span, orphan every child that references its `:id`. Degrade+ ;; to a JSON-safe copy so the span still emits.+ (json/encode (json-safe entry))))]+ (log/with-context {:eval-session-id session-id}+ (log/info line)))+ ;; Ultimate safety net: never let trace serialization/emission break the traced run. Catch+ ;; `Exception`, not `Throwable`, so JVM `Error`s and `InterruptedException` still propagate —+ ;; this runs in the span's `finally`, often on a tool virtual thread.+ (catch Exception t+ (log/warn t "ai-tracing: failed to serialize/emit eval span"))))+ nil)diff --git a/src/metabase/ai_tracing/settings.clj b/src/metabase/ai_tracing/settings.cljnew file mode 100644index 00000000000..1aecd9b4818--- /dev/null+++ b/src/metabase/ai_tracing/settings.clj@@ -0,0 +1,21 @@+(ns metabase.ai-tracing.settings+ "Env-var-only settings for the eval-time AI tracing module.+ Configured exclusively via environment variables (`:setter :none`)."+ (:require+ [metabase.settings.core :refer [defsetting]]+ [metabase.util.i18n :refer [deferred-tru]]))++(set! *warn-on-reflection* true)++(defsetting ai-eval-capture+ (deferred-tru+ (str "Enable eval-time AI agent trace capture. When true, agent / MCP / agent-api runs capture a "+ "full trace (prompts, completions, tool I/O, custom spans) and write it as structured JSONL to "+ "a per-session file via the dedicated `metabase.ai-tracing.log` logger. Separate from "+ "MB_TRACING_ENABLED. Off by default; intended for dedicated eval instances, NOT production "+ "(traces contain full, unredacted user content)."))+ :type :boolean+ :default false+ :visibility :internal+ :export? false+ :setter :none)diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex 82e2bfcd6c5..14790a4896b 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -4,6 +4,7 @@[metabase.actions-rest.api][metabase.activity-feed.api][metabase.agent-api.api]+ [metabase.ai-tracing.api][metabase.analytics.api][metabase.analytics.api.proxy][metabase.api-keys.api]@@ -80,6 +81,7 @@(comment metabase.actions-rest.api/keep-memetabase.activity-feed.api/keep-memetabase.agent-api.api/keep-me+ metabase.ai-tracing.api/keep-memetabase.analytics.api/keep-memetabase.analytics.api.proxy/keep-memetabase.api-keys.api/keep-me@@ -193,6 +195,7 @@"/embed" (+message-only-exceptions metabase.embedding-rest.api/embedding-routes)"/embed-mcp" (+auth metabase.mcp.callback-api/routes)"/embed-theme" (+auth metabase.embedding-rest.api/theme-routes)+ "/eval-trace" (metabase.ai-tracing.api/+eval-capture-enabled metabase.ai-tracing.api/routes)"/field" (+auth metabase.warehouse-schema-rest.api/field-routes)"/frontend-errors" metabase.frontend-errors.api/routes"/geojson" 'metabase.geojson.apidiff --git a/src/metabase/core/init.clj b/src/metabase/core/init.cljindex ada7f75686e..92504518c8e 100644--- a/src/metabase/core/init.clj+++ b/src/metabase/core/init.clj@@ -9,6 +9,7 @@[metabase.actions.init][metabase.activity-feed.init][metabase.agent-api.init]+ [metabase.ai-tracing.init][metabase.analytics.init][metabase.api-scope.init][metabase.api.init]diff --git a/src/metabase/mcp/api.clj b/src/metabase/mcp/api.cljindex 834447e3860..d32b865c3fe 100644--- a/src/metabase/mcp/api.clj+++ b/src/metabase/mcp/api.clj@@ -5,6 +5,7 @@[clojure.core.async :as a][clojure.string :as str][compojure.response :as compojure.response]+ [metabase.ai-tracing.core :as ait][metabase.api.common :as api][metabase.api.macros.scope :as scope][metabase.api.open-api :as open-api]@@ -109,9 +110,24 @@(defn- handle-ping [id _params](jsonrpc-response id {}))-(defn- dispatch-request- "Dispatch a single JSON-RPC request. Returns a response map or nil for notifications."- [{:keys [id method params] :as _msg} session-id token-scopes request-context]+(defn- eval-session-override+ "An eval-session id the harness supplies via the `x-eval-session-id` header so it can name (and+ later fetch) the trace itself — the MCP analogue of metabot's `eval_session_id`. opencode negotiates+ the `Mcp-Session-Id` internally, so without this the harness can't know which `<uuid>.jsonl` to read.++ Validates through `ait/checked-session-id` — the mint-time boundary, and the single source of truth+ for the safe-id contract — and maps its throw on an unsafe/over-long id to nil, so a bad header falls+ back to the Mcp-Session-Id correlator rather than 500ing ahead of `dispatch-request`'s try/catch. The+ `when-let` guards the absent-header case, so we never reach `checked-session-id`'s nil -> fresh-uuid+ branch (which would invent a trace file the harness never named)."+ [request]+ (when-let [id (get-in request [:headers "x-eval-session-id"])]+ (try (ait/checked-session-id id) (catch Exception _ nil))))++(defn- dispatch-method+ "Route a single JSON-RPC `method` to its handler, returning a response map or nil (notifications).+ A handler that throws is turned into a JSON-RPC internal error rather than propagating."+ [id method params session-id token-scopes request-context](try(case method"notifications/initialized" nil@@ -127,6 +143,33 @@(log/error e "Error dispatching JSON-RPC method" method)(jsonrpc-error id -32603 (or (ex-message e) "Internal error")))))+(defn- dispatch-request+ "Dispatch a single JSON-RPC request. Returns a response map or nil for notifications."+ [{:keys [id method params] :as _msg} session-id token-scopes request-context eval-session-id]+ ;; Eval tracing (inert unless MB_AI_EVAL_CAPTURE): establish a session and open a per-request root+ ;; span; tool/resource/agent-api spans nest under it automatically. Key on the harness-supplied+ ;; `eval-session-id` when given (so it owns the trace file name), else the MCP session's UUID+ ;; correlator so an entire conversation's requests append to one `<uuid>.jsonl`. We key on the UUID+ ;; prefix (not the full `<uuid>.<base64>` id): it's stable across the conversation AND always+ ;; filesystem/URL-safe, whereas the full id can carry a base64 payload that `require-valid-session`+ ;; accepts but `safe-session-id-re` rejects (e.g. `=` padding) — passing that to `with-eval-session`+ ;; would throw out here, ahead of the try/catch.+ ;;+ ;; When BOTH are absent (a stateless / pre-initialize request with no header), this is nil and+ ;; `with-eval-session` mints a fresh uuid — so such requests get their own `<uuid>.jsonl` rather than+ ;; grouping. That's fine for the eval flow, which always supplies `eval-session-id`; the ungrouped+ ;; files are reaped by the appender's IdlePurgePolicy.+ (ait/with-eval-session (or eval-session-id (some-> session-id (str/split #"\.") first))+ (ait/eval-span (str "mcp." method) {:mcp/method method+ :mcp/request-id id+ :mcp/params params+ :mcp/user-id api/*current-user-id*+ :mcp/scopes token-scopes}+ (let [response (dispatch-method id method params session-id token-scopes request-context)]+ ;; record the materialized JSON-RPC result/error (the request's output)+ (ait/record! {:mcp/response response})+ response))))+;;; ----------------------------------------------------- SSE ------------------------------------------------------(defn- accepts-sse?@@ -222,9 +265,10 @@(defn- handle-post"Handle a POST request containing one or more JSON-RPC messages."[user-id request]- (let [body (:body request)- session-id (get-in request [:headers "mcp-session-id"])- batch? (sequential? body)]+ (let [body (:body request)+ session-id (get-in request [:headers "mcp-session-id"])+ eval-session-id (eval-session-override request)+ batch? (sequential? body)](cond(nil? body)(json-response 400 (jsonrpc-error nil -32700 "Parse error: empty body"))@@ -270,7 +314,10 @@;; (gated PII) alongside client identity — the view no longer joins the session.request-context {:user-agent (get-in request [:headers "user-agent"]):ip-address (request/ip-address request)}- responses (into [] (keep #(dispatch-request % session-id (:token-scopes request) request-context)) messages)]+ dispatch-msg (fn [msg]+ (dispatch-request msg session-id (:token-scopes request)+ request-context eval-session-id))+ responses (into [] (keep dispatch-msg) messages)](cond(empty? responses){:status 202 :headers {} :body ""}diff --git a/src/metabase/mcp/tools.clj b/src/metabase/mcp/tools.cljindex bda75b1fdd2..ee1711046ff 100644--- a/src/metabase/mcp/tools.clj+++ b/src/metabase/mcp/tools.clj@@ -6,6 +6,7 @@[clojure.set :as set][clojure.string :as str][metabase.agent-api.api :as agent-api]+ [metabase.ai-tracing.core :as ait][metabase.api.common :as api][metabase.api.macros.defendpoint.tools-manifest :as tools-manifest][metabase.config.core :as config]@@ -539,36 +540,41 @@([token-scopes session-id tool-name arguments](call-tool token-scopes session-id tool-name arguments {:supports-mcp-ui? true}))([token-scopes session-id tool-name arguments options]- (let [start (System/nanoTime)- record! (fn [status error-code error-message]- (mcp.usage/record-mcp-tool-call!- {:tool-name tool-name- :user-id api/*current-user-id*- :session-id session-id- :status status- :duration-ms (quot (- (System/nanoTime) start) 1000000)- :error-code error-code- :error-message error-message- ;; Identity + PII are denormalized onto the row (the view no longer joins the- ;; session): client from the call's `_meta`, tenant from the current user,- ;; IP/UA from the request. The recorder falls back to the session row's stored- ;; client when `_meta` carries none, and gates the PII columns.- :client-info (:client-info options)- :tenant-id (some-> api/*current-user* deref :tenant_id)- :user-agent (get-in options [:request-context :user-agent])- :ip-address (get-in options [:request-context :ip-address])}))]- (try- (let [result (dispatch-tool-call token-scopes session-id tool-name arguments options)- error? (boolean (:isError result))]- (record! (if error? "error" "success")- (when error? (or (::error-code result) error-code-internal))- (when error? (some-> result :content first :text)))- ;; `::error-code` is an internal classification marker — never expose it to the client.- (dissoc result ::error-code))- (catch Throwable e- ;; A handler that throws instead of returning error content (notably a UI tool- ;; `:response-fn`, whose path isn't wrapped in `dispatch-tool-call`) would otherwise- ;; skip instrumentation and under-report errors. Record the failure, then rethrow so the- ;; transport layer still surfaces it to the client.- (record! "error" error-code-internal (ex-message e))- (throw e))))))+ ;; Eval span (inert unless capturing) nesting under the MCP request span; records the tool+ ;; name + args, and the resulting content as output.+ (ait/with-tool-call {:ai/tool-name tool-name :ai/tool-args arguments}+ (let [start (System/nanoTime)+ record! (fn [status error-code error-message]+ (mcp.usage/record-mcp-tool-call!+ {:tool-name tool-name+ :user-id api/*current-user-id*+ :session-id session-id+ :status status+ :duration-ms (quot (- (System/nanoTime) start) 1000000)+ :error-code error-code+ :error-message error-message+ ;; Identity + PII are denormalized onto the row (the view no longer joins the+ ;; session): client from the call's `_meta`, tenant from the current user,+ ;; IP/UA from the request. The recorder falls back to the session row's stored+ ;; client when `_meta` carries none, and gates the PII columns.+ :client-info (:client-info options)+ :tenant-id (some-> api/*current-user* deref :tenant_id)+ :user-agent (get-in options [:request-context :user-agent])+ :ip-address (get-in options [:request-context :ip-address])}))]+ (try+ (let [result (dispatch-tool-call token-scopes session-id tool-name arguments options)+ error? (boolean (:isError result))]+ (record! (if error? "error" "success")+ (when error? (or (::error-code result) error-code-internal))+ (when error? (some-> result :content first :text)))+ ;; `::error-code` is an internal classification marker — never expose it to the client.+ (let [result (dissoc result ::error-code)]+ (ait/record! {:ai/tool-output result})+ result))+ (catch Throwable e+ ;; A handler that throws instead of returning error content (notably a UI tool+ ;; `:response-fn`, whose path isn't wrapped in `dispatch-tool-call`) would otherwise+ ;; skip instrumentation and under-report errors. Record the failure, then rethrow so the+ ;; transport layer still surfaces it to the client.+ (record! "error" error-code-internal (ex-message e))+ (throw e)))))))diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex 2e763aa503e..41973b304a6 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -3,6 +3,7 @@(:require[clojure.java.io :as io][clojure.string :as str]+ [metabase.ai-tracing.core :as ait][metabase.analytics-interface.core :as analytics][metabase.api-scope.core :as api-scope][metabase.api.common :as api]@@ -260,6 +261,10 @@:system (:content system-msg):parts input-parts:tools (vec tools)}))+ (when (ait/capture-active?)+ (ait/record! {:ai/system (:content system-msg)+ :ai/input-parts input-parts+ :ai/tool-count (count tools)}))(eduction (streaming/post-process-xf (get-in memory [:state :queries] {})(get-in memory [:state :charts] {})link-registry-atom)@@ -505,14 +510,24 @@memory @memory-atomparts-atom (atom [])link-registry-atom (atom (get-in memory [:state :link-registry] {}))- llm-call (call-llm memory context profile tools iteration tracking-opts link-registry-atom)xf (comp (accumulate-usage-xf usage-atom (:model profile))(u/tee-xf parts-atom));; We use `reduce` instead of `transduce` because rf is the outer reducing;; function (e.g. parts->aisdk-sse-xf wrapping streaming-writer-rf) whose completion;; arity emits a finish message — that must only fire once, at the end of the;; entire agent loop, not after every iteration.- result' (reduce (xf rf) result llm-call)+ ;; `call-llm` runs inside the eval span so the request attrs it records attach here.+ result' (ait/with-llm-call {:ai/iteration iteration+ :ai/model (:model profile)}+ (let [llm-call (call-llm memory context profile tools iteration+ tracking-opts link-registry-atom)+ reduced-result (reduce (xf rf) result llm-call)]+ (when (ait/capture-active?)+ (ait/record! {:ai/output-text (collect-text-from-parts @parts-atom)+ :ai/tool-io (summarize-tool-ios @parts-atom)+ ;; preserve data parts (e.g. generated_entity) emitted this iteration+ :ai/data-parts (filterv #(= :data (:type %)) @parts-atom)}))+ reduced-result))parts @parts-atom];; Sync link registry back to memory after streaming completes(swap! memory-atom assoc-in [:state :link-registry] @link-registry-atom)@@ -525,25 +540,28 @@:all-parts parts}))(log/debug "Iteration" {:n iteration :parts-count (count parts)})(if (empty? parts)- (assoc loop-state :status :done :result (rf result (final-state-part @memory-atom)))+ ;; LLM returned nothing this iteration — a natural stop, surfaced for the turn span.+ (assoc loop-state :status :done :finish-reason :empty-response+ :result (rf result (final-state-part @memory-atom)))(do(log/debug "Got parts" {:count (count parts) :types (mapv :type parts)})(swap! memory-atom update-memory parts)(cond(reduced? result')- (assoc loop-state :status :reduced :result @result')+ ;; consumer signalled early termination (e.g. client disconnect / cancellation)+ (assoc loop-state :status :reduced :finish-reason :reduced :result @result')(should-continue? iteration max-iter terminal-tools parts)(assoc loop-state :result result' :iteration (inc iteration)):else- (do (log/info "Agent loop complete"- {:iterations iteration- ;; TODO: decide if we want this reason to float up to frontend- :reason (finish-reason iteration max-iter terminal-tools parts)})- (assoc loop-state- :status :done- :result (rf result' (final-state-part @memory-atom))))))))))+ (let [reason (finish-reason iteration max-iter terminal-tools parts)]+ (log/info "Agent loop complete" {:iterations iteration :reason reason})+ (assoc loop-state+ :status :done+ ;; surfaced so run-agent-loop can record it on the turn span+ :finish-reason reason+ :result (rf result' (final-state-part @memory-atom))))))))));;; Public API@@ -573,6 +591,16 @@:version 1:data debug-log})+(defn- eval-session-part+ "Create a data part carrying just the eval session id. Emitted at the end of the stream when+ `MB_AI_EVAL_CAPTURE` is enabled, so the benchmark/eval harness can locate the per-session+ `<session-id>.jsonl` trace file written by `metabase.ai-tracing.log`."+ [session-id]+ {:type :data+ :data-type "eval_session"+ :version 1+ :data {:session-id session-id}})+(def ^:private profile-id->required-permission"Map from profile-id to the metabot permission that must be `:yes` for a userto use that profile. Profiles not listed here have no profile-level permission gate."@@ -612,6 +640,13 @@[:state {:optional true} [:maybe ::state]][:context {:optional true} [:maybe ::context]][:tracking-opts {:optional true} [:maybe ::tracking-opts]]+ ;; eval-only: when supplied (e.g. by the benchmark harness), names the per-session+ ;; trace file so the caller can read it back without parsing the stream. Length + charset+ ;; are enforced here (rather than only deep in `ait/checked-session-id`) so a bad id 400s+ ;; at the boundary instead of surfacing as a generic agent error. `ait/max-session-id-length`+ ;; and `ait/safe-session-id-re` are the single source of truth for the cap and the charset.+ [:eval-session-id {:optional true}+ [:maybe [:and [:string {:max ait/max-session-id-length}] [:re ait/safe-session-id-re]]]][:debug? {:optional true} [:maybe :boolean]]]](let [profile-id (:profile-id opts)debug? (:debug? opts)@@ -637,17 +672,51 @@scope/*current-user-capabilities* (get-in opts [:context :capabilities] #{})scope/*current-loadable-skill-ids* (atom #{})](try- (let [agent (init-agent opts)- {result :result- iteration :iteration} (->> (initial-loop-state agent rf init (atom {}))- (iterate loop-step)- (drop-while #(= :continue (:status %)))- first)]- (analytics/observe! :metabase-metabot/agent-iterations labels iteration)- ;; Emit debug log as a data part if debug mode was active- (if (and debug? (seq @*debug-log*))- (rf result (debug-log-part @*debug-log*))- result))+ ;; `with-eval-session` establishes the eval capture (gated by MB_AI_EVAL_CAPTURE,+ ;; inherited when an in-process `capture-reducible` already bound one). Spans stream+ ;; to the per-session JSONL file as they finish.+ (ait/with-eval-session (:eval-session-id opts)+ (let [turn-result+ (ait/with-agent-turn {:ai/profile-id (name profile-id)+ :ai/msg-count (count (:messages opts))+ ;; the user's latest question — trace-level input for evals+ :ai/user-input (some->> (:messages opts)+ (filter #(= "user" (some-> % :role name)))+ last :content)+ ;; full request — makes the trace a self-contained record+ :ai/messages (:messages opts)+ :ai/context (:context opts)+ :ai/state (:state opts)}+ (let [agent (init-agent opts)+ usage-atom (atom {})+ {result :result+ iteration :iteration+ finish-reason :finish-reason} (->> (initial-loop-state agent rf init usage-atom)+ (iterate loop-step)+ (drop-while #(= :continue (:status %)))+ first)]+ (analytics/observe! :metabase-metabot/agent-iterations labels iteration)+ ;; Materialized turn-level outputs for evals: token usage, why the loop+ ;; stopped, and the final agent state (link registry / generated entities).+ (when (ait/capture-active?)+ (ait/record! {:ai/finish-reason finish-reason+ :ai/usage @usage-atom+ :ai/final-state (some-> (:memory-atom agent) deref :state)}))+ ;; A :reduced stop (client disconnect / cancellation) has already terminated+ ;; the reducing fn — stepping it again would violate the transducer contract.+ ;; Append trailing parts only on a normal finish: the debug log, and the+ ;; eval-session pointer that lets the harness locate the per-session+ ;; `<session-id>.jsonl` (it reads that file, not the stream, so a skipped+ ;; pointer on disconnect costs nothing). Gate the pointer on a non-nil+ ;; `*session-id*` too: the in-process `capture-reducible`/`capturing` path is+ ;; capture-active but deliberately file-less (session id nil, read `:trace`),+ ;; so emitting a pointer there would name a `<nil>.jsonl` that never exists.+ (if (= :reduced finish-reason)+ result+ (cond-> result+ (and debug? (seq @*debug-log*)) (rf (debug-log-part @*debug-log*))+ (and (ait/capture-active?) ait/*session-id*) (rf (eval-session-part ait/*session-id*))))))]+ turn-result))(catch Exception e(analytics/inc! :metabase-metabot/agent-errors labels)(let [{:keys [api-error status provider body]} (ex-data e)diff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.cljindex 5dd844a0ec5..f9e13832398 100644--- a/src/metabase/metabot/api.clj+++ b/src/metabase/metabot/api.clj@@ -4,6 +4,7 @@[clojure.core.async :as a][clojure.string :as str][medley.core :as m]+ [metabase.ai-tracing.core :as ait][metabase.analytics.core :as analytics.core][metabase.api.common :as api][metabase.api.macros :as api.macros]@@ -102,7 +103,7 @@`start` event's `messageId` so the client can correlate streamed messageswith feedback."[{:keys [metabot-id profile-id message context history conversation-id state debug?- assistant-msg-id external-id]}]+ eval-session-id assistant-msg-id external-id]}](let [enriched-context (metabot.context/create-context context {:metabot-id metabot-id:profile-id (keyword profile-id)})messages (concat history [message])]@@ -121,12 +122,13 @@(transduce xf(streaming-writer-rf os canceled-chan canceled?)(agent/run-agent-loop- (cond-> {:messages messages- :state state- :metabot-id metabot-id- :profile-id (keyword profile-id)- :context enriched-context- :tracking-opts {:session-id conversation-id}}+ (cond-> {:messages messages+ :state state+ :metabot-id metabot-id+ :profile-id (keyword profile-id)+ :context enriched-context+ :eval-session-id eval-session-id+ :tracking-opts {:session-id conversation-id}}debug? (assoc :debug? true))))(catch org.eclipse.jetty.io.EofException _(vreset! canceled? true)@@ -186,7 +188,7 @@it into:- `hostname`: extracted from the origin URL, always recorded.- `pii-info`: gated by `analytics-pii-retention-enabled` — nil when off."- [{:keys [metabot_id profile_id message context history conversation_id state debug]} request-info]+ [{:keys [metabot_id profile_id message context history conversation_id state debug eval_session_id]} request-info](let [message (metabot.envelope/user-message message)metabot-id (metabot.config/resolve-dynamic-metabot-id metabot_id)_ (metabot.config/check-metabot-enabled! metabot-id)@@ -211,6 +213,7 @@:conversation-id conversation_id:state state:debug? debug?+ :eval-session-id eval_session_id:assistant-msg-id assistant-msg-id:external-id assistant-external-id}))))@@ -255,6 +258,12 @@[:queries {:optional true} [:map-of :string :any]][:charts {:optional true} [:map-of :string :any]][:chart-configs {:optional true} [:map-of :string :any]]]]+ ;; eval-only: lets the benchmark harness name the per-session trace file it will read back.+ ;; Length + charset enforced at this HTTP boundary so a bad id 400s cleanly instead of+ ;; throwing deep in `ait/checked-session-id` and surfacing as a generic agent error.+ ;; `ait/max-session-id-length` / `ait/safe-session-id-re` are the single source of truth.+ [:eval_session_id {:optional true}+ [:maybe [:and [:string {:max ait/max-session-id-length}] [:re ait/safe-session-id-re]]]][:debug {:optional true} [:maybe :boolean]]]req](metabot.context/log body :llm.log/fe->be)diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 8a263fc00e1..49a823ebad2 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -4,6 +4,7 @@[clojure.java.io :as io][clojure.string :as str][clojure.walk :as walk]+ [metabase.ai-tracing.core :as ait][metabase.llm.settings :as llm][metabase.metabot.schema.v2 :as schema.v2][metabase.premium-features.core :as premium-features]@@ -521,35 +522,41 @@Chunks have a ::duration-ms key added for internal use which is not part of the aisdk spec."[tool-call-id tool-name tool chunks]- (with-span :info {:name :metabot.agent/run-tool- :tool-name tool-name- :tool-call-id tool-call-id}- (let [start-ms (u/start-timer)- assoc-ms (fn [duration-ms]- (fn [chunk]- (cond-> chunk- (= (:type chunk) :tool-output-available) (assoc ::duration-ms duration-ms))))- results (try- (let [{:keys [arguments]} (into {} (aisdk-xf) chunks)- arguments (or (coerce-stringified-json arguments) {})- decode (tool-decode-fn tool)- arguments (cond-> arguments decode decode)]- (log/debug "Executing tool" {:tool-name tool-name :arguments arguments})- (let [tool-fn (tool-call-fn tool)- result (tool-fn arguments)]- (log/debug "Tool returned" {:tool-name tool-name :result-type (type result)})- (collect-tool-result tool-call-id tool-name result)))- (catch Exception e- (if (:agent-error? (ex-data e))- (log/debugf "Tool %s: agent validation error: %s" tool-name (ex-message e))- (log/warn e "Tool execution failed" {:tool-name tool-name}))- [{:type :tool-output-available- :toolCallId tool-call-id- :toolName tool-name- :error {:message (concise-tool-error e)- :type (str (type e))}}]))]- (mapv (assoc-ms (u/since-ms start-ms))- results))))+ (ait/with-tool-call {:ai/tool-name tool-name+ :ai/tool-call-id tool-call-id}+ (with-span :info {:name :metabot.agent/run-tool+ :tool-name tool-name+ :tool-call-id tool-call-id}+ (let [start-ms (u/start-timer)+ assoc-ms (fn [duration-ms]+ (fn [chunk]+ (cond-> chunk+ (= (:type chunk) :tool-output-available) (assoc ::duration-ms duration-ms))))+ results (try+ (let [{:keys [arguments]} (into {} (aisdk-xf) chunks)+ arguments (or (coerce-stringified-json arguments) {})+ decode (tool-decode-fn tool)+ arguments (cond-> arguments decode decode)]+ (log/debug "Executing tool" {:tool-name tool-name :arguments arguments})+ (when (ait/capture-active?)+ (ait/record! {:ai/tool-args arguments}))+ (let [tool-fn (tool-call-fn tool)+ result (tool-fn arguments)]+ (log/debug "Tool returned" {:tool-name tool-name :result-type (type result)})+ (collect-tool-result tool-call-id tool-name result)))+ (catch Exception e+ (if (:agent-error? (ex-data e))+ (log/debugf "Tool %s: agent validation error: %s" tool-name (ex-message e))+ (log/warn e "Tool execution failed" {:tool-name tool-name}))+ [{:type :tool-output-available+ :toolCallId tool-call-id+ :toolName tool-name+ :error {:message (concise-tool-error e)+ :type (str (type e))}}]))]+ (when (ait/capture-active?)+ (ait/record! {:ai/tool-output results}))+ (mapv (assoc-ms (u/since-ms start-ms))+ results)))))(defn tool-executor-xf"Transducer that executes tool calls in parallel on virtual threads.diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.cljindex c450dd3346e..ced7b82744e 100644--- a/test/metabase/agent_api/api_test.clj+++ b/test/metabase/agent_api/api_test.clj@@ -8,6 +8,8 @@[java-time.api :as t][metabase.agent-api.api :as agent-api.api][metabase.agent-api.settings :as agent-api.settings]+ [metabase.ai-tracing.log :as ait.log]+ [metabase.ai-tracing.settings :as ai-tracing.settings][metabase.collections.models.collection :as collection][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -93,6 +95,25 @@(is (= "AI features are not enabled."(mt/user-http-request :rasta :get 403 "agent/v1/ping"))))))+(deftest eval-tracing-wraps-agent-api-requests-test+ (testing "with capture on, the routes wrapper opens a span recording status/response/user-id"+ (let [nodes (atom [])]+ ;; Force capture on and collect the emitted spans in-memory (redef the sink so no file is+ ;; written). The wrapper mints a fresh session per direct HTTP request.+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)+ ait.log/emit! (fn [node _session-id]+ (swap! nodes conj node)+ nil)]+ (is (= {:message "pong"} (mt/user-http-request :rasta :get 200 "agent/v1/ping")))+ (let [span (first (filter #(str/starts-with? (str (:name %)) "agent-api.") @nodes))]+ (is (some? span) "an agent-api.* span was emitted for the request")+ (is (= "agent-api.get /api/agent/v1/ping" (:name span)))+ (is (= 200 (get-in span [:attributes :http/status])))+ ;; the plain-data body is recorded (a streaming body would be omitted, not stringified)+ (is (= {:message "pong"} (get-in span [:attributes :http/response])))+ ;; +auth binds *current-user-id* inside the handler, so the respond-time record! sees rasta+ (is (= (mt/user->id :rasta) (get-in span [:attributes :http/user-id]))))))))+;;; ------------------------------------------------- Functional Tests --------------------------------------------------(deftest search-testdiff --git a/test/metabase/ai_tracing/api_test.clj b/test/metabase/ai_tracing/api_test.cljnew file mode 100644index 00000000000..06c4173d3e7--- /dev/null+++ b/test/metabase/ai_tracing/api_test.clj@@ -0,0 +1,57 @@+(ns metabase.ai-tracing.api-test+ (:require+ [clojure.java.io :as io]+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing use-fixtures]]+ [metabase.ai-tracing.api :as ait.api]+ [metabase.ai-tracing.settings :as ai-tracing.settings]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures])+ (:import+ (java.io File)))++(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db :test-users))++(deftest ^:parallel trace-file-rejects-unsafe-ids-test+ (testing "trace-file returns nil for any id that could escape the trace dir (path traversal)"+ (doseq [bad ["../../etc/passwd" "a/b" ".." "." ".hidden" "has space" "" "foo/../bar" "/abs"]]+ (is (nil? (#'ait.api/trace-file bad))+ (str "should reject " (pr-str bad))))))++(deftest ^:parallel trace-file-resolves-safe-id-test+ (testing "a safe id resolves to a <id>.jsonl file sitting directly inside the trace dir"+ (let [^File f (#'ait.api/trace-file "abc-123.def_4")]+ (is (some? f))+ (is (= "abc-123.def_4.jsonl" (.getName f)))+ (is (= (.getCanonicalFile ^File (#'ait.api/trace-dir))+ (.getParentFile f))+ "defense in depth: the resolved file's parent is the canonical trace dir"))))++;;; -------------------------------------------- Endpoint (integration) ---------------------------------------------+;; Not ^:parallel: redefs the capture setting and writes to the real trace dir.++(deftest endpoint-gated-on-capture-then-superuser-test+ (testing "the route is invisible (404) when MB_AI_EVAL_CAPTURE is off — the default"+ (mt/user-http-request :crowberto :get 404 "eval-trace/some-id"))+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)]+ (testing "with capture on, a non-superuser is forbidden (traces hold unredacted content)"+ (mt/user-http-request :rasta :get 403 "eval-trace/some-id"))+ (testing "with capture on, a superuser gets 404 for a session with no trace file"+ (is (re-find #"Eval trace not found"+ (str (mt/user-http-request :crowberto :get 404 "eval-trace/no-such-session")))))))++(deftest endpoint-streams-the-trace-file-test+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)]+ (let [sid (str "eval-api-test-" (random-uuid))+ ^File f (#'ait.api/trace-file sid)]+ (io/make-parents f)+ (spit f "{\"session\":\"s\",\"id\":\"n1\"}\n{\"session\":\"s\",\"id\":\"n2\"}\n")+ (try+ (testing "a superuser gets a 200 resolving to that session's <id>.jsonl (Ring streams the file)"+ ;; The test client surfaces the File body as its path rather than streaming its bytes, so we+ ;; assert the endpoint resolved to the right file at 200 — the byte-streaming is Ring's job.+ (let [body (str (mt/user-http-request :crowberto :get 200 (str "eval-trace/" sid)))]+ (is (str/includes? body (str sid ".jsonl")))))+ (finally (.delete f))))))diff --git a/test/metabase/ai_tracing/core_test.clj b/test/metabase/ai_tracing/core_test.cljnew file mode 100644index 00000000000..8c6794fd6ba--- /dev/null+++ b/test/metabase/ai_tracing/core_test.clj@@ -0,0 +1,142 @@+(ns metabase.ai-tracing.core-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.ai-tracing.core :as ait]+ [metabase.ai-tracing.log :as ait.log]+ [metabase.ai-tracing.settings :as ai-tracing.settings]+ [metabase.test :as mt]))++(set! *warn-on-reflection* true)++(deftest ^:parallel gate-inert-without-binding-test+ (testing "with no capture binding, ops are inert (production safety) but the body still runs"+ (is (false? (ait/capture-active?)))+ (let [ran (atom false)]+ (is (= 42 (ait/with-agent-turn {:ai/profile-id "p"}+ (reset! ran true)+ (ait/record! {:ai/x 1}) ; no-op, must not throw+ 42)))+ (is (true? @ran) "body ran"))+ (is (false? (ait/capture-active?)) "binding did not leak")))++(deftest ^:parallel capture-nesting-test+ (testing "capturing builds a nested turn -> llm -> tool tree"+ (let [{:keys [result trace]}+ (ait/capturing+ (ait/with-agent-turn {:ai/profile-id "internal"}+ (ait/with-llm-call {:ai/model "m" :ai/iteration 0}+ (ait/record! {:ai/output-text "hi"})+ (ait/with-tool-call {:ai/tool-name "search"}+ (ait/record! {:ai/tool-args {:q "x"} :ai/tool-output [{:r 1}]})+ :tool-done))+ :turn-done))]+ (is (= :turn-done result))+ (is (= 1 (count trace)))+ (let [turn (first trace)+ llm (first (:children turn))+ tool (first (:children llm))]+ (is (= :turn (:type turn)))+ (is (= "agent.turn" (:name turn)))+ (is (= :llm (:type llm)))+ (is (= "hi" (get-in llm [:attributes :ai/output-text])))+ (is (= :tool (:type tool)))+ (is (= "tool.search" (:name tool)))+ (is (= {:q "x"} (get-in tool [:attributes :ai/tool-args])))))))++(deftest ^:parallel record-test+ (testing "record! merges attrs onto the current span"+ (let [{:keys [trace]}+ (ait/capturing+ (ait/with-llm-call {:ai/model "m"}+ (ait/record! {:ai/output-text "a"})+ (ait/record! {:ai/iteration 2})+ :ok))+ llm (first trace)]+ (is (= "a" (get-in llm [:attributes :ai/output-text])))+ (is (= 2 (get-in llm [:attributes :ai/iteration]))))))++(deftest ^:parallel timing-fields-test+ (testing "finished spans carry a precise duration + wall-clock epoch millis"+ (let [{:keys [trace]} (ait/capturing (ait/with-llm-call {} (Thread/sleep 2) :ok))+ llm (first trace)]+ (is (number? (:duration-ms llm)))+ (is (<= 0 (:duration-ms llm)))+ (is (integer? (:start-epoch-ms llm)))+ (is (integer? (:end-epoch-ms llm)))+ (is (<= (long (:start-epoch-ms llm)) (long (:end-epoch-ms llm))))+ (is (nil? (:start-ns llm)) "internal monotonic marker stripped from finished node"))))++(deftest ^:parallel error-recorded-and-rethrown-test+ (testing "an exception in a span body is recorded as an event, attaches the node, and re-throws"+ (let [sink (atom [])]+ (is (thrown? Exception+ (binding [ait/*capture* sink ait/*parent* nil ait/*retain-tree* true]+ (ait/with-llm-call {:ai/model "m"}+ (throw (ex-info "boom" {}))))))+ (let [llm (first @sink)]+ (is (= :llm (:type llm)))+ (is (= [:error] (mapv :event (:events llm))))))))++(deftest ^:parallel concurrent-tools-nest-correctly-test+ (testing "tools on separate threads (bound-fn conveyance) nest under the right parent"+ (let [{:keys [trace]}+ (ait/capturing+ (ait/with-llm-call {:ai/model "m"}+ (let [fs (mapv (fn [i]+ (let [f (bound-fn []+ (ait/with-tool-call {:ai/tool-name (str "t" i)}+ (Thread/sleep 3) i))]+ (future (f))))+ (range 4))]+ (mapv deref fs))))+ llm (first trace)]+ (is (= :llm (:type llm)))+ (is (= 4 (count (:children llm))))+ (is (= #{"tool.t0" "tool.t1" "tool.t2" "tool.t3"}+ (set (map :name (:children llm))))))))++;; Not ^:parallel: references the destructive `emit!` (redef'd to a no-op here), which the+;; `validate-deftest` lint disallows in parallel tests regardless of the thread-local redef.+(deftest capture-reducible-test+ (testing "capture-reducible realizes a reducible and returns {:result :trace}"+ ;; redef the sink so the test never writes a trace file+ (mt/with-dynamic-fn-redefs [ait.log/emit! (constantly nil)]+ (let [reducible (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ (ait/with-agent-turn {:ai/profile-id "p"}+ (-> init (rf {:type :text :text "a"}) (rf {:type :text :text "b"})))))+ {:keys [result trace]} (ait/capture-reducible reducible)]+ (is (= [{:type :text :text "a"} {:type :text :text "b"}] result))+ (is (= 1 (count trace)))+ (is (= "agent.turn" (:name (first trace))))))))++;; Not ^:parallel: redefs the destructive `emit!` to a spy.+(deftest capturing-writes-no-file-test+ (testing "capturing does NOT mint a session id, so emit! gets nil and writes no <id>.jsonl"+ (let [sessions (atom [])]+ (mt/with-dynamic-fn-redefs [ait.log/emit! (fn [_node session-id] (swap! sessions conj session-id) nil)]+ (let [{:keys [trace]} (ait/capturing (ait/with-llm-call {:ai/model "m"} :ok))]+ (is (= 1 (count trace)) "the span still comes back in :trace")+ (is (= [nil] @sessions)+ "emit! received a nil session-id (no file routing); the trace is the return value only"))))))++(deftest ^:parallel checked-session-id-test+ (testing "supplied ids that name a safe file pass through verbatim"+ (is (= "abc-123.def_4" (ait/checked-session-id "abc-123.def_4"))))+ (testing "a nil supplied id mints a fresh uuid"+ (let [id (ait/checked-session-id nil)]+ (is (string? id))+ (is (uuid? (parse-uuid id)))))+ (testing "unsafe ids throw rather than reaching the log-file path (no traversal)"+ (doseq [bad ["../../etc/passwd" ".." "a/b" ".hidden" "has space" ""]]+ (is (thrown? clojure.lang.ExceptionInfo (ait/checked-session-id bad))+ (str "should reject " (pr-str bad))))))++(deftest ^:parallel with-eval-session-rejects-unsafe-id-test+ (testing "with-eval-session validates a caller-supplied id when capture is enabled"+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)]+ (is (thrown? clojure.lang.ExceptionInfo+ (ait/with-eval-session "../../pwned" :body)))))+ (testing "with the gate off it stays inert — the body runs, no validation"+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly false)]+ (is (= :body (ait/with-eval-session "../../pwned" :body))))))diff --git a/test/metabase/ai_tracing/log_test.clj b/test/metabase/ai_tracing/log_test.cljnew file mode 100644index 00000000000..1dc25c4a96b--- /dev/null+++ b/test/metabase/ai_tracing/log_test.clj@@ -0,0 +1,160 @@+(ns metabase.ai-tracing.log-test+ (:require+ [clojure.test :refer [deftest is testing]]+ ;; needed to install a recording logger factory for the MDC-routing probe; no util.log equivalent+ ^{:clj-kondo/ignore [:discouraged-namespace]}+ [clojure.tools.logging]+ ^{:clj-kondo/ignore [:discouraged-namespace]}+ [clojure.tools.logging.impl]+ [metabase.ai-tracing.core :as ait]+ [metabase.ai-tracing.log :as ait.log]+ [metabase.ai-tracing.settings :as ai-tracing.settings]+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [metabase.util.log.capture :as log.capture])+ (:import+ (org.apache.logging.log4j ThreadContext)))++(set! *warn-on-reflection* true)++(defn- entries+ "Parse captured `metabase.ai-tracing.log` messages (raw JSONL) into maps with string keys."+ [messages-fn]+ (mapv (comp json/decode :message) (messages-fn)))++(deftest ^:parallel node->entry-shape-test+ (testing "node->entry serializes a finished node + session into the JSONL map (string keys after encode)"+ (let [node {:type :llm :name "llm.call" :id "n1" :parent-id "p0"+ :attributes {:ai/model "claude" :ai/output-text "hi"}+ :events [{:event :note}]+ :duration-ms 12.5 :start-epoch-ms 1000 :end-epoch-ms 2000}+ e (#'ait.log/node->entry node "sess-1")]+ (is (= {:session "sess-1" :id "n1" :parent "p0" :type :llm :name "llm.call"+ :start-epoch-ms 1000 :end-epoch-ms 2000 :dur-ms 12.5+ :attributes {:ai/model "claude" :ai/output-text "hi"}+ :events [{:event :note}]}+ e))+ (testing "round-trips through JSON cleanly"+ (let [j (json/decode (json/encode e))]+ (is (= "sess-1" (get j "session")))+ (is (= "llm" (get j "type")))+ (is (= "claude" (get-in j ["attributes" "ai/model"]))))))))++(deftest emit-degrades-non-encodable-attrs-test+ (testing "a non-encodable attribute value degrades to a string instead of dropping the whole span"+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ ;; a bare Object has no JSON encoding — the direct encode throws and we fall back to json-safe+ (ait.log/emit! {:type :turn :name "agent.turn" :id "n" :parent-id nil+ :attributes {:ai/final-state (Object.) :ai/model "m"} :events []+ :duration-ms 1.0 :start-epoch-ms 1 :end-epoch-ms 2}+ "sess-x")+ (let [es (entries messages)]+ (is (= 1 (count es)) "span still emitted despite the non-encodable attribute")+ (is (= "agent.turn" (get (first es) "name")))+ (is (= "m" (get-in (first es) ["attributes" "ai/model"])) "encodable attrs are preserved")+ (is (string? (get-in (first es) ["attributes" "ai/final-state"]))+ "the non-encodable value degraded to a string")))))++(deftest emit-degrades-non-encodable-map-key-test+ (testing "a map attribute with a non-scalar KEY still emits — json-safe must not itself throw"+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ ;; A vector-keyed map is rejected by the JSON encoder AND by json-safe's value-only recursion+ ;; before the fix — the re-encode would throw and drop the (root) span. The key must stringify.+ (ait.log/emit! {:type :turn :name "agent.turn" :id "n" :parent-id nil+ :attributes {:ai/data {[1 2] "v"}} :events []+ :duration-ms 1.0 :start-epoch-ms 1 :end-epoch-ms 2}+ "sess-x")+ (let [es (entries messages)]+ (is (= 1 (count es)) "span still emitted despite the non-scalar map key")+ (is (= "v" (get-in (first es) ["attributes" "ai/data" "[1 2]"]))+ "the non-scalar key degraded to a string, value preserved")))))++(deftest one-entry-per-span-test+ (testing "each finished span streams exactly one JSONL line, carrying the session id in the payload"+ ;; Drive the file-routed path (with-eval-session), which is what emits to the sink. `capturing`+ ;; deliberately leaves *session-id* nil so emit! no-ops (its trace comes back as :trace instead).+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)]+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ (ait/with-eval-session "one-per-span-sid"+ (ait/with-agent-turn {:ai/profile-id "p"}+ (ait/with-llm-call {:ai/model "m"}+ (ait/with-tool-call {:ai/tool-name "search"} :ok))))+ (let [es (entries messages)]+ (is (= 3 (count es)) "one line per span (turn, llm, tool)")+ (is (= #{"agent.turn" "llm.call" "tool.search"} (set (map #(get % "name") es))))+ (is (= ["one-per-span-sid"] (distinct (map #(get % "session") es))) "all share one session")+ (is (every? #(get % "session") es)))))))++(deftest nesting-parent-chain-test+ (testing "parent ids chain turn -> llm -> tool; root has no parent"+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)]+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ (ait/with-eval-session "nesting-sid"+ (ait/with-agent-turn {}+ (ait/with-llm-call {}+ (ait/with-tool-call {:ai/tool-name "t"} :ok))))+ (let [by-name (into {} (map (juxt #(get % "name") identity)) (entries messages))+ turn (by-name "agent.turn")+ llm (by-name "llm.call")+ tool (by-name "tool.t")]+ (is (nil? (get turn "parent")))+ (is (= (get turn "id") (get llm "parent")))+ (is (= (get llm "id") (get tool "parent"))))))))++(deftest session-mint-vs-supplied-test+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)]+ (testing "with-eval-session supplies the session id verbatim"+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ (ait/with-eval-session "supplied-sid"+ (ait/with-llm-call {} :ok))+ (is (= ["supplied-sid"] (distinct (map #(get % "session") (entries messages)))))))+ (testing "with-eval-session mints a fresh uuid when none supplied"+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ (ait/with-eval-session nil+ (ait/with-llm-call {} :ok))+ (let [sid (get (first (entries messages)) "session")]+ (is (string? sid))+ (is (not= "supplied-sid" sid)))))))++(deftest session-inheritance-test+ (testing "a nested with-eval-session inherits the outer session (no re-mint)"+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly true)]+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ (ait/with-eval-session "outer"+ (ait/with-agent-turn {}+ (ait/with-eval-session "inner-should-be-ignored"+ (ait/with-llm-call {} :ok))))+ (is (= ["outer"] (distinct (map #(get % "session") (entries messages)))))))))++(deftest inert-when-disabled-test+ (testing "with-eval-session emits nothing when capture is disabled"+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly false)]+ (log.capture/with-log-messages-for-level [messages [metabase.ai-tracing.log :info]]+ (is (= :ok (ait/with-eval-session nil (ait/with-llm-call {} :ok))))+ (is (= [] (entries messages)))))))++#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}+(defn- recording-logger-factory+ "A tools.logging factory that records the Log4j2 ThreadContext present at each write."+ [sink]+ (reify clojure.tools.logging.impl/LoggerFactory+ (name [_] "recording")+ (get-logger [_ _logger-ns]+ (reify clojure.tools.logging.impl/Logger+ (enabled? [_ _level] true)+ (write! [_ _level _throwable message]+ (swap! sink conj {:mdc (into {} (ThreadContext/getImmutableContext))+ :message message}))))))++(deftest mdc-routing-key-test+ (testing "emit! sets mb-eval-session-id in the ThreadContext at log time (drives RoutingAppender)"+ (let [sink (atom [])]+ (binding [clojure.tools.logging/*logger-factory* (recording-logger-factory sink)]+ (ait.log/emit! {:type :llm :name "llm.call" :id "n" :parent-id nil+ :attributes {} :events [] :duration-ms 1.0+ :start-epoch-ms 1 :end-epoch-ms 2}+ "route-me"))+ (is (= 1 (count @sink)))+ (is (= "route-me" (get-in (first @sink) [:mdc "mb-eval-session-id"]))+ "the routing key is set on the logging thread")+ (is (= "route-me" (get (json/decode (:message (first @sink))) "session"))))))diff --git a/test/metabase/mcp/api_test.clj b/test/metabase/mcp/api_test.cljindex 692ba46fdf3..1943ef0e5b2 100644--- a/test/metabase/mcp/api_test.clj+++ b/test/metabase/mcp/api_test.clj@@ -4,6 +4,7 @@[clojure.test :refer :all][clojure.walk :as walk][metabase.agent-api.settings :as agent-api.settings]+ [metabase.ai-tracing.core :as ait][metabase.api.macros.scope :as scope][metabase.collections.models.collection :as collection][metabase.lib.core :as lib]@@ -269,6 +270,29 @@delete-response (mcp-delete {"mcp-session-id" session-id})](is (= 200 (:status delete-response))))))+(deftest ^:parallel eval-session-override-test+ (testing "the x-eval-session-id header is honored only when it's a safe trace-file name"+ (let [override #'mcp.api/eval-session-override]+ (testing "a bare uuid (what the harness mints) is honored"+ (is (= "1ae768c9-5773-48dd-afca-c75780dae84c"+ (override {:headers {"x-eval-session-id" "1ae768c9-5773-48dd-afca-c75780dae84c"}}))))+ (testing "a safe slug is honored"+ (is (= "evalprobe_1.2" (override {:headers {"x-eval-session-id" "evalprobe_1.2"}}))))+ (testing "path traversal / unsafe chars are rejected (fall back to the Mcp-Session-Id)"+ (is (nil? (override {:headers {"x-eval-session-id" "../etc/passwd"}})))+ (is (nil? (override {:headers {"x-eval-session-id" "a/b"}})))+ (is (nil? (override {:headers {"x-eval-session-id" ".hidden"}}))))+ (testing "a value over max-session-id-length is rejected even though every char is safe"+ ;; 201 chars: regex-safe but one past the real cap — the boundary a looser copy of the+ ;; contract would wrongly accept (and then 500 downstream in checked-session-id).+ (is (= 200 ait/max-session-id-length))+ (is (nil? (override {:headers {"x-eval-session-id" (apply str (repeat 201 "a"))}})))+ (testing "the id exactly at the cap is honored, returned verbatim"+ (let [id (apply str (repeat 200 "a"))]+ (is (= id (override {:headers {"x-eval-session-id" id}}))))))+ (testing "absent header yields nil"+ (is (nil? (override {:headers {}})))))))+(def ^:private all-tool-names#{"construct_query""construct_native_query"@@ -1318,7 +1342,7 @@(defn- dispatch-initialized-request [msg token-scopes](let [session-id (str (random-uuid))]- (#'mcp.api/dispatch-request msg session-id token-scopes nil)))+ (#'mcp.api/dispatch-request msg session-id token-scopes nil nil)))(defn- with-scoped-test-resource! [f](let [registry @#'mcp.resources/registry@@ -1365,6 +1389,7 @@(jsonrpc-request "resources/list")"session-id"#{"agent:other"}+ nilnil)uris (set (map :uri (get-in response [:result :resources])))](is (contains? uris construct-query-uri)@@ -1378,6 +1403,7 @@(jsonrpc-request "resources/list")"session-id"#{"agent:search"}+ nilnil)uris (set (map :uri (get-in response [:result :resources])))](is (contains? uris scoped-test-uri)))))))diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 00a3b35a0fd..79999d8ca3a 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -1,6 +1,9 @@(ns metabase.metabot.agent.core-test(:require[clojure.test :refer :all]+ [metabase.ai-tracing.core :as ait]+ [metabase.ai-tracing.log :as ait.log]+ [metabase.ai-tracing.settings :as ai-tracing.settings][metabase.analytics-interface.core :as analytics][metabase.analytics.snowplow-test :as snowplow-test][metabase.lib.core :as lib]@@ -19,6 +22,15 @@(def ^:private test-provider "openrouter/anthropic/claude-haiku-4-5")+;; Pin eval-capture OFF for the whole namespace so the exact stream-shape assertions below are+;; deterministic regardless of the ambient MB_AI_EVAL_CAPTURE — a dedicated eval instance sets it,+;; which would otherwise append an extra `eval_session` data part to every agent stream and break+;; every `=?` on `run-agent-loop` output. The capture-ON path is covered by `eval-tracing-nesting-test`,+;; which forces capture via `capture-reducible` (independent of this gate) and is unaffected here.+(use-fixtures :each (fn [thunk]+ (mt/with-dynamic-fn-redefs [ai-tracing.settings/ai-eval-capture (constantly false)]+ (thunk))))+(defn- run-agent-loop!"run-agent-loop for side effects, discarding results.Runs as admin so the base metabot permission check passes."@@ -371,6 +383,78 @@(is (= 3 @llm-call-count)"Should have exactly 3 LLM calls (search, construct, final text)")))))))))+(deftest eval-tracing-nesting-test+ (testing "capture-reducible over the real agent loop builds a turn -> llm -> tool span tree"+ ;; This exercises the cross-module wiring the ai-tracing docstrings promise: the turn/llm spans+ ;; are opened in metabase.metabot.agent.core, the tool span in metabase.metabot.self.core, and+ ;; the tool runs on the virtual-thread executor — so this also proves *parent* is conveyed across+ ;; `bound-fn*` into the tool thread. Only the LLM + search backend are mocked; everything else is+ ;; real. `capture-reducible` binds the capture unconditionally (ignores MB_AI_EVAL_CAPTURE), and+ ;; we redef `emit!` to a no-op so the test never writes a per-session JSONL file.+ (mt/with-current-user (mt/user->id :crowberto)+ (mt/with-temporary-setting-values [llm-metabot-provider test-provider]+ (let [llm-call-count (atom 0)+ llm-responses [;; Iteration 1: call the search tool (produces a real tool span)+ [{:type :start :id "msg-1"}+ {:type :tool-input+ :id "call-search-1"+ :function "search"+ :arguments {:semantic_queries ["orders table"]+ :keyword_queries ["orders"]+ :entity_types ["table"]}}+ {:type :usage :usage {:promptTokens 100 :completionTokens 20}+ :model "test" :id "msg-1"}]+ ;; Iteration 2: final text (no tool)+ [{:type :start :id "msg-2"}+ {:type :text :text "Here are your orders."}+ {:type :usage :usage {:promptTokens 200 :completionTokens 10}+ :model "test" :id "msg-2"}]]]+ (mt/with-dynamic-fn-redefs [ait.log/emit! (fn [& _] nil)+ openrouter/openrouter (fn [_opts]+ (let [n (swap! llm-call-count inc)]+ (mut/mock-llm-response (get llm-responses (dec n) []))))+ metabot-search/search (fn [_args]+ [{:id (mt/id :orders)+ :type "table"+ :name "orders"+ :display_name "Orders"+ :description "Confirmed orders."+ :database_id (mt/id)}])]+ (let [{:keys [trace result]} (mt/with-log-level [metabase.metabot.agent.core :warn]+ (ait/capture-reducible+ (agent/run-agent-loop+ {:messages [{:role :user :content "Show me orders"}]+ :state {}+ :profile-id :internal+ :context {}})))]+ (is (= 2 @llm-call-count) "search iteration + final text iteration")+ (is (= 1 (count trace)) "single root span")+ (testing "the in-process (file-less) path emits no eval_session pointer into the stream"+ ;; capture-reducible leaves *session-id* nil (trace comes back as :trace, no file), so+ ;; a pointer here would name a <nil>.jsonl that never exists.+ (is (empty? (filter #(= "eval_session" (:data-type %)) result))))+ (let [turn (first trace)+ llms (:children turn)+ tool-spans (mapcat :children llms)+ search (first (filter #(= "tool.search" (:name %)) tool-spans))]+ (testing "root is the agent turn"+ (is (= :turn (:type turn)))+ (is (= "agent.turn" (:name turn)))+ (is (nil? (:parent-id turn))))+ (testing "each iteration is an llm.call child of the turn"+ (is (= 2 (count llms)))+ (is (every? #(= :llm (:type %)) llms))+ (is (every? #(= (:id turn) (:parent-id %)) llms)))+ (testing "the search tool span nests under the llm.call it ran in"+ (is (= :tool (:type search)))+ ;; the decoded tool arguments are recorded on the span (metabase.metabot.self.core)+ (is (= {:semantic_queries ["orders table"]+ :keyword_queries ["orders"]+ :entity_types ["table"]}+ (get-in search [:attributes :ai/tool-args])))+ (let [parent-llm (first (filter #(= (:parent-id search) (:id %)) llms))]+ (is (some? parent-llm) "tool's parent-id resolves to an llm.call in the tree")))))))))))+(deftest cumulative-usage-test(mt/as-admin(mt/with-temporary-setting-values [llm-metabot-provider test-provider]diff --git a/test_config/log4j2-test.xml b/test_config/log4j2-test.xmlindex aa33f1d8ed1..6b7cb4a00c1 100644--- a/test_config/log4j2-test.xml+++ b/test_config/log4j2-test.xml@@ -25,6 +25,12 @@<Logger name="metabase" level="INFO"/><Logger name="metabase-enterprise" level="INFO"/>+ <!-- Eval-trace sink (metabase.ai-tracing): additivity=false with no appender keeps the JSONL+ spans out of the shared test log. Tests that assert on emitted spans attach their own+ appender via metabase.util.log.capture, so this does not hide them. Mirrors the prod+ resources/log4j2.xml, minus the per-session RoutingAppender (tests don't read the files). -->+ <Logger name="metabase.ai-tracing.log" level="info" additivity="false"/>+<Logger name="metabase.test.gentest" level="INFO" additivity="false"><AppenderRef ref="ConsoleInfoLevel"/><AppenderRef ref="Console"/>