Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT - Debug opt-in helper for agent request & response JSON (#72780)

Tyler Plude119889d16fd6
diff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.clj
index fb3d1b907c9..a3220e9c953 100644
--- a/src/metabase/metabot/self/claude.clj
+++ b/src/metabase/metabot/self/claude.clj
@@ -4,6 +4,7 @@
[malli.json-schema :as mjs]
[metabase.llm.settings :as llm]
[metabase.metabot.self.core :as core]
+ [metabase.metabot.self.debug :as debug]
[metabase.metabot.self.schema :as schema]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
@@ -226,10 +227,10 @@
:or {model "claude-haiku-4-5"}} :- core/LLMRequestOpts]
(let [messages (parts->claude-messages input)
all-tools (when (seq tools) (mapv tool->claude tools))
- req (cond-> {:model model
- :max_tokens (or max-tokens 4096)
- :stream true
- :messages messages}
+ req (cond-> {:model model
+ :max_tokens (or max-tokens 4096)
+ :stream true
+ :messages messages}
system (assoc :system system)
all-tools (assoc :tools all-tools)
(and all-tools
@@ -260,7 +261,11 @@
:headers {"anthropic-version" "2023-06-01"
"content-type" "application/json"}
:body (json/encode req)})]
- (core/sse-reducible (:body response)))
+ (-> (core/sse-reducible (:body response))
+ (debug/capture-stream {:provider "anthropic"
+ :model model
+ :url "/v1/messages"
+ :request req})))
(catch Exception e
(core/rethrow-api-error! "anthropic" anthropic-errors e))))))
diff --git a/src/metabase/metabot/self/debug.clj b/src/metabase/metabot/self/debug.clj
new file mode 100644
index 00000000000..1a5aaae7ece
--- /dev/null
+++ b/src/metabase/metabot/self/debug.clj
@@ -0,0 +1,80 @@
+(ns metabase.metabot.self.debug
+ "Opt-in helper for dumping the fully composed LLM request payload and the raw
+ provider response (SSE events) to disk. Each LLM call produces one file at
+ `logs/ai/requests/agent_<ts>.json`.
+
+ Enabled by setting `MB_METABOT_DEBUG_LLM_REQUESTS=true`. Disabled (zero overhead)
+ otherwise, including in dev.
+
+ Wired into the provider adapters (`claude-raw`, `openai-raw`, `openrouter-raw`)
+ via [[capture-stream]], which tees every raw SSE event into a vector and flushes
+ the file on reduction completion."
+ (:require
+ [clojure.java.io :as io]
+ [metabase.config.core :as config]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log])
+ (:import
+ (java.io File)
+ (java.time LocalDateTime)
+ (java.time.format DateTimeFormatter)))
+
+(set! *warn-on-reflection* true)
+
+(defn- enabled? []
+ (boolean (config/config-bool :mb-metabot-debug-llm-requests)))
+
+(def ^:private log-dir "logs/ai/requests")
+
+(def ^:private ^DateTimeFormatter ts-fmt
+ (DateTimeFormatter/ofPattern "yyyyMMdd_HHmmss_SSS"))
+
+(defn- now-ts []
+ (.format (LocalDateTime/now) ts-fmt))
+
+(defn- log-file ^File []
+ (io/file log-dir (str "agent_" (now-ts) ".json")))
+
+(defn write-request-log!
+ "Write a composed LLM request + response payload to `logs/ai/requests/agent_<ts>.json`.
+ Silently no-ops unless `MB_METABOT_DEBUG_LLM_REQUESTS` is truthy. The `entry`
+ map should minimally contain `:provider :model :request :response`; additional
+ keys (e.g. `:url`, `:error`) pass through unchanged."
+ [entry]
+ (when (enabled?)
+ (try
+ (.mkdirs (io/file log-dir))
+ (let [f (log-file)]
+ (with-open [w (io/writer f)]
+ (.write w ^String (json/encode entry {:pretty true})))
+ (log/debugf "Wrote AI request log to %s" (.getPath f)))
+ (catch Exception e
+ (log/warn e "Failed to write AI request log")))))
+
+(defn capture-stream
+ "Wrap an SSE reducible so every raw provider event is teed into a vector while
+ still flowing through to the downstream consumer. When the reduction completes
+ (or throws), the composed request (`log-context`, typically
+ `{:provider ... :model ... :url ... :request ...}`) plus the captured
+ `:response` vector is written via [[write-request-log!]].
+
+ When `MB_METABOT_DEBUG_LLM_REQUESTS` is unset or false, returns `reducible`
+ unchanged so there's zero overhead."
+ [reducible log-context]
+ (if-not (enabled?)
+ reducible
+ (reify clojure.lang.IReduceInit
+ (reduce [_ rf init]
+ (let [events (atom [])
+ tee-rf (fn [acc event]
+ (swap! events conj event)
+ (rf acc event))]
+ (try
+ (let [result (reduce tee-rf init reducible)]
+ (write-request-log! (assoc log-context :response @events))
+ result)
+ (catch Exception e
+ (write-request-log! (assoc log-context
+ :response @events
+ :error (ex-message e)))
+ (throw e))))))))
diff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.clj
index 3e462cfd381..77cbb197dd7 100644
--- a/src/metabase/metabot/self/openai.clj
+++ b/src/metabase/metabot/self/openai.clj
@@ -4,6 +4,7 @@
[malli.json-schema :as mjs]
[metabase.llm.settings :as llm]
[metabase.metabot.self.core :as core]
+ [metabase.metabot.self.debug :as debug]
[metabase.metabot.self.schema :as schema]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
@@ -232,7 +233,11 @@
:as :stream
:headers {"Content-Type" "application/json"}
:body (json/encode req)})]
- (core/sse-reducible (:body response)))
+ (-> (core/sse-reducible (:body response))
+ (debug/capture-stream {:provider "openai"
+ :model model
+ :url "/v1/responses"
+ :request req})))
(catch Exception e
(core/rethrow-api-error! "openai" openai-errors e)))))
diff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.clj
index 482f7952221..cc0d0f41531 100644
--- a/src/metabase/metabot/self/openrouter.clj
+++ b/src/metabase/metabot/self/openrouter.clj
@@ -12,6 +12,7 @@
[malli.json-schema :as mjs]
[metabase.llm.settings :as llm]
[metabase.metabot.self.core :as core]
+ [metabase.metabot.self.debug :as debug]
[metabase.metabot.self.schema :as schema]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
@@ -298,7 +299,11 @@
"HTTP-Referer" "https://metabase.com"
"X-Title" "Metabase"}
:body (json/encode req)})]
- (core/sse-reducible (:body response)))
+ (-> (core/sse-reducible (:body response))
+ (debug/capture-stream {:provider "openrouter"
+ :model model
+ :url "/v1/chat/completions"
+ :request req})))
(catch Exception e
(core/rethrow-api-error! "openrouter" openrouter-errors e))))))