Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1231 Remove user-intent tracking (#72199)

appleby95b6569503b1
diff --git a/src/metabase/metabot/agent/analytics.clj b/src/metabase/metabot/agent/analytics.clj
deleted file mode 100644
index ea47a7edcde..00000000000
--- a/src/metabase/metabot/agent/analytics.clj
+++ /dev/null
@@ -1,130 +0,0 @@
-(ns metabase.metabot.agent.analytics
- "Snowplow and Prometheus analytics and intent-classification helpers for the metabot agent."
- (:require
- [clojure.string :as str]
- [metabase.analytics.core :as analytics]
- [metabase.api.common :as api]
- [metabase.metabot.self :as self]
- [metabase.metabot.settings :as metabot.settings]
- [metabase.util.json :as json]
- [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-;;; User intent classification
-
-(def ^:private user-intents
- {"create_or_modify_sql_query"
- {:description "User wants to write new SQL from scratch or edit existing SQL"
- :examples ["Write a SQL query to find all orders from last month."
- "Edit the query to include customer email addresses (while looking at a SQL query)"]}
- "fix_broken_sql_query"
- {:description "User has broken SQL that needs fixing"
- :examples ["My SQL query is giving an error, can you fix it?"
- "Fix this query."]}
- "query_data"
- {:description "User wants to query data to get insights."
- :examples ["Show me sales by region for Q1."
- "How many users signed up last week?"]}
- "data_discovery"
- {:description "User looking for relevant tables/data or existing Metabase dashboards/questions"
- :examples ["Which table has customer information?"
- "What data can I use to analyze user behavior?"]}
- "editing_visualizations"
- {:description "User wants to build or edit visualizations"
- :examples ["Create a bar chart for revenue by product."
- "Turn this into a bar chart."]}
- "interpret_results"
- {:description "User needs help understanding data/charts"
- :examples ["What does this chart mean?"
- "Explain the results of this query."]}
- "create_dashboard"
- {:description "User wants to build dashboard"
- :examples ["Build a dashboard for sales KPIs."
- "Create a dashboard with multiple visualizations."]}
- "navigation"
- {:description "User wants to navigate to a specific Metabase UI location or feature"
- :examples ["Take me to the SQL editor."
- "Show me the admin settings."]}
- "metabase_help"
- {:description "User needs help using Metabase"
- :examples ["How do I share a dashboard?"
- "Help me set up a new database connection."
- "How do I use custom expressions?"]}
- "other"
- {:description "User request doesn't fit standard categories"
- :examples ["Any other request not covered above."]}})
-
-(defn- render-user-intents-table
- "Render user-intents as a plain-text table."
- []
- (let [header "| Intent Category | Description | Examples |"
- sep "|-----------------|-------------|----------|"
- rows (map (fn [[intent {:keys [description examples]}]]
- (str "| " intent " | " description " | " (str/join "; " examples) " |"))
- user-intents)]
- (str/join "\n" (concat [header sep] rows))))
-
-(defn- build-intent-prompt [conversation-context]
- (str "You are a user intent classification engine for requests to an AI agent operating on top of Metabase.\n"
- "Given a conversation, classify the user's intent into one of the following categories:\n\n"
- (render-user-intents-table)
- "\n\nThis is the conversation context:\n"
- "<conversation_context>\n"
- conversation-context
- "\n</conversation_context>\n\n"
- "## Output format\n"
- "Return the category wrapped in a <category> tag so that we can easily extract it from the string.\n"
- "Only respond with the <category> tag including the intent category name - nothing else.\n\n"
- "**Example:**\n"
- "User: \"I want to see sales by region for Q1.\"\n"
- "You: \"<category>query_data</category>\"\n\n"
- "Classify the user's intent into one of the Intent Categories above.\n"))
-
-(defn- classify-user-intent!
- "Classify the user's message into one of the [[user-intent]] categories.
- Returns the intent string.
- Raises an exception if the classified intent is not in the known intents.
- Uses plain-text extraction (no tool calls) so it works with models that don't support function calling."
- [messages]
- (let [context (json/encode (take-last 3 messages))
- text (transduce (keep (fn [{:keys [type text]}] (when (= type :text) text)))
- str
- (self/call-llm (metabot.settings/llm-metabot-provider-lite)
- nil
- [{:role "user" :content (build-intent-prompt context)}]
- []
- ;; Omit :request-id from tracking-opts to skip snowplow token_usage event
- ;; since this is for internal usage.
- {:tag "user-intent-classification"}))
- intent (second (re-find #"<category>([^<]+)</category>" text))]
- (when-not (contains? user-intents intent)
- (throw (ex-info (str "Classified intent '" intent "' not in known intents")
- {:intent intent
- :valid-intents (keys user-intents)})))
- intent))
-
-(defn- track-user-intent!
- "Fire user_intent :snowplow/ai_service_event synchronously."
- [intent tracking-opts]
- (analytics/track-event!
- :snowplow/ai_service_event
- {:hashed-metabase-license-token (analytics/hashed-metabase-token-or-uuid)
- :request-id (analytics/uuid->ai-service-hex-uuid (:request-id tracking-opts))
- :source (:source tracking-opts)
- :event "user_intent"
- :user-id api/*current-user-id*
- :session-id (:session-id tracking-opts)
- :profile (some-> (:profile-id tracking-opts) name)
- :event-details {"intent" intent}}))
-
-(defn classify-and-track-user-intent-async!
- "Classify and [[track-user-intent!]] in a background future."
- [messages tracking-opts]
- (future
- (try
- (-> messages
- classify-user-intent!
- (track-user-intent! tracking-opts))
- (catch Exception e
- (log/warn e "Failed to track user intent")))))
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index 2dd096e4818..0d1817a6f25 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -7,7 +7,6 @@
[metabase.api-scope.core :as api-scope]
[metabase.api.common :as api]
[metabase.config.core :as config]
- [metabase.metabot.agent.analytics :as agent-analytics]
[metabase.metabot.agent.links :as links]
[metabase.metabot.agent.memory :as memory]
[metabase.metabot.agent.messages :as messages]
@@ -16,7 +15,6 @@
[metabase.metabot.provider-util :as provider-util]
[metabase.metabot.scope :as scope]
[metabase.metabot.self :as self]
- [metabase.metabot.settings :as metabot.settings]
[metabase.metabot.tools :as tools]
[metabase.util :as u]
[metabase.util.json :as json]
@@ -168,8 +166,7 @@
[:map
[:session-id {:optional true} [:maybe ms/UUIDString]]
[:source {:optional true} [:maybe :string]]
- [:tag {:optional true} [:maybe :string]]
- [:track-user-intent? {:optional true} [:maybe :boolean]]])
+ [:tag {:optional true} [:maybe :string]]])
;;; Iteration control
@@ -590,8 +587,6 @@
(let [profile-id (:profile-id opts)
debug? (:debug? opts)
labels {:profile-id (name profile-id)}
- track-user-intent? (and (metabot.settings/llm-metabot-internal-tasks-enabled?)
- (some-> opts :tracking-opts :track-user-intent?))
perms (or scope/*current-user-metabot-permissions*
(if api/*is-superuser?*
scope/all-yes-permissions
@@ -612,10 +607,6 @@
scope/*current-user-metabot-permissions* perms]
(try
(let [agent (init-agent opts)
- _ (when track-user-intent?
- (agent-analytics/classify-and-track-user-intent-async!
- (:messages opts)
- (:tracking-opts agent)))
{result :result
iteration :iteration} (->> (initial-loop-state agent rf init (atom {}))
(iterate loop-step)
diff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.clj
index 6b1e42bc104..fb732e00cf3 100644
--- a/src/metabase/metabot/api.clj
+++ b/src/metabase/metabot/api.clj
@@ -188,8 +188,7 @@
:metabot-id metabot-id
:profile-id (keyword profile-id)
:context enriched-context
- :tracking-opts {:session-id conversation-id
- :track-user-intent? true}}
+ :tracking-opts {:session-id conversation-id}}
debug? (assoc :debug? true))))
(catch org.eclipse.jetty.io.EofException _
(log/debug "Client disconnected during native agent streaming"))
diff --git a/src/metabase/metabot/api/document.clj b/src/metabase/metabot/api/document.clj
index 10c1ca75405..925d2c096fe 100644
--- a/src/metabase/metabot/api/document.clj
+++ b/src/metabase/metabot/api/document.clj
@@ -100,8 +100,7 @@
:profile-id :document-generate-content
:state {}
:context context
- :tracking-opts {:source "document_generate_content"
- :track-user-intent? true}}))
+ :tracking-opts {:source "document_generate_content"}}))
chart-output (latest-chart-structured-output parts)
draft-card (draft-card-from-chart-output chart-output)
description (or (:description chart-output)
diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.clj
index 5c05d49d720..7886eb86dab 100644
--- a/src/metabase/metabot/settings.clj
+++ b/src/metabase/metabot/settings.clj
@@ -148,20 +148,6 @@
(validate-metabot-provider! new-value))
(setting/set-value-of-type! :string :llm-metabot-provider new-value)))
-(defsetting llm-metabot-provider-lite
- (deferred-tru "The AI provider and model for lightweight Metabot tasks (e.g. user intent classification).")
- :type :string
- :encryption :no
- :default "anthropic/claude-haiku-4-5"
- :visibility :settings-manager
- :export? false
- :deprecated-name :ee-ai-metabot-provider-lite
- :doc false
- :setter (fn [new-value]
- (when new-value
- (validate-metabot-provider! new-value))
- (setting/set-value-of-type! :string :llm-metabot-provider-lite new-value)))
-
(defn- token-configured?
[token]
(boolean (and (string? token)
@@ -189,17 +175,6 @@
configured-provider-api-key
token-configured?))))
-(defsetting llm-metabot-internal-tasks-enabled?
- (deferred-tru "Controls whether Metabot performs internal tasks that might require background tasks or additional LLM calls (e.g. user intent classification).")
- :type :boolean
- :visibility :settings-manager
- :default false
- :export? false
- :deprecated-name :ee-ai-metabot-internal-tasks-enabled?
- :doc false
- :getter #(and (setting/get-value-of-type :boolean :llm-metabot-internal-tasks-enabled?)
- (llm-provider-configured? (llm-metabot-provider-lite))))
-
(defsetting llm-metabot-configured?
"Whether the API key for the selected Metabot provider is configured."
:type :boolean
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 43f90e0ec45..71114ae4191 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -4,8 +4,6 @@
[metabase.analytics.prometheus :as prometheus]
[metabase.analytics.snowplow-test :as snowplow-test]
[metabase.lib.core :as lib]
- [metabase.llm.settings :as llm.settings]
- [metabase.metabot.agent.analytics :as agent-analytics]
[metabase.metabot.agent.core :as agent]
[metabase.metabot.agent.memory :as memory]
[metabase.metabot.api :as api]
@@ -13,27 +11,12 @@
[metabase.metabot.self.openrouter :as openrouter]
[metabase.metabot.test-util :as mut]
[metabase.metabot.tools.search :as metabot-search]
- [metabase.test :as mt]
- [metabase.test.util :as tu]
- [metabase.util.malli :as mu]))
+ [metabase.test :as mt]))
(set! *warn-on-reflection* true)
(def ^:private test-provider "openrouter/anthropic/claude-haiku-4-5")
-;; Mock tool for testing
-(mu/defn test-search-tool
- "Mock search tool that returns test data."
- [{:keys [_query]} :- [:map {:closed true}
- [:query :string]]]
- {:structured-output {:data [{:id 1 :name "Test Result"}]}})
-
-(def test-tools
- {"search" {:tool-name "search"
- :schema (:schema (meta #'test-search-tool))
- :doc (:doc (meta #'test-search-tool))
- :fn test-search-tool}})
-
(defn- run-agent-loop!
"run-agent-loop for side effects, discarding results.
Runs as admin so the base metabot permission check passes."
@@ -681,112 +664,6 @@
"session_id" "00000000-0000-0000-0000-000000000001"}}]
token-events)))))))))))
-;;; ===================== User Intent Classification Tests =====================
-
-(def ^:private mock-llm-response-for-intent
- [{:type :start :id "msg-1"}
- {:type :text :text "Done"}
- {:type :usage :usage {:promptTokens 10 :completionTokens 5}
- :model "test-model" :id "msg-1"}])
-
-(deftest user-intent-snowplow-fires-event-test
- (mt/with-temporary-setting-values [llm-metabot-provider test-provider
- llm-metabot-internal-tasks-enabled? true
- llm-anthropic-api-key "sk-ant-test"]
- (testing "fires :snowplow/ai_service_event 'user_intent' when :track-user-intent? is true"
- (let [rasta-id (mt/user->id :rasta)]
- (with-redefs [openrouter/openrouter (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))
- self/call-llm (fn [& _] [{:type :text :text "<category>query_data</category>"}])]
- (mt/with-current-user rasta-id
- (snowplow-test/with-fake-snowplow-collector
- (run-agent-loop! {:messages [{:role :user :content "Show sales by region"}]
- :state {}
- :context {}
- :profile-id :internal
- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000002"
- :track-user-intent? true}})
- (tu/poll-until 5000
- (some #(= "user_intent" (get-in % [:properties "data" "data" "event"]))
- @snowplow-test/*snowplow-collector*))
- (let [events (snowplow-test/pop-event-data-and-user-id!)
- intent-events (filter #(= "user_intent" (get-in % [:data "event"])) events)]
- (is (=? [{:user-id (str rasta-id)
- :data {"event" "user_intent"
- "source" "metabot_agent"
- "profile" "internal"
- "session_id" "00000000-0000-0000-0000-000000000002"
- "event_details" {"intent" "query_data"}}}]
- intent-events))))))))))
-
-(deftest user-intent-not-tracked-when-disabled-test
- (mt/with-temporary-setting-values [llm-metabot-provider test-provider]
- (testing "does not fire user_intent event when :track-user-intent? is false"
- (let [classify-called (atom false)]
- (with-redefs [openrouter/openrouter
- (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))
-
- agent-analytics/classify-and-track-user-intent-async!
- (fn [_ _] (reset! classify-called true))]
- (run-agent-loop! {:messages [{:role :user :content "test"}]
- :state {}
- :context {}
- :profile-id :internal
- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000003"}})
- (is (false? @classify-called)))))))
-
-(deftest user-intent-not-tracked-when-internal-tasks-disabled-test
- (mt/with-temporary-setting-values [llm-metabot-provider test-provider]
- (let [opts {:messages [{:role :user :content "Show sales by region"}]
- :state {}
- :context {}
- :profile-id :internal
- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000005"
- :track-user-intent? true}}
- classify-called (atom 0)]
- (with-redefs [openrouter/openrouter
- (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))
-
- agent-analytics/classify-and-track-user-intent-async!
- (fn [_ _] (swap! classify-called inc))]
- (testing "does not call classify-and-track-user-intent-async! when llm-metabot-internal-tasks-enabled? is false"
- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? false
- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"
- llm-anthropic-api-key "sk-ant-test"]
- (run-agent-loop! opts)
- (is (zero? @classify-called))))
- (testing "does not call classify-and-track-user-intent-async! when api key is not configured"
- (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]
- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true
- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"]
- (run-agent-loop! opts)
- (is (zero? @classify-called)))))
- (testing "calls classify-and-track-user-intent-async! when llm-metabot-internal-tasks-enabled? is true"
- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true
- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"
- llm-anthropic-api-key "sk-ant-test"]
- (run-agent-loop! opts)
- (is (= 1 @classify-called))))))))
-
-(deftest user-intent-classifier-exception-swallowed-test
- (mt/with-temporary-setting-values [llm-metabot-provider test-provider
- llm-metabot-internal-tasks-enabled? true
- llm-anthropic-api-key "sk-ant-test"]
- (testing "does not propagate exception if classify-user-intent throws"
- (let [classify-called (atom false)]
- (with-redefs [openrouter/openrouter (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))
- self/call-llm (fn [& _]
- (reset! classify-called true)
- (throw (ex-info "LLM error" {})))]
- ;; Should not throw — exception is caught inside the background future
- (run-agent-loop! {:messages [{:role :user :content "test"}]
- :state {}
- :context {}
- :profile-id :internal
- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000004"
- :track-user-intent? true}})
- (tu/poll-until 5000 @classify-called)
- (is (true? @classify-called)))))))
-
(deftest chart-configs-loaded-into-charts-test
(let [query (lib/native-query (mt/metadata-provider) "select 1")
chart-config {:display_type "pie"
diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.clj
index b01d3fce90a..16bceff3bd9 100644
--- a/test/metabase/metabot/settings_test.clj
+++ b/test/metabase/metabot/settings_test.clj
@@ -74,8 +74,7 @@
(mt/with-premium-features #{}
(is (boolean? (metabot.settings/metabot-enabled?)))
(is (boolean? (metabot.settings/embedded-metabot-enabled?)))
- (is (string? (metabot.settings/llm-metabot-provider)))
- (is (string? (metabot.settings/llm-metabot-provider-lite))))))
+ (is (string? (metabot.settings/llm-metabot-provider))))))
(deftest metabot-configured-with-metabase-provider-and-proxy-url-test
(testing "returns true when metabase-proxied provider and proxy URL is set"
@@ -162,32 +161,3 @@
(is (= "metabase/anthropic/claude-sonnet-4" (metabot.settings/llm-metabot-provider))))
(mt/with-temporary-setting-values [llm-metabot-provider "metabase/openrouter/anthropic/claude-haiku-4-5"]
(is (= "metabase/openrouter/anthropic/claude-haiku-4-5" (metabot.settings/llm-metabot-provider)))))))
-
-;;; ------------------------------------------- Internal Tasks Tests -------------------------------------------
-
-(deftest internal-tasks-disabled-by-default-test
- (testing "returns false when the setting is not explicitly enabled"
- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? nil
- llm-anthropic-api-key "sk-ant-test"]
- (is (false? (metabot.settings/llm-metabot-internal-tasks-enabled?))))))
-
-(deftest internal-tasks-requires-lite-api-key-test
- (testing "returns false when enabled but lite provider has no api key"
- (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]
- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true
- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"]
- (is (false? (metabot.settings/llm-metabot-internal-tasks-enabled?)))))))
-
-(deftest internal-tasks-enabled-with-anthropic-test
- (testing "returns true when enabled and anthropic lite provider api key is configured"
- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true
- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"
- llm-anthropic-api-key "sk-ant-test"]
- (is (true? (metabot.settings/llm-metabot-internal-tasks-enabled?))))))
-
-(deftest internal-tasks-enabled-with-openai-test
- (testing "works with openai lite provider"
- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true
- llm-metabot-provider-lite "openai/gpt-4.1-mini"
- llm-openai-api-key "sk-openai-test"]
- (is (true? (metabot.settings/llm-metabot-internal-tasks-enabled?))))))