The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1231 Remove user-intent tracking (#72199)
appleby95b6569503b1
diff --git a/src/metabase/metabot/agent/analytics.clj b/src/metabase/metabot/agent/analytics.cljdeleted file mode 100644index ea47a7edcde..00000000000--- a/src/metabase/metabot/agent/analytics.clj+++ /dev/null@@ -1,130 +0,0 @@-(ns metabase.metabot.agent.analytics- "Snowplow and Prometheus analytics and intent-classification helpers for the metabot agent."- (:require- [clojure.string :as str]- [metabase.analytics.core :as analytics]- [metabase.api.common :as api]- [metabase.metabot.self :as self]- [metabase.metabot.settings :as metabot.settings]- [metabase.util.json :as json]- [metabase.util.log :as log]))--(set! *warn-on-reflection* true)--;;; User intent classification--(def ^:private user-intents- {"create_or_modify_sql_query"- {:description "User wants to write new SQL from scratch or edit existing SQL"- :examples ["Write a SQL query to find all orders from last month."- "Edit the query to include customer email addresses (while looking at a SQL query)"]}- "fix_broken_sql_query"- {:description "User has broken SQL that needs fixing"- :examples ["My SQL query is giving an error, can you fix it?"- "Fix this query."]}- "query_data"- {:description "User wants to query data to get insights."- :examples ["Show me sales by region for Q1."- "How many users signed up last week?"]}- "data_discovery"- {:description "User looking for relevant tables/data or existing Metabase dashboards/questions"- :examples ["Which table has customer information?"- "What data can I use to analyze user behavior?"]}- "editing_visualizations"- {:description "User wants to build or edit visualizations"- :examples ["Create a bar chart for revenue by product."- "Turn this into a bar chart."]}- "interpret_results"- {:description "User needs help understanding data/charts"- :examples ["What does this chart mean?"- "Explain the results of this query."]}- "create_dashboard"- {:description "User wants to build dashboard"- :examples ["Build a dashboard for sales KPIs."- "Create a dashboard with multiple visualizations."]}- "navigation"- {:description "User wants to navigate to a specific Metabase UI location or feature"- :examples ["Take me to the SQL editor."- "Show me the admin settings."]}- "metabase_help"- {:description "User needs help using Metabase"- :examples ["How do I share a dashboard?"- "Help me set up a new database connection."- "How do I use custom expressions?"]}- "other"- {:description "User request doesn't fit standard categories"- :examples ["Any other request not covered above."]}})--(defn- render-user-intents-table- "Render user-intents as a plain-text table."- []- (let [header "| Intent Category | Description | Examples |"- sep "|-----------------|-------------|----------|"- rows (map (fn [[intent {:keys [description examples]}]]- (str "| " intent " | " description " | " (str/join "; " examples) " |"))- user-intents)]- (str/join "\n" (concat [header sep] rows))))--(defn- build-intent-prompt [conversation-context]- (str "You are a user intent classification engine for requests to an AI agent operating on top of Metabase.\n"- "Given a conversation, classify the user's intent into one of the following categories:\n\n"- (render-user-intents-table)- "\n\nThis is the conversation context:\n"- "<conversation_context>\n"- conversation-context- "\n</conversation_context>\n\n"- "## Output format\n"- "Return the category wrapped in a <category> tag so that we can easily extract it from the string.\n"- "Only respond with the <category> tag including the intent category name - nothing else.\n\n"- "**Example:**\n"- "User: \"I want to see sales by region for Q1.\"\n"- "You: \"<category>query_data</category>\"\n\n"- "Classify the user's intent into one of the Intent Categories above.\n"))--(defn- classify-user-intent!- "Classify the user's message into one of the [[user-intent]] categories.- Returns the intent string.- Raises an exception if the classified intent is not in the known intents.- Uses plain-text extraction (no tool calls) so it works with models that don't support function calling."- [messages]- (let [context (json/encode (take-last 3 messages))- text (transduce (keep (fn [{:keys [type text]}] (when (= type :text) text)))- str- (self/call-llm (metabot.settings/llm-metabot-provider-lite)- nil- [{:role "user" :content (build-intent-prompt context)}]- []- ;; Omit :request-id from tracking-opts to skip snowplow token_usage event- ;; since this is for internal usage.- {:tag "user-intent-classification"}))- intent (second (re-find #"<category>([^<]+)</category>" text))]- (when-not (contains? user-intents intent)- (throw (ex-info (str "Classified intent '" intent "' not in known intents")- {:intent intent- :valid-intents (keys user-intents)})))- intent))--(defn- track-user-intent!- "Fire user_intent :snowplow/ai_service_event synchronously."- [intent tracking-opts]- (analytics/track-event!- :snowplow/ai_service_event- {:hashed-metabase-license-token (analytics/hashed-metabase-token-or-uuid)- :request-id (analytics/uuid->ai-service-hex-uuid (:request-id tracking-opts))- :source (:source tracking-opts)- :event "user_intent"- :user-id api/*current-user-id*- :session-id (:session-id tracking-opts)- :profile (some-> (:profile-id tracking-opts) name)- :event-details {"intent" intent}}))--(defn classify-and-track-user-intent-async!- "Classify and [[track-user-intent!]] in a background future."- [messages tracking-opts]- (future- (try- (-> messages- classify-user-intent!- (track-user-intent! tracking-opts))- (catch Exception e- (log/warn e "Failed to track user intent")))))diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex 2dd096e4818..0d1817a6f25 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -7,7 +7,6 @@[metabase.api-scope.core :as api-scope][metabase.api.common :as api][metabase.config.core :as config]- [metabase.metabot.agent.analytics :as agent-analytics][metabase.metabot.agent.links :as links][metabase.metabot.agent.memory :as memory][metabase.metabot.agent.messages :as messages]@@ -16,7 +15,6 @@[metabase.metabot.provider-util :as provider-util][metabase.metabot.scope :as scope][metabase.metabot.self :as self]- [metabase.metabot.settings :as metabot.settings][metabase.metabot.tools :as tools][metabase.util :as u][metabase.util.json :as json]@@ -168,8 +166,7 @@[:map[:session-id {:optional true} [:maybe ms/UUIDString]][:source {:optional true} [:maybe :string]]- [:tag {:optional true} [:maybe :string]]- [:track-user-intent? {:optional true} [:maybe :boolean]]])+ [:tag {:optional true} [:maybe :string]]]);;; Iteration control@@ -590,8 +587,6 @@(let [profile-id (:profile-id opts)debug? (:debug? opts)labels {:profile-id (name profile-id)}- track-user-intent? (and (metabot.settings/llm-metabot-internal-tasks-enabled?)- (some-> opts :tracking-opts :track-user-intent?))perms (or scope/*current-user-metabot-permissions*(if api/*is-superuser?*scope/all-yes-permissions@@ -612,10 +607,6 @@scope/*current-user-metabot-permissions* perms](try(let [agent (init-agent opts)- _ (when track-user-intent?- (agent-analytics/classify-and-track-user-intent-async!- (:messages opts)- (:tracking-opts agent))){result :resultiteration :iteration} (->> (initial-loop-state agent rf init (atom {}))(iterate loop-step)diff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.cljindex 6b1e42bc104..fb732e00cf3 100644--- a/src/metabase/metabot/api.clj+++ b/src/metabase/metabot/api.clj@@ -188,8 +188,7 @@:metabot-id metabot-id:profile-id (keyword profile-id):context enriched-context- :tracking-opts {:session-id conversation-id- :track-user-intent? true}}+ :tracking-opts {:session-id conversation-id}}debug? (assoc :debug? true))))(catch org.eclipse.jetty.io.EofException _(log/debug "Client disconnected during native agent streaming"))diff --git a/src/metabase/metabot/api/document.clj b/src/metabase/metabot/api/document.cljindex 10c1ca75405..925d2c096fe 100644--- a/src/metabase/metabot/api/document.clj+++ b/src/metabase/metabot/api/document.clj@@ -100,8 +100,7 @@:profile-id :document-generate-content:state {}:context context- :tracking-opts {:source "document_generate_content"- :track-user-intent? true}}))+ :tracking-opts {:source "document_generate_content"}}))chart-output (latest-chart-structured-output parts)draft-card (draft-card-from-chart-output chart-output)description (or (:description chart-output)diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.cljindex 5c05d49d720..7886eb86dab 100644--- a/src/metabase/metabot/settings.clj+++ b/src/metabase/metabot/settings.clj@@ -148,20 +148,6 @@(validate-metabot-provider! new-value))(setting/set-value-of-type! :string :llm-metabot-provider new-value)))-(defsetting llm-metabot-provider-lite- (deferred-tru "The AI provider and model for lightweight Metabot tasks (e.g. user intent classification).")- :type :string- :encryption :no- :default "anthropic/claude-haiku-4-5"- :visibility :settings-manager- :export? false- :deprecated-name :ee-ai-metabot-provider-lite- :doc false- :setter (fn [new-value]- (when new-value- (validate-metabot-provider! new-value))- (setting/set-value-of-type! :string :llm-metabot-provider-lite new-value)))-(defn- token-configured?[token](boolean (and (string? token)@@ -189,17 +175,6 @@configured-provider-api-keytoken-configured?))))-(defsetting llm-metabot-internal-tasks-enabled?- (deferred-tru "Controls whether Metabot performs internal tasks that might require background tasks or additional LLM calls (e.g. user intent classification).")- :type :boolean- :visibility :settings-manager- :default false- :export? false- :deprecated-name :ee-ai-metabot-internal-tasks-enabled?- :doc false- :getter #(and (setting/get-value-of-type :boolean :llm-metabot-internal-tasks-enabled?)- (llm-provider-configured? (llm-metabot-provider-lite))))-(defsetting llm-metabot-configured?"Whether the API key for the selected Metabot provider is configured.":type :booleandiff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 43f90e0ec45..71114ae4191 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -4,8 +4,6 @@[metabase.analytics.prometheus :as prometheus][metabase.analytics.snowplow-test :as snowplow-test][metabase.lib.core :as lib]- [metabase.llm.settings :as llm.settings]- [metabase.metabot.agent.analytics :as agent-analytics][metabase.metabot.agent.core :as agent][metabase.metabot.agent.memory :as memory][metabase.metabot.api :as api]@@ -13,27 +11,12 @@[metabase.metabot.self.openrouter :as openrouter][metabase.metabot.test-util :as mut][metabase.metabot.tools.search :as metabot-search]- [metabase.test :as mt]- [metabase.test.util :as tu]- [metabase.util.malli :as mu]))+ [metabase.test :as mt]))(set! *warn-on-reflection* true)(def ^:private test-provider "openrouter/anthropic/claude-haiku-4-5")-;; Mock tool for testing-(mu/defn test-search-tool- "Mock search tool that returns test data."- [{:keys [_query]} :- [:map {:closed true}- [:query :string]]]- {:structured-output {:data [{:id 1 :name "Test Result"}]}})--(def test-tools- {"search" {:tool-name "search"- :schema (:schema (meta #'test-search-tool))- :doc (:doc (meta #'test-search-tool))- :fn test-search-tool}})-(defn- run-agent-loop!"run-agent-loop for side effects, discarding results.Runs as admin so the base metabot permission check passes."@@ -681,112 +664,6 @@"session_id" "00000000-0000-0000-0000-000000000001"}}]token-events)))))))))))-;;; ===================== User Intent Classification Tests =====================--(def ^:private mock-llm-response-for-intent- [{:type :start :id "msg-1"}- {:type :text :text "Done"}- {:type :usage :usage {:promptTokens 10 :completionTokens 5}- :model "test-model" :id "msg-1"}])--(deftest user-intent-snowplow-fires-event-test- (mt/with-temporary-setting-values [llm-metabot-provider test-provider- llm-metabot-internal-tasks-enabled? true- llm-anthropic-api-key "sk-ant-test"]- (testing "fires :snowplow/ai_service_event 'user_intent' when :track-user-intent? is true"- (let [rasta-id (mt/user->id :rasta)]- (with-redefs [openrouter/openrouter (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))- self/call-llm (fn [& _] [{:type :text :text "<category>query_data</category>"}])]- (mt/with-current-user rasta-id- (snowplow-test/with-fake-snowplow-collector- (run-agent-loop! {:messages [{:role :user :content "Show sales by region"}]- :state {}- :context {}- :profile-id :internal- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000002"- :track-user-intent? true}})- (tu/poll-until 5000- (some #(= "user_intent" (get-in % [:properties "data" "data" "event"]))- @snowplow-test/*snowplow-collector*))- (let [events (snowplow-test/pop-event-data-and-user-id!)- intent-events (filter #(= "user_intent" (get-in % [:data "event"])) events)]- (is (=? [{:user-id (str rasta-id)- :data {"event" "user_intent"- "source" "metabot_agent"- "profile" "internal"- "session_id" "00000000-0000-0000-0000-000000000002"- "event_details" {"intent" "query_data"}}}]- intent-events))))))))))--(deftest user-intent-not-tracked-when-disabled-test- (mt/with-temporary-setting-values [llm-metabot-provider test-provider]- (testing "does not fire user_intent event when :track-user-intent? is false"- (let [classify-called (atom false)]- (with-redefs [openrouter/openrouter- (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))-- agent-analytics/classify-and-track-user-intent-async!- (fn [_ _] (reset! classify-called true))]- (run-agent-loop! {:messages [{:role :user :content "test"}]- :state {}- :context {}- :profile-id :internal- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000003"}})- (is (false? @classify-called)))))))--(deftest user-intent-not-tracked-when-internal-tasks-disabled-test- (mt/with-temporary-setting-values [llm-metabot-provider test-provider]- (let [opts {:messages [{:role :user :content "Show sales by region"}]- :state {}- :context {}- :profile-id :internal- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000005"- :track-user-intent? true}}- classify-called (atom 0)]- (with-redefs [openrouter/openrouter- (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))-- agent-analytics/classify-and-track-user-intent-async!- (fn [_ _] (swap! classify-called inc))]- (testing "does not call classify-and-track-user-intent-async! when llm-metabot-internal-tasks-enabled? is false"- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? false- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"- llm-anthropic-api-key "sk-ant-test"]- (run-agent-loop! opts)- (is (zero? @classify-called))))- (testing "does not call classify-and-track-user-intent-async! when api key is not configured"- (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"]- (run-agent-loop! opts)- (is (zero? @classify-called)))))- (testing "calls classify-and-track-user-intent-async! when llm-metabot-internal-tasks-enabled? is true"- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"- llm-anthropic-api-key "sk-ant-test"]- (run-agent-loop! opts)- (is (= 1 @classify-called))))))))--(deftest user-intent-classifier-exception-swallowed-test- (mt/with-temporary-setting-values [llm-metabot-provider test-provider- llm-metabot-internal-tasks-enabled? true- llm-anthropic-api-key "sk-ant-test"]- (testing "does not propagate exception if classify-user-intent throws"- (let [classify-called (atom false)]- (with-redefs [openrouter/openrouter (fn [_] (mut/mock-llm-response mock-llm-response-for-intent))- self/call-llm (fn [& _]- (reset! classify-called true)- (throw (ex-info "LLM error" {})))]- ;; Should not throw — exception is caught inside the background future- (run-agent-loop! {:messages [{:role :user :content "test"}]- :state {}- :context {}- :profile-id :internal- :tracking-opts {:session-id "00000000-0000-0000-0000-000000000004"- :track-user-intent? true}})- (tu/poll-until 5000 @classify-called)- (is (true? @classify-called)))))))-(deftest chart-configs-loaded-into-charts-test(let [query (lib/native-query (mt/metadata-provider) "select 1")chart-config {:display_type "pie"diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.cljindex b01d3fce90a..16bceff3bd9 100644--- a/test/metabase/metabot/settings_test.clj+++ b/test/metabase/metabot/settings_test.clj@@ -74,8 +74,7 @@(mt/with-premium-features #{}(is (boolean? (metabot.settings/metabot-enabled?)))(is (boolean? (metabot.settings/embedded-metabot-enabled?)))- (is (string? (metabot.settings/llm-metabot-provider)))- (is (string? (metabot.settings/llm-metabot-provider-lite))))))+ (is (string? (metabot.settings/llm-metabot-provider))))))(deftest metabot-configured-with-metabase-provider-and-proxy-url-test(testing "returns true when metabase-proxied provider and proxy URL is set"@@ -162,32 +161,3 @@(is (= "metabase/anthropic/claude-sonnet-4" (metabot.settings/llm-metabot-provider))))(mt/with-temporary-setting-values [llm-metabot-provider "metabase/openrouter/anthropic/claude-haiku-4-5"](is (= "metabase/openrouter/anthropic/claude-haiku-4-5" (metabot.settings/llm-metabot-provider)))))))--;;; ------------------------------------------- Internal Tasks Tests ---------------------------------------------(deftest internal-tasks-disabled-by-default-test- (testing "returns false when the setting is not explicitly enabled"- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? nil- llm-anthropic-api-key "sk-ant-test"]- (is (false? (metabot.settings/llm-metabot-internal-tasks-enabled?))))))--(deftest internal-tasks-requires-lite-api-key-test- (testing "returns false when enabled but lite provider has no api key"- (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"]- (is (false? (metabot.settings/llm-metabot-internal-tasks-enabled?)))))))--(deftest internal-tasks-enabled-with-anthropic-test- (testing "returns true when enabled and anthropic lite provider api key is configured"- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"- llm-anthropic-api-key "sk-ant-test"]- (is (true? (metabot.settings/llm-metabot-internal-tasks-enabled?))))))--(deftest internal-tasks-enabled-with-openai-test- (testing "works with openai lite provider"- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true- llm-metabot-provider-lite "openai/gpt-4.1-mini"- llm-openai-api-key "sk-openai-test"]- (is (true? (metabot.settings/llm-metabot-internal-tasks-enabled?))))))