The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[Metabot] Remove metabot v3 token feature (#71600)
Sloan Spargeraa84a83fbaf8
diff --git a/deps.edn b/deps.ednindex a76f70e40a3..dd7f05a9bab 100644--- a/deps.edn+++ b/deps.edn@@ -330,7 +330,7 @@:test{:extra-paths ["test_config"]:exec-fn metabase.test-runner/find-and-run-tests-cli- :exec-args {:exclude-tags [:metabot-v3/e2e]}+ :exec-args {}:jvm-opts ["-Dmb.run.mode=test""-Dmacaw.run.mode=test""-Dmb.db.in.memory=true"diff --git a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj b/enterprise/backend/src/metabase_enterprise/api_routes/routes.cljindex 8a56a36129d..cb172f55607 100644--- a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj+++ b/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj@@ -57,7 +57,6 @@:remote-sync (deferred-tru "Remote Sync"):etl-connections (deferred-tru "ETL Connections"):etl-connections-pg (deferred-tru "ETL Connections PG replication")- :metabot-v3 (deferred-tru "MetaBot"):scim (deferred-tru "SCIM configuration"):semantic-search (deferred-tru "Semantic Search"):serialization (deferred-tru "Serialization")diff --git a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.cljindex 0225753f974..eb984a879e0 100644--- a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj@@ -20,7 +20,7 @@"Sets up JWT authentication and premium features for Agent API tests."[& body]`(sso.test-setup/with-jwt-default-setup!- (mt/with-additional-premium-features #{:agent-api :metabot-v3}+ (mt/with-additional-premium-features #{:agent-api}~@body)))(defn- current-epoch-seconds []diff --git a/enterprise/backend/test/metabase_enterprise/agent_api/workspace_test.clj b/enterprise/backend/test/metabase_enterprise/agent_api/workspace_test.cljindex 35332717676..16c1bae4cf5 100644--- a/enterprise/backend/test/metabase_enterprise/agent_api/workspace_test.clj+++ b/enterprise/backend/test/metabase_enterprise/agent_api/workspace_test.clj@@ -29,7 +29,7 @@"Sets up JWT auth and premium features needed for agent API + workspaces."[& body]`(sso.test-setup/with-jwt-default-setup!- (mt/with-additional-premium-features #{:agent-api :metabot-v3 :workspaces :transforms}+ (mt/with-additional-premium-features #{:agent-api :workspaces :transforms}~@body)))(defn- auth-headersdiff --git a/enterprise/backend/test/metabase_enterprise/api/session_test.clj b/enterprise/backend/test/metabase_enterprise/api/session_test.cljindex f4bd95e4326..81f85ccb46c 100644--- a/enterprise/backend/test/metabase_enterprise/api/session_test.clj+++ b/enterprise/backend/test/metabase_enterprise/api/session_test.clj@@ -28,7 +28,6 @@:embedding-simple:embedding-hub:hosting- :metabot-v3:no-upsell:official-collections:query-reference-validation@@ -74,7 +73,6 @@:embedding_sdk true:embedding_simple true:hosting true- :metabot_v3 true:official_collections true:query_reference_validation true:remote_sync truediff --git a/enterprise/backend/test/metabase_enterprise/metabot/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/api_test.cljindex 1165b279c3d..06980eb3eab 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/api_test.clj@@ -9,39 +9,38 @@(set! *warn-on-reflection* true)(deftest feedback-endpoint-test- (mt/with-premium-features #{:metabot-v3}- (let [store-url "http://hm.example"- fake-token "test-fake-token-for-feedback"]- (testing "Submits feedback to Harbormaster with token and base URL"- (mt/with-temporary-setting-values [store-api-url store-url]- (let [captured (atom nil)- feedback {:metabot_id 1- :feedback {:positive true- :message_id "m-1"- :freeform_feedback "ok"}- :conversation_data {}- :version "v0.0.0"- :submission_time "2025-01-01T00:00:00Z"- :is_admin false}- expected-url (str store-url "/api/v2/metabot/feedback/" fake-token)]- (mt/with-dynamic-fn-redefs- [premium-features/premium-embedding-token (constantly fake-token)- http/post (fn [url opts]- (reset! captured {:url url- :body (json/decode+kw (:body opts))}))]- (let [_resp (mt/user-http-request :rasta :post 204 "metabot/feedback" feedback)]- (is (= {:url expected-url :body feedback}- @captured)))))))-- (testing "Returns 500 when http post fails"- (mt/with-temporary-setting-values [store-api-url store-url]+ (let [store-url "http://hm.example"+ fake-token "test-fake-token-for-feedback"]+ (testing "Submits feedback to Harbormaster with token and base URL"+ (mt/with-temporary-setting-values [store-api-url store-url]+ (let [captured (atom nil)+ feedback {:metabot_id 1+ :feedback {:positive true+ :message_id "m-1"+ :freeform_feedback "ok"}+ :conversation_data {}+ :version "v0.0.0"+ :submission_time "2025-01-01T00:00:00Z"+ :is_admin false}+ expected-url (str store-url "/api/v2/metabot/feedback/" fake-token)](mt/with-dynamic-fn-redefs[premium-features/premium-embedding-token (constantly fake-token)- http/post (fn [_url _opts]- (throw (ex-info "boom" {:status 404})))]- (mt/user-http-request :rasta :post 500 "metabot/feedback" {:any "payload"}))))+ http/post (fn [url opts]+ (reset! captured {:url url+ :body (json/decode+kw (:body opts))}))]+ (let [_resp (mt/user-http-request :rasta :post 204 "metabot/feedback" feedback)]+ (is (= {:url expected-url :body feedback}+ @captured)))))))- (testing "Throws when premium token is missing"+ (testing "Returns 500 when http post fails"+ (mt/with-temporary-setting-values [store-api-url store-url](mt/with-dynamic-fn-redefs- [premium-features/premium-embedding-token (constantly nil)]- (mt/user-http-request :rasta :post 400 "metabot/feedback" {:foo "bar"}))))))+ [premium-features/premium-embedding-token (constantly fake-token)+ http/post (fn [_url _opts]+ (throw (ex-info "boom" {:status 404})))]+ (mt/user-http-request :rasta :post 500 "metabot/feedback" {:any "payload"}))))++ (testing "Throws when premium token is missing"+ (mt/with-dynamic-fn-redefs+ [premium-features/premium-embedding-token (constantly nil)]+ (mt/user-http-request :rasta :post 400 "metabot/feedback" {:foo "bar"})))))diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.tsindex dc4d060fea2..373d6290a57 100644--- a/frontend/src/metabase-types/api/mocks/settings.ts+++ b/frontend/src/metabase-types/api/mocks/settings.ts@@ -134,7 +134,6 @@ export const createMockTokenFeatures = (ai_entity_analysis: false,ai_sql_fixer: false,ai_sql_generation: false,- metabot_v3: false,database_routing: false,development_mode: false,table_data_editing: false,diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex f4bc72bd825..9e1457224bb 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -264,7 +264,6 @@ const tokenStatusFeatures = ["embedding-hub","hosting","metabase-store-managed",- "metabot-v3","no-upsell","official-collections","query-reference-validation",@@ -349,7 +348,6 @@ export const tokenFeatures = ["ai_sql_fixer","ai_sql_generation","ai_entity_analysis",- "metabot_v3","database_routing","development_mode","etl_connections",diff --git a/frontend/src/metabase/metabot/components/AIQuestionAnalysisButton.unit.spec.tsx b/frontend/src/metabase/metabot/components/AIQuestionAnalysisButton.unit.spec.tsxindex c87f3c47b86..409789d133d 100644--- a/frontend/src/metabase/metabot/components/AIQuestionAnalysisButton.unit.spec.tsx+++ b/frontend/src/metabase/metabot/components/AIQuestionAnalysisButton.unit.spec.tsx@@ -6,7 +6,6 @@ import { renderWithProviders, screen, waitFor } from "__support__/ui";import { mockStreamedEndpoint } from "metabase/api/ai-streaming/test-utils";import { MetabotProvider } from "metabase/metabot/context";import { getMetabotInitialState } from "metabase/metabot/state/reducer-utils";-import { createMockTokenFeatures } from "metabase-types/api/mocks";import { createMockState } from "metabase-types/store/mocks";import { AIQuestionAnalysisButton } from "./AIQuestionAnalysisButton";@@ -25,9 +24,6 @@ function setup({const settings = mockSettings({"llm-metabot-configured?": true,"metabot-enabled?": isMetabotEnabled,- "token-features": createMockTokenFeatures({- metabot_v3: true,- }),});setupEnterprisePlugins();diff --git a/frontend/src/metabase/metabot/components/MetabotAppBarButton.unit.spec.tsx b/frontend/src/metabase/metabot/components/MetabotAppBarButton.unit.spec.tsxindex d84ab79e9da..caa9a064e78 100644--- a/frontend/src/metabase/metabot/components/MetabotAppBarButton.unit.spec.tsx+++ b/frontend/src/metabase/metabot/components/MetabotAppBarButton.unit.spec.tsx@@ -3,7 +3,6 @@ import userEvent from "@testing-library/user-event";import { setupEnterprisePlugins } from "__support__/enterprise";import { mockSettings } from "__support__/settings";import { renderWithProviders, screen } from "__support__/ui";-import { createMockTokenFeatures } from "metabase-types/api/mocks";import { createMockState } from "metabase-types/store/mocks";import { MetabotProvider } from "../context";@@ -16,7 +15,6 @@ function setup({const settings = mockSettings({"llm-metabot-configured?": true,"metabot-enabled?": isMetabotEnabled,- "token-features": createMockTokenFeatures({ metabot_v3: true }),});setupEnterprisePlugins();diff --git a/frontend/src/metabase/metabot/tests/error-boundary.unit.spec.tsx b/frontend/src/metabase/metabot/tests/error-boundary.unit.spec.tsxindex 8fc1729e2e6..8bb03214589 100644--- a/frontend/src/metabase/metabot/tests/error-boundary.unit.spec.tsx+++ b/frontend/src/metabase/metabot/tests/error-boundary.unit.spec.tsx@@ -5,10 +5,7 @@ import { assocIn } from "icepick";import { setupEnterprisePlugins } from "__support__/enterprise";import { mockSettings } from "__support__/settings";import { renderWithProviders, screen } from "__support__/ui";-import {- createMockTokenFeatures,- createMockUser,-} from "metabase-types/api/mocks";+import { createMockUser } from "metabase-types/api/mocks";import { createMockState } from "metabase-types/store/mocks";import { Metabot } from "../components/Metabot";@@ -34,7 +31,6 @@ jest.mock("../components/MetabotChat", () => {function setup() {const settings = mockSettings({"llm-metabot-configured?": true,- "token-features": createMockTokenFeatures({ metabot_v3: true }),});setupEnterprisePlugins();diff --git a/frontend/src/metabase/metabot/tests/utils.tsx b/frontend/src/metabase/metabot/tests/utils.tsxindex 792c6d747bd..b0f06c3c961 100644--- a/frontend/src/metabase/metabot/tests/utils.tsx+++ b/frontend/src/metabase/metabot/tests/utils.tsx@@ -20,10 +20,7 @@ import {mockStreamedEndpoint,} from "metabase/api/ai-streaming/test-utils";import type { User } from "metabase-types/api";-import {- createMockTokenFeatures,- createMockUser,-} from "metabase-types/api/mocks";+import { createMockUser } from "metabase-types/api/mocks";import type { State } from "metabase-types/store";import { createMockState } from "metabase-types/store/mocks";@@ -151,9 +148,6 @@ export function setup(const settings = mockSettings({"llm-metabot-configured?": true,"is-hosted?": options?.isHosted ?? false,- "token-features": createMockTokenFeatures({- metabot_v3: true,- }),});setupEnterprisePlugins();diff --git a/frontend/src/metabase/rich_text_editing/tiptap/extensions/Command/CommandSuggestion.unit.spec.tsx b/frontend/src/metabase/rich_text_editing/tiptap/extensions/Command/CommandSuggestion.unit.spec.tsxindex 9f3ef8f3ff3..6dc657ef178 100644--- a/frontend/src/metabase/rich_text_editing/tiptap/extensions/Command/CommandSuggestion.unit.spec.tsx+++ b/frontend/src/metabase/rich_text_editing/tiptap/extensions/Command/CommandSuggestion.unit.spec.tsx@@ -20,7 +20,6 @@ import {createMockRecentCollectionItem,createMockRecentTableItem,createMockSearchResult,- createMockTokenFeatures,createMockUser,createMockUserPermissions,} from "metabase-types/api/mocks";@@ -454,9 +453,7 @@ describe("CommandSuggestion", () => {describe("when metabot is enabled", () => {beforeEach(() => {- mockSettings({- "token-features": createMockTokenFeatures({ metabot_v3: true }),- });+ mockSettings({});setupEnterprisePlugins();});diff --git a/frontend/src/metabase/rich_text_editing/tiptap/extensions/MetabotEmbed/MetabotEmbed.unit.spec.tsx b/frontend/src/metabase/rich_text_editing/tiptap/extensions/MetabotEmbed/MetabotEmbed.unit.spec.tsxindex 6011b79c6f7..37353443bfa 100644--- a/frontend/src/metabase/rich_text_editing/tiptap/extensions/MetabotEmbed/MetabotEmbed.unit.spec.tsx+++ b/frontend/src/metabase/rich_text_editing/tiptap/extensions/MetabotEmbed/MetabotEmbed.unit.spec.tsx@@ -3,7 +3,6 @@ import userEvent from "@testing-library/user-event";import { setupEnterprisePlugins } from "__support__/enterprise";import { mockSettings } from "__support__/settings";import { renderWithProviders, screen, within } from "__support__/ui";-import { createMockTokenFeatures } from "metabase-types/api/mocks";import { createMockState } from "metabase-types/store/mocks";import { MetabotComponent } from "./MetabotEmbed";@@ -28,9 +27,7 @@ describe("MetabotEmbed", () => {beforeEach(() => {jest.clearAllMocks();- mockSettings({- "token-features": createMockTokenFeatures({ metabot_v3: true }),- });+ mockSettings({});setupEnterprisePlugins();});diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.cljindex 6cc5e1c00a3..cf16f5ff920 100644--- a/src/metabase/analytics/stats.clj+++ b/src/metabase/analytics/stats.clj@@ -905,9 +905,6 @@{:name :cache-preemptive:available (premium-features/enable-preemptive-caching?):enabled (t2/exists? :model/CacheConfig :refresh_automatically true)}- {:name :metabot-v3- :available (premium-features/enable-metabot-v3?)- :enabled (premium-features/enable-metabot-v3?)}{:name :remote-sync:available (premium-features/enable-remote-sync?):enabled (premium-features/enable-remote-sync?)}diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.cljindex be91a665069..2345afd5e23 100644--- a/src/metabase/premium_features/core.clj+++ b/src/metabase/premium_features/core.clj@@ -58,7 +58,6 @@enable-email-restrict-recipients?enable-embedding-sdk-origins?enable-embedding-simple-feature?- enable-metabot-v3?enable-official-collections?enable-preemptive-caching?enable-query-reference-validation?diff --git a/src/metabase/premium_features/settings.clj b/src/metabase/premium_features/settings.cljindex 1b11909dbf9..e3c8478018e 100644--- a/src/metabase/premium_features/settings.clj+++ b/src/metabase/premium_features/settings.clj@@ -239,10 +239,6 @@"Is this a development instance that should have watermarks?":development-mode)-(define-premium-feature ^{:added "0.52.0"} enable-metabot-v3?- "Enable the newest LLM-based MetaBot? (The one that lives in [[metabase.metabot.core]].)"- :metabot-v3)-; the "-feature" suffix avoids name collision with the setting getter(define-premium-feature ^{:added "0.55.0"} enable-embedding-simple-feature?"Should we enable modular embedding?"@@ -334,7 +330,6 @@:etl_connections (enable-etl-connections?):etl_connections_pg (enable-etl-connections-pg?):hosting (is-hosted?)- :metabot_v3 (enable-metabot-v3?):official_collections (enable-official-collections?):query_reference_validation (enable-query-reference-validation?):remote_sync (enable-remote-sync?)diff --git a/test/metabase/mcp/api_test.clj b/test/metabase/mcp/api_test.cljindex a413fa6393b..346e07d97dd 100644--- a/test/metabase/mcp/api_test.clj+++ b/test/metabase/mcp/api_test.clj@@ -407,18 +407,17 @@(deftest invalid-bearer-token-returns-401-test(testing "POST with invalid bearer token returns 401 with invalid_token error"- (mt/with-premium-features #{:metabot-v3}- (mt/with-temporary-setting-values [site-url "http://localhost:3000"]- (oauth-server/reset-provider!)- (try- (let [response (mcp-request-with-bearer "totally-bogus-token" 401- (jsonrpc-request "initialize")- {})]- (is (=? {:status 401- :headers {"WWW-Authenticate" #(str/includes? % "invalid_token")}}- response)))- (finally- (oauth-server/reset-provider!)))))))+ (mt/with-temporary-setting-values [site-url "http://localhost:3000"]+ (oauth-server/reset-provider!)+ (try+ (let [response (mcp-request-with-bearer "totally-bogus-token" 401+ (jsonrpc-request "initialize")+ {})]+ (is (=? {:status 401+ :headers {"WWW-Authenticate" #(str/includes? % "invalid_token")}}+ response)))+ (finally+ (oauth-server/reset-provider!))))));;; --------------------------------------------- Scope Filtering ---------------------------------------------------@@ -465,18 +464,17 @@(deftest expired-oauth-bearer-token-returns-401-test(testing "POST with expired OAuth bearer token returns 401"- (mt/with-premium-features #{:metabot-v3}- (mt/with-temporary-setting-values [site-url "http://localhost:3000"]- (t2/with-transaction [_conn nil {:rollback-only true}]- (oauth-server/reset-provider!)- (let [user-id (mt/user->id :crowberto)- token (insert-expired-oauth-token! user-id (str (random-uuid)))- response (mcp-request-with-bearer token 401- (jsonrpc-request "initialize")- {})]- (is (=? {:status 401- :headers {"WWW-Authenticate" #(str/includes? % "invalid_token")}}- response))))))))+ (mt/with-temporary-setting-values [site-url "http://localhost:3000"]+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (oauth-server/reset-provider!)+ (let [user-id (mt/user->id :crowberto)+ token (insert-expired-oauth-token! user-id (str (random-uuid)))+ response (mcp-request-with-bearer token 401+ (jsonrpc-request "initialize")+ {})]+ (is (=? {:status 401+ :headers {"WWW-Authenticate" #(str/includes? % "invalid_token")}}+ response)))))))(deftest tools-call-scope-enforcement-test(testing "tool call is rejected when token scopes don't include the required scope"diff --git a/test/metabase/metabot/api/document_test.clj b/test/metabase/metabot/api/document_test.cljindex f0f98432ec6..e9e6c8ae3ae 100644--- a/test/metabase/metabot/api/document_test.clj+++ b/test/metabase/metabot/api/document_test.clj@@ -15,70 +15,67 @@(deftest generate-content-backwards-compatible-route-test(mt/with-temporary-setting-values [llm-metabot-provider test-provider]- (mt/with-premium-features #{:metabot-v3}+ (with-redefs [openrouter/openrouter+ (fn [_]+ (mut/mock-llm-response+ [{:type :start :id "msg-1"}+ {:type :text :text "No chart available"}+ {:type :usage :usage {:promptTokens 100 :completionTokens 20}+ :model "test-model" :id "msg-1"}]))]+ (is (= {:draft_card nil+ :description nil+ :error "No chart available"}+ (mt/user-http-request :crowberto+ :post 200 "metabot/document/generate-content"+ {:instructions "Show me sales data"}))))))++(deftest generate-content-prometheus-test+ (mt/with-temporary-setting-values [llm-metabot-provider test-provider]+ (mt/with-prometheus-system! [_ system](with-redefs [openrouter/openrouter(fn [_](mut/mock-llm-response[{:type :start :id "msg-1"}{:type :text :text "No chart available"}{:type :usage :usage {:promptTokens 100 :completionTokens 20}- :model "test-model" :id "msg-1"}]))]- (is (= {:draft_card nil- :description nil- :error "No chart available"}- (mt/user-http-request :crowberto- :post 200 "metabot/document/generate-content"- {:instructions "Show me sales data"})))))))--(deftest generate-content-prometheus-test- (mt/with-temporary-setting-values [llm-metabot-provider test-provider]- (mt/with-premium-features #{:metabot-v3}- (mt/with-prometheus-system! [_ system]- (with-redefs [openrouter/openrouter- (fn [_]- (mut/mock-llm-response- [{:type :start :id "msg-1"}- {:type :text :text "No chart available"}- {:type :usage :usage {:promptTokens 100 :completionTokens 20}- :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))]- (mt/user-http-request :crowberto- :post 200 "metabot/document/generate-content"- {:instructions "Show me sales data"}))- (is (== 1 (mt/metric-value system :metabase-metabot/agent-requests- {:profile-id "document-generate-content"})))- (is (== 1 (:sum (mt/metric-value system :metabase-metabot/agent-iterations- {:profile-id "document-generate-content"}))))- (is (== 1 (mt/metric-value system :metabase-metabot/llm-requests+ :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))]+ (mt/user-http-request :crowberto+ :post 200 "metabot/document/generate-content"+ {:instructions "Show me sales data"}))+ (is (== 1 (mt/metric-value system :metabase-metabot/agent-requests+ {:profile-id "document-generate-content"})))+ (is (== 1 (:sum (mt/metric-value system :metabase-metabot/agent-iterations+ {:profile-id "document-generate-content"}))))+ (is (== 1 (mt/metric-value system :metabase-metabot/llm-requests+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))+ (is (== 100 (mt/metric-value system :metabase-metabot/llm-input-tokens{:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))- (is (== 100 (mt/metric-value system :metabase-metabot/llm-input-tokens- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))- (is (== 20 (mt/metric-value system :metabase-metabot/llm-output-tokens- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))))))+ (is (== 20 (mt/metric-value system :metabase-metabot/llm-output-tokens+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"}))))))(deftest generate-content-snowplow-test(mt/with-temporary-setting-values [llm-metabot-provider test-provider]- (mt/with-premium-features #{:metabot-v3}- (let [rasta-id (mt/user->id :rasta)]- (with-redefs [openrouter/openrouter- (fn [_]- (mut/mock-llm-response- [{:type :start :id "msg-1"}- {:type :text :text "No chart available"}- {:type :usage :usage {:promptTokens 100 :completionTokens 20}- :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))]- (snowplow-test/with-fake-snowplow-collector- (mt/user-http-request :rasta- :post 200 "metabot/document/generate-content"- {:instructions "Show me sales data"})- (let [events (snowplow-test/pop-event-data-and-user-id!)- token-events (filter #(contains? (:data %) "total_tokens") events)]- (is (=? [{:user-id (str rasta-id)- :data {"model_id" "openrouter/anthropic/claude-haiku-4-5"- "total_tokens" 120- "prompt_tokens" 100- "completion_tokens" 20- "estimated_costs_usd" 0.0- "duration_ms" nat-int?- "source" "document_generate_content"- "tag" "agent"}}]- token-events)))))))))+ (let [rasta-id (mt/user->id :rasta)]+ (with-redefs [openrouter/openrouter+ (fn [_]+ (mut/mock-llm-response+ [{:type :start :id "msg-1"}+ {:type :text :text "No chart available"}+ {:type :usage :usage {:promptTokens 100 :completionTokens 20}+ :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))]+ (snowplow-test/with-fake-snowplow-collector+ (mt/user-http-request :rasta+ :post 200 "metabot/document/generate-content"+ {:instructions "Show me sales data"})+ (let [events (snowplow-test/pop-event-data-and-user-id!)+ token-events (filter #(contains? (:data %) "total_tokens") events)]+ (is (=? [{:user-id (str rasta-id)+ :data {"model_id" "openrouter/anthropic/claude-haiku-4-5"+ "total_tokens" 120+ "prompt_tokens" 100+ "completion_tokens" 20+ "estimated_costs_usd" 0.0+ "duration_ms" nat-int?+ "source" "document_generate_content"+ "tag" "agent"}}]+ token-events))))))))diff --git a/test/metabase/metabot/api/metabot_test.clj b/test/metabase/metabot/api/metabot_test.cljindex e39f1aaed22..a1ee6481c39 100644--- a/test/metabase/metabot/api/metabot_test.clj+++ b/test/metabase/metabot/api/metabot_test.clj@@ -40,175 +40,172 @@model-data {:description "Model desc":dataset_query (lib/->legacy-MBQL model-source-query):type :model}]- (mt/with-premium-features #{:metabot-v3}- (mt/with-temp [:model/Collection {coll-id :id} {}- :model/Collection {child-id1 :id} {:location (collection/location-path coll-id)}- :model/Collection {child-id2 :id} {:location (collection/location-path coll-id)}- :model/Collection {child-id3 :id} {:location (collection/location-path coll-id child-id2)}- :model/Card _ (assoc model-data :name "Model1" :collection_id coll-id)- :model/Card _ (assoc metric-data :name "Metric1" :collection_id child-id1)- :model/Card _ (assoc model-data :name "Model2" :collection_id child-id2)- :model/Card _ (assoc metric-data :name "Metric2" :collection_id child-id3)- :model/Metabot {metabot-id :id} {:name "metabot" :collection_id coll-id}]- (let [prompts {"Metric1" ["metric question1" "metric question2" "metric question3"]- "Metric2" ["metric question4" "metric question5" "metric question6"]- "Model1" ["model question1" "model question2" "model question3"]- "Model2" ["model question4" "model question5" "model question6"]}- generate-prompt (fn [promptables]- (map (fn [promptable]- {:questions (-> promptable :name prompts)})- promptables))- prompt-generator (fn [request]- (-> request- (update :metrics generate-prompt)- (update :tables generate-prompt)- (set/rename-keys {:metrics :metric_questions- :tables :table_questions})))]- ;; --------------------------- Generating sample prompts ---------------------------- (testing "should generate prompt suggestions for metabot"- (with-redefs [metabot.example-question-generator/generate-example-questions prompt-generator]- ;; Trigger prompt generation by calling the regenerate endpoint- (mt/user-http-request :crowberto :post 204- (format "metabot/metabot/%d/prompt-suggestions/regenerate" metabot-id)))- (let [added-prompts (t2/select [:model/MetabotPrompt [:card.name :model_name] :prompt]- :metabot_id metabot-id- {:join [[:report_card :card] [:= :card.id :card_id]]- :order-by [:metabot_prompt.id]})]- ;; Verify prompts were added to the database- (is (= prompts- (-> (group-by :model_name added-prompts)- (update-vals #(map :prompt %)))))))- ;; --------------------------- Querying sample prompts ---------------------------- (let [expected-prompts (into #{} (mapcat val) prompts)- all-prompts (mt/user-http-request :rasta :get 200- (format "metabot/metabot/%d/prompt-suggestions" metabot-id))]- (testing "can get all prompts"- (is (= (count expected-prompts) (:total all-prompts)))- (is (= expected-prompts (into #{} (map :prompt) (:prompts all-prompts)))))- (testing "can page through the prompts"- (let [offset 3- limit 5- url (format "metabot/metabot/%d/prompt-suggestions?offset=%d&limit=%d"- metabot-id offset limit)- page-prompts (mt/user-http-request :rasta :get 200 url)]- (is (= {:prompts (->> all-prompts :prompts (drop offset) (take limit))- :limit limit, :offset offset, :total (:total all-prompts)}- page-prompts))))- (testing "can filter by model type"- (doseq [model-type ["metric" "model"]]- (let [url (format "metabot/metabot/%d/prompt-suggestions?model=%s" metabot-id model-type)- expected-prompts (->> all-prompts :prompts (filter (comp #{model-type} :model)))- filtered-prompts (mt/user-http-request :rasta :get 200 url)]- (is (= {:prompts expected-prompts- :limit nil, :offset nil, :total (count expected-prompts)}- filtered-prompts)))))- (testing "can filter by model type and model ID"- (let [selected-prompt (rand-nth (:prompts all-prompts))- url (format "metabot/metabot/%d/prompt-suggestions?model=%s&model_id=%d"- metabot-id (:model selected-prompt) (:model_id selected-prompt))- expected-prompts (->> all-prompts :prompts (filter (comp #{(:model_id selected-prompt)} :model_id)))+ (mt/with-temp [:model/Collection {coll-id :id} {}+ :model/Collection {child-id1 :id} {:location (collection/location-path coll-id)}+ :model/Collection {child-id2 :id} {:location (collection/location-path coll-id)}+ :model/Collection {child-id3 :id} {:location (collection/location-path coll-id child-id2)}+ :model/Card _ (assoc model-data :name "Model1" :collection_id coll-id)+ :model/Card _ (assoc metric-data :name "Metric1" :collection_id child-id1)+ :model/Card _ (assoc model-data :name "Model2" :collection_id child-id2)+ :model/Card _ (assoc metric-data :name "Metric2" :collection_id child-id3)+ :model/Metabot {metabot-id :id} {:name "metabot" :collection_id coll-id}]+ (let [prompts {"Metric1" ["metric question1" "metric question2" "metric question3"]+ "Metric2" ["metric question4" "metric question5" "metric question6"]+ "Model1" ["model question1" "model question2" "model question3"]+ "Model2" ["model question4" "model question5" "model question6"]}+ generate-prompt (fn [promptables]+ (map (fn [promptable]+ {:questions (-> promptable :name prompts)})+ promptables))+ prompt-generator (fn [request]+ (-> request+ (update :metrics generate-prompt)+ (update :tables generate-prompt)+ (set/rename-keys {:metrics :metric_questions+ :tables :table_questions})))]+ ;; --------------------------- Generating sample prompts ---------------------------+ (testing "should generate prompt suggestions for metabot"+ (with-redefs [metabot.example-question-generator/generate-example-questions prompt-generator]+ ;; Trigger prompt generation by calling the regenerate endpoint+ (mt/user-http-request :crowberto :post 204+ (format "metabot/metabot/%d/prompt-suggestions/regenerate" metabot-id)))+ (let [added-prompts (t2/select [:model/MetabotPrompt [:card.name :model_name] :prompt]+ :metabot_id metabot-id+ {:join [[:report_card :card] [:= :card.id :card_id]]+ :order-by [:metabot_prompt.id]})]+ ;; Verify prompts were added to the database+ (is (= prompts+ (-> (group-by :model_name added-prompts)+ (update-vals #(map :prompt %)))))))+ ;; --------------------------- Querying sample prompts ---------------------------+ (let [expected-prompts (into #{} (mapcat val) prompts)+ all-prompts (mt/user-http-request :rasta :get 200+ (format "metabot/metabot/%d/prompt-suggestions" metabot-id))]+ (testing "can get all prompts"+ (is (= (count expected-prompts) (:total all-prompts)))+ (is (= expected-prompts (into #{} (map :prompt) (:prompts all-prompts)))))+ (testing "can page through the prompts"+ (let [offset 3+ limit 5+ url (format "metabot/metabot/%d/prompt-suggestions?offset=%d&limit=%d"+ metabot-id offset limit)+ page-prompts (mt/user-http-request :rasta :get 200 url)]+ (is (= {:prompts (->> all-prompts :prompts (drop offset) (take limit))+ :limit limit, :offset offset, :total (:total all-prompts)}+ page-prompts))))+ (testing "can filter by model type"+ (doseq [model-type ["metric" "model"]]+ (let [url (format "metabot/metabot/%d/prompt-suggestions?model=%s" metabot-id model-type)+ expected-prompts (->> all-prompts :prompts (filter (comp #{model-type} :model)))filtered-prompts (mt/user-http-request :rasta :get 200 url)](is (= {:prompts expected-prompts:limit nil, :offset nil, :total (count expected-prompts)}- filtered-prompts))))- (testing "can return a sample"- (let [limit 5- url (format "metabot/metabot/%d/prompt-suggestions?sample=true&limit=%d"- metabot-id limit)- expected-prompts? (fn [prompts]- (and (= (count prompts) limit)- (set/subset? (set prompts) (set (:prompts all-prompts)))))- sample-prompts (mt/user-http-request :rasta :get 200 url)]- (is (=? {:prompts expected-prompts?- :limit limit, :offset nil, :total (:total all-prompts)}- sample-prompts))))- ;; --------------------------- Deleting & regenerating sample prompts ---------------------------- (let [all-prompt-ids (into #{} (map :id) (:prompts all-prompts))- current-prompt-ids #(t2/select-pks-set :model/MetabotPrompt :metabot_id metabot-id)- selected-prompt (rand-nth (:prompts all-prompts))- url (format "metabot/metabot/%d/prompt-suggestions/%d" metabot-id (:id selected-prompt))- remaining-prompt-ids (disj all-prompt-ids (:id selected-prompt))]-- (testing "deleting a specific prompt"- (testing "normal users cannot delete"- (mt/user-http-request :rasta :delete 403 url)- (is (= all-prompt-ids (current-prompt-ids))))- (testing "admins can delete"- (mt/user-http-request :crowberto :delete 204 url)- (is (= remaining-prompt-ids (current-prompt-ids)))))-- (testing "generating new prompts"- (let [url (format "metabot/metabot/%d/prompt-suggestions/regenerate" metabot-id)]- (testing "normal users are not allowed"- (mt/user-http-request :rasta :post 403 url)- (is (= remaining-prompt-ids (current-prompt-ids))))- (testing "admin users are allowed"- (with-redefs [metabot.example-question-generator/generate-example-questions prompt-generator]- (mt/user-http-request :crowberto :post 204 url)))))-- (let [new-prompt-ids (current-prompt-ids)]- (is (= (count all-prompt-ids) (count new-prompt-ids)))- (is (empty? (set/intersection all-prompt-ids new-prompt-ids)))- (testing "can delete all prompts"- (let [url (format "metabot/metabot/%d/prompt-suggestions" metabot-id)]- (testing "normal users cannot delete"- (mt/user-http-request :rasta :delete 403 url)- (is (= new-prompt-ids (current-prompt-ids))))- (testing "admins can delete"- (mt/user-http-request :crowberto :delete 204 url)- (is (nil? (current-prompt-ids)))))))))))))))+ filtered-prompts)))))+ (testing "can filter by model type and model ID"+ (let [selected-prompt (rand-nth (:prompts all-prompts))+ url (format "metabot/metabot/%d/prompt-suggestions?model=%s&model_id=%d"+ metabot-id (:model selected-prompt) (:model_id selected-prompt))+ expected-prompts (->> all-prompts :prompts (filter (comp #{(:model_id selected-prompt)} :model_id)))+ filtered-prompts (mt/user-http-request :rasta :get 200 url)]+ (is (= {:prompts expected-prompts+ :limit nil, :offset nil, :total (count expected-prompts)}+ filtered-prompts))))+ (testing "can return a sample"+ (let [limit 5+ url (format "metabot/metabot/%d/prompt-suggestions?sample=true&limit=%d"+ metabot-id limit)+ expected-prompts? (fn [prompts]+ (and (= (count prompts) limit)+ (set/subset? (set prompts) (set (:prompts all-prompts)))))+ sample-prompts (mt/user-http-request :rasta :get 200 url)]+ (is (=? {:prompts expected-prompts?+ :limit limit, :offset nil, :total (:total all-prompts)}+ sample-prompts))))+ ;; --------------------------- Deleting & regenerating sample prompts ---------------------------+ (let [all-prompt-ids (into #{} (map :id) (:prompts all-prompts))+ current-prompt-ids #(t2/select-pks-set :model/MetabotPrompt :metabot_id metabot-id)+ selected-prompt (rand-nth (:prompts all-prompts))+ url (format "metabot/metabot/%d/prompt-suggestions/%d" metabot-id (:id selected-prompt))+ remaining-prompt-ids (disj all-prompt-ids (:id selected-prompt))]++ (testing "deleting a specific prompt"+ (testing "normal users cannot delete"+ (mt/user-http-request :rasta :delete 403 url)+ (is (= all-prompt-ids (current-prompt-ids))))+ (testing "admins can delete"+ (mt/user-http-request :crowberto :delete 204 url)+ (is (= remaining-prompt-ids (current-prompt-ids)))))++ (testing "generating new prompts"+ (let [url (format "metabot/metabot/%d/prompt-suggestions/regenerate" metabot-id)]+ (testing "normal users are not allowed"+ (mt/user-http-request :rasta :post 403 url)+ (is (= remaining-prompt-ids (current-prompt-ids))))+ (testing "admin users are allowed"+ (with-redefs [metabot.example-question-generator/generate-example-questions prompt-generator]+ (mt/user-http-request :crowberto :post 204 url)))))++ (let [new-prompt-ids (current-prompt-ids)]+ (is (= (count all-prompt-ids) (count new-prompt-ids)))+ (is (empty? (set/intersection all-prompt-ids new-prompt-ids)))+ (testing "can delete all prompts"+ (let [url (format "metabot/metabot/%d/prompt-suggestions" metabot-id)]+ (testing "normal users cannot delete"+ (mt/user-http-request :rasta :delete 403 url)+ (is (= new-prompt-ids (current-prompt-ids))))+ (testing "admins can delete"+ (mt/user-http-request :crowberto :delete 204 url)+ (is (nil? (current-prompt-ids))))))))))))))(deftest metabot-list-test(testing "GET /api/metabot/metabot"- (mt/with-premium-features #{:metabot-v3}- (with-clean-metabots- (mt/with-temp [:model/Metabot {metabot-id-1 :id} {:name "Alpha Metabot"}- :model/Metabot {metabot-id-2 :id} {:name "Beta Metabot"}- :model/Metabot {metabot-id-3 :id} {:name "Gamma Metabot"}]-- (testing "should return all metabots in alphabetical order by name"- (let [{response :items} (mt/user-http-request :crowberto :get 200 "metabot/metabot")]- (is (= 3 (count response)))- (is (= ["Alpha Metabot" "Beta Metabot" "Gamma Metabot"]- (mapv :name response)))- (is (= [metabot-id-1 metabot-id-2 metabot-id-3]- (mapv :id response)))))-- (testing "should require superuser permissions"- (is (= "You don't have permissions to do that."- (mt/user-http-request :rasta :get 403 "metabot/metabot")))))))))--(deftest metabot-get-single-test- (testing "GET /api/metabot/metabot/:id"- (mt/with-premium-features #{:metabot-v3}- (mt/with-temp [:model/Collection {collection-id :id} {:name "Test Collection"}- :model/Metabot {metabot-id :id} {:name "Test Metabot"- :description "Test Description"- :use_verified_content true- :collection_id collection-id}]-- (testing "should return metabot with all fields"- (let [response (mt/user-http-request :crowberto :get 200- (format "metabot/metabot/%d" metabot-id))]- (is (= metabot-id (:id response)))- (is (= "Test Metabot" (:name response)))- (is (= "Test Description" (:description response)))- (is (true? (:use_verified_content response)))- (is (= collection-id (:collection_id response)))))+ (with-clean-metabots+ (mt/with-temp [:model/Metabot {metabot-id-1 :id} {:name "Alpha Metabot"}+ :model/Metabot {metabot-id-2 :id} {:name "Beta Metabot"}+ :model/Metabot {metabot-id-3 :id} {:name "Gamma Metabot"}]++ (testing "should return all metabots in alphabetical order by name"+ (let [{response :items} (mt/user-http-request :crowberto :get 200 "metabot/metabot")]+ (is (= 3 (count response)))+ (is (= ["Alpha Metabot" "Beta Metabot" "Gamma Metabot"]+ (mapv :name response)))+ (is (= [metabot-id-1 metabot-id-2 metabot-id-3]+ (mapv :id response)))))(testing "should require superuser permissions"(is (= "You don't have permissions to do that."- (mt/user-http-request :rasta :get 403- (format "metabot/metabot/%d" metabot-id)))))+ (mt/user-http-request :rasta :get 403 "metabot/metabot"))))))))- (testing "should return 404 for non-existent metabot"- (is (= "Not found."- (mt/user-http-request :crowberto :get 404- (format "metabot/metabot/%d" Integer/MAX_VALUE)))))))))+(deftest metabot-get-single-test+ (testing "GET /api/metabot/metabot/:id"+ (mt/with-temp [:model/Collection {collection-id :id} {:name "Test Collection"}+ :model/Metabot {metabot-id :id} {:name "Test Metabot"+ :description "Test Description"+ :use_verified_content true+ :collection_id collection-id}]++ (testing "should return metabot with all fields"+ (let [response (mt/user-http-request :crowberto :get 200+ (format "metabot/metabot/%d" metabot-id))]+ (is (= metabot-id (:id response)))+ (is (= "Test Metabot" (:name response)))+ (is (= "Test Description" (:description response)))+ (is (true? (:use_verified_content response)))+ (is (= collection-id (:collection_id response)))))++ (testing "should require superuser permissions"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :get 403+ (format "metabot/metabot/%d" metabot-id)))))++ (testing "should return 404 for non-existent metabot"+ (is (= "Not found."+ (mt/user-http-request :crowberto :get 404+ (format "metabot/metabot/%d" Integer/MAX_VALUE))))))))(deftest metabot-put-test(testing "PUT /api/metabot/metabot/:id"- (mt/with-premium-features #{:metabot-v3 :content-verification}+ (mt/with-premium-features #{:content-verification}(mt/with-temp [:model/Collection {collection-id-1 :id} {:name "Collection 1"}:model/Collection {collection-id-2 :id} {:name "Collection 2"}:model/Metabot {metabot-id :id} {:name "Test Metabot"@@ -269,7 +266,7 @@{:use_verified_content true}))))(testing "should prevent enabling verified content without premium feature"- (mt/with-premium-features #{:metabot-v3} ; Only metabot-v3, no content-verification+ (mt/with-premium-features #{} ; no content-verification(is (= "Content verification is a paid feature not currently available to your instance. Please upgrade to use it. Learn more at metabase.com/upgrade/"(:message (mt/user-http-request :crowberto :put 402(format "metabot/metabot/%d" metabot-id)@@ -278,7 +275,7 @@(deftest metabot-prompt-regeneration-on-config-change-test(mt/dataset test-data(testing "PUT /api/metabot/metabot/:id should regenerate prompts when config changes"- (mt/with-premium-features #{:metabot-v3 :content-verification}+ (mt/with-premium-features #{:content-verification}(let [metric-query {:type :query, :database (mt/id), :query {:source-table (mt/id :products)}}model-query {:type :query, :database (mt/id), :query {:source-table (mt/id :products)}}](mt/with-temp [:model/Collection {collection-id-1 :id} {:name "Collection 1"}@@ -374,43 +371,42 @@(deftest prompt-suggestions-collection-permissions-test(testing "GET /api/metabot/metabot/:id/prompt-suggestions respects collection permissions"(mt/dataset test-data- (mt/with-premium-features #{:metabot-v3}- (let [mp (mt/metadata-provider)- model-query (lib/query mp (lib.metadata/table mp (mt/id :products)))]- (mt/with-temp [:model/Collection {accessible-coll-id :id} {:name "Accessible Collection"}- :model/Collection {restricted-coll-id :id} {:name "Restricted Collection"}- :model/Card {accessible-card-id :id} {:name "Accessible Model"- :type :model- :collection_id accessible-coll-id- :dataset_query model-query}- :model/Card {restricted-card-id :id} {:name "Restricted Model"- :type :model- :collection_id restricted-coll-id- :dataset_query model-query}- :model/Metabot {metabot-id :id} {:name "Test Metabot"}- :model/MetabotPrompt _ {:metabot_id metabot-id- :prompt "Accessible prompt"- :model :model- :card_id accessible-card-id}- :model/MetabotPrompt _ {:metabot_id metabot-id- :prompt "Restricted prompt"- :model :model- :card_id restricted-card-id}]- ;; Revoke default All Users access to restricted collection- (perms/revoke-collection-permissions! (perms-group/all-users) restricted-coll-id)-- (testing "admin sees all prompts"- (let [response (mt/user-http-request :crowberto :get 200- (format "metabot/metabot/%d/prompt-suggestions" metabot-id))- prompts (set (map :prompt (:prompts response)))]- (is (= 2 (:total response)))- (is (contains? prompts "Accessible prompt"))- (is (contains? prompts "Restricted prompt"))))-- (testing "non-admin user only sees prompts for cards in accessible collections"- (let [response (mt/user-http-request :rasta :get 200- (format "metabot/metabot/%d/prompt-suggestions" metabot-id))- prompts (set (map :prompt (:prompts response)))]- (is (= 1 (:total response)))- (is (contains? prompts "Accessible prompt"))- (is (not (contains? prompts "Restricted prompt")))))))))))+ (let [mp (mt/metadata-provider)+ model-query (lib/query mp (lib.metadata/table mp (mt/id :products)))]+ (mt/with-temp [:model/Collection {accessible-coll-id :id} {:name "Accessible Collection"}+ :model/Collection {restricted-coll-id :id} {:name "Restricted Collection"}+ :model/Card {accessible-card-id :id} {:name "Accessible Model"+ :type :model+ :collection_id accessible-coll-id+ :dataset_query model-query}+ :model/Card {restricted-card-id :id} {:name "Restricted Model"+ :type :model+ :collection_id restricted-coll-id+ :dataset_query model-query}+ :model/Metabot {metabot-id :id} {:name "Test Metabot"}+ :model/MetabotPrompt _ {:metabot_id metabot-id+ :prompt "Accessible prompt"+ :model :model+ :card_id accessible-card-id}+ :model/MetabotPrompt _ {:metabot_id metabot-id+ :prompt "Restricted prompt"+ :model :model+ :card_id restricted-card-id}]+ ;; Revoke default All Users access to restricted collection+ (perms/revoke-collection-permissions! (perms-group/all-users) restricted-coll-id)++ (testing "admin sees all prompts"+ (let [response (mt/user-http-request :crowberto :get 200+ (format "metabot/metabot/%d/prompt-suggestions" metabot-id))+ prompts (set (map :prompt (:prompts response)))]+ (is (= 2 (:total response)))+ (is (contains? prompts "Accessible prompt"))+ (is (contains? prompts "Restricted prompt"))))++ (testing "non-admin user only sees prompts for cards in accessible collections"+ (let [response (mt/user-http-request :rasta :get 200+ (format "metabot/metabot/%d/prompt-suggestions" metabot-id))+ prompts (set (map :prompt (:prompts response)))]+ (is (= 1 (:total response)))+ (is (contains? prompts "Accessible prompt"))+ (is (not (contains? prompts "Restricted prompt"))))))))))diff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex 891fc27fd31..1fa371ef6dc 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -362,84 +362,82 @@:model "claude-haiku-4-5"}))(deftest endpoints-require-authentication-test- (mt/with-premium-features #{:metabot-v3}- (testing "Metabot v3 endpoints require authentication"- (testing "/agent-streaming"- (is (= "Unauthenticated"- (mt/client :post 401 "metabot/agent-streaming"- {:message "Test"- :context {}- :conversation_id (str (random-uuid))- :history []- :state {}}))))- (testing "/feedback"- (is (= "Unauthenticated"- (mt/client :post 401 "metabot/feedback"- {:feedback {}})))))))+ (testing "Metabot v3 endpoints require authentication"+ (testing "/agent-streaming"+ (is (= "Unauthenticated"+ (mt/client :post 401 "metabot/agent-streaming"+ {:message "Test"+ :context {}+ :conversation_id (str (random-uuid))+ :history []+ :state {}}))))+ (testing "/feedback"+ (is (= "Unauthenticated"+ (mt/client :post 401 "metabot/feedback"+ {:feedback {}}))))))(deftest metabot-enabled-setting-test(mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider test-provider]- (mt/with-premium-features #{:metabot-v3}- (let [base-request {:message "Test"- :context {}- :conversation_id (str (random-uuid))- :history []- :state {}}]- (with-redefs [openrouter/openrouter (fn [_]- (mut/mock-llm-response- [{:type :start :id "msg-1"}- {:type :text :text "Hello"}- {:type :usage :usage {:promptTokens 1 :completionTokens 1}- :model "test-model" :id "msg-1"}]))]- (testing "Regular metabot is blocked when metabot-enabled is false"- (mt/with-temporary-setting-values [metabot-enabled? false]- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (mt/user-http-request :rasta :post 403 "metabot/agent-streaming"- base-request))))+ (let [base-request {:message "Test"+ :context {}+ :conversation_id (str (random-uuid))+ :history []+ :state {}}]+ (with-redefs [openrouter/openrouter (fn [_]+ (mut/mock-llm-response+ [{:type :start :id "msg-1"}+ {:type :text :text "Hello"}+ {:type :usage :usage {:promptTokens 1 :completionTokens 1}+ :model "test-model" :id "msg-1"}]))]+ (testing "Regular metabot is blocked when metabot-enabled is false"+ (mt/with-temporary-setting-values [metabot-enabled? false]+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (mt/user-http-request :rasta :post 403 "metabot/agent-streaming"+ base-request))))- (testing "Regular metabot works when metabot-enabled is true"- (mt/with-temporary-setting-values [metabot-enabled? true]- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"- (assoc base-request :conversation_id (str (random-uuid)))))))+ (testing "Regular metabot works when metabot-enabled is true"+ (mt/with-temporary-setting-values [metabot-enabled? true]+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"+ (assoc base-request :conversation_id (str (random-uuid)))))))- (testing "Embedded metabot is blocked when embedded-metabot-enabled? is false"- (mt/with-temporary-setting-values [embedded-metabot-enabled? false]- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (mt/user-http-request :rasta :post 403 "metabot/agent-streaming"- (assoc base-request- :metabot_id metabot.config/embedded-metabot-id- :conversation_id (str (random-uuid)))))))+ (testing "Embedded metabot is blocked when embedded-metabot-enabled? is false"+ (mt/with-temporary-setting-values [embedded-metabot-enabled? false]+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (mt/user-http-request :rasta :post 403 "metabot/agent-streaming"+ (assoc base-request+ :metabot_id metabot.config/embedded-metabot-id+ :conversation_id (str (random-uuid)))))))- (testing "Embedded metabot works when embedded-metabot-enabled? is true"- (mt/with-temporary-setting-values [embedded-metabot-enabled? true]- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"- (assoc base-request- :metabot_id metabot.config/embedded-metabot-id- :conversation_id (str (random-uuid)))))))+ (testing "Embedded metabot works when embedded-metabot-enabled? is true"+ (mt/with-temporary-setting-values [embedded-metabot-enabled? true]+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"+ (assoc base-request+ :metabot_id metabot.config/embedded-metabot-id+ :conversation_id (str (random-uuid)))))))- (testing "Regular metabot still works when only embedded is disabled"- (mt/with-temporary-setting-values [metabot-enabled? true- embedded-metabot-enabled? false]- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"- (assoc base-request :conversation_id (str (random-uuid)))))))+ (testing "Regular metabot still works when only embedded is disabled"+ (mt/with-temporary-setting-values [metabot-enabled? true+ embedded-metabot-enabled? false]+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"+ (assoc base-request :conversation_id (str (random-uuid)))))))- (testing "Embedded metabot still works when only regular is disabled"- (mt/with-temporary-setting-values [metabot-enabled? false- embedded-metabot-enabled? true]- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"- (assoc base-request- :metabot_id metabot.config/embedded-metabot-id- :conversation_id (str (random-uuid))))))))))))+ (testing "Embedded metabot still works when only regular is disabled"+ (mt/with-temporary-setting-values [metabot-enabled? false+ embedded-metabot-enabled? true]+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"+ (assoc base-request+ :metabot_id metabot.config/embedded-metabot-id+ :conversation_id (str (random-uuid)))))))))))(deftest extract-usage-test(testing "takes last cumulative usage per model"diff --git a/test/metabase/metabot/config_test.clj b/test/metabase/metabot/config_test.cljindex aae123fe65b..5eb4b719a0a 100644--- a/test/metabase/metabot/config_test.clj+++ b/test/metabase/metabot/config_test.clj@@ -7,42 +7,40 @@(deftest resolve-dynamic-metabot-id-test(testing "metabot ID resolution precedence"- (mt/with-premium-features #{:metabot-v3}- (mt/with-temporary-setting-values [metabot.settings/metabot-id nil]- (testing "explicit metabot-id takes precedence"- (is (= "explicit-id"- (metabot.config/resolve-dynamic-metabot-id "explicit-id"))))+ (mt/with-temporary-setting-values [metabot.settings/metabot-id nil]+ (testing "explicit metabot-id takes precedence"+ (is (= "explicit-id"+ (metabot.config/resolve-dynamic-metabot-id "explicit-id"))))- (testing "falls back to environment variable"- (mt/with-temporary-setting-values [metabot.settings/metabot-id "env-metabot-id"]- (is (= "env-metabot-id"- (metabot.config/resolve-dynamic-metabot-id nil)))))+ (testing "falls back to environment variable"+ (mt/with-temporary-setting-values [metabot.settings/metabot-id "env-metabot-id"]+ (is (= "env-metabot-id"+ (metabot.config/resolve-dynamic-metabot-id nil)))))- (testing "falls back to internal default when no explicit or env value"- (is (= metabot.config/internal-metabot-id- (metabot.config/resolve-dynamic-metabot-id nil))))))))+ (testing "falls back to internal default when no explicit or env value"+ (is (= metabot.config/internal-metabot-id+ (metabot.config/resolve-dynamic-metabot-id nil)))))))(deftest resolve-dynamic-profile-id-test(testing "profile ID resolution precedence"- (mt/with-premium-features #{:metabot-v3}- (testing "explicit profile-id takes highest precedence"- (is (= "explicit-profile"- (metabot.config/resolve-dynamic-profile-id "explicit-profile" "any-metabot-id"))))+ (testing "explicit profile-id takes highest precedence"+ (is (= "explicit-profile"+ (metabot.config/resolve-dynamic-profile-id "explicit-profile" "any-metabot-id"))))- (testing "metabot-id mapping takes second precedence"- (is (= "internal"- (metabot.config/resolve-dynamic-profile-id nil metabot.config/internal-metabot-id)))- (is (= "embedding_next"- (metabot.config/resolve-dynamic-profile-id nil metabot.config/embedded-metabot-id))))+ (testing "metabot-id mapping takes second precedence"+ (is (= "internal"+ (metabot.config/resolve-dynamic-profile-id nil metabot.config/internal-metabot-id)))+ (is (= "embedding_next"+ (metabot.config/resolve-dynamic-profile-id nil metabot.config/embedded-metabot-id))))- (testing "falls back to default when no matches"- (is (= "embedding_next"- (metabot.config/resolve-dynamic-profile-id nil "unknown-metabot-id"))))+ (testing "falls back to default when no matches"+ (is (= "embedding_next"+ (metabot.config/resolve-dynamic-profile-id nil "unknown-metabot-id"))))- (testing "single arity version uses dynamic metabot resolution"- (mt/with-temporary-setting-values [metabot.settings/metabot-id metabot.config/embedded-metabot-id]- (is (= "embedding_next"- (metabot.config/resolve-dynamic-profile-id nil))))))))+ (testing "single arity version uses dynamic metabot resolution"+ (mt/with-temporary-setting-values [metabot.settings/metabot-id metabot.config/embedded-metabot-id]+ (is (= "embedding_next"+ (metabot.config/resolve-dynamic-profile-id nil)))))))(deftest check-metabot-enabled-test(testing "0-arity throws when both are disabled, passes when either is enabled"@@ -59,24 +57,23 @@(deftest integrated-resolution-test(testing "combination of metabot-id and profile-id precedence resolution"- (mt/with-premium-features #{:metabot-v3}- (testing "explicit params override everything"- (mt/with-temporary-setting-values [metabot.settings/metabot-id nil]- (let [metabot-id (metabot.config/resolve-dynamic-metabot-id "custom-metabot")- profile-id (metabot.config/resolve-dynamic-profile-id "custom-profile" metabot-id)]- (is (= "custom-metabot" metabot-id))- (is (= "custom-profile" profile-id)))))+ (testing "explicit params override everything"+ (mt/with-temporary-setting-values [metabot.settings/metabot-id nil]+ (let [metabot-id (metabot.config/resolve-dynamic-metabot-id "custom-metabot")+ profile-id (metabot.config/resolve-dynamic-profile-id "custom-profile" metabot-id)]+ (is (= "custom-metabot" metabot-id))+ (is (= "custom-profile" profile-id)))))- (testing "env metabot-id resolves profile via metabot-id mapping"- (mt/with-temporary-setting-values [metabot.settings/metabot-id metabot.config/embedded-metabot-id]- (let [metabot-id (metabot.config/resolve-dynamic-metabot-id nil)- profile-id (metabot.config/resolve-dynamic-profile-id nil metabot-id)]- (is (= metabot.config/embedded-metabot-id metabot-id))- (is (= "embedding_next" profile-id)))))+ (testing "env metabot-id resolves profile via metabot-id mapping"+ (mt/with-temporary-setting-values [metabot.settings/metabot-id metabot.config/embedded-metabot-id]+ (let [metabot-id (metabot.config/resolve-dynamic-metabot-id nil)+ profile-id (metabot.config/resolve-dynamic-profile-id nil metabot-id)]+ (is (= metabot.config/embedded-metabot-id metabot-id))+ (is (= "embedding_next" profile-id)))))- (testing "defaults work together"- (mt/with-temporary-setting-values [metabot.settings/metabot-id nil]- (let [metabot-id (metabot.config/resolve-dynamic-metabot-id nil)- profile-id (metabot.config/resolve-dynamic-profile-id nil metabot-id)]- (is (= metabot.config/internal-metabot-id metabot-id))- (is (= "internal" profile-id))))))))+ (testing "defaults work together"+ (mt/with-temporary-setting-values [metabot.settings/metabot-id nil]+ (let [metabot-id (metabot.config/resolve-dynamic-metabot-id nil)+ profile-id (metabot.config/resolve-dynamic-profile-id nil metabot-id)]+ (is (= metabot.config/internal-metabot-id metabot-id))+ (is (= "internal" profile-id)))))))diff --git a/test/metabase/metabot/middleware/auth_test.clj b/test/metabase/metabot/middleware/auth_test.cljindex 96fa2a575bd..6f3b933577a 100644--- a/test/metabase/metabot/middleware/auth_test.clj+++ b/test/metabase/metabot/middleware/auth_test.clj@@ -41,38 +41,37 @@(assoc :body (java.io.ByteArrayInputStream. (.getBytes body "UTF-8")))))(deftest verify-slack-request-test- (mt/with-premium-features #{:metabot-v3}- (with-redefs [mw.auth/current-unix-timestamp (constantly test-timestamp)]- (testing "Valid signature w/ signing secret configured"- (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]- (let [body "test-body"- timestamp (str test-timestamp)- signature (compute-slack-signature body timestamp test-signing-secret)- result (wrapped-slack-handler (slack-request body timestamp signature))]- (is (true? (:slack/validated? result))))))+ (with-redefs [mw.auth/current-unix-timestamp (constantly test-timestamp)]+ (testing "Valid signature w/ signing secret configured"+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]+ (let [body "test-body"+ timestamp (str test-timestamp)+ signature (compute-slack-signature body timestamp test-signing-secret)+ result (wrapped-slack-handler (slack-request body timestamp signature))]+ (is (true? (:slack/validated? result))))))- (testing "Invalid signature"- (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]- (let [body "test-body"- timestamp (str test-timestamp)- signature "v0=invalid-signature"- result (wrapped-slack-handler (slack-request body timestamp signature))]- (is (false? (:slack/validated? result))))))+ (testing "Invalid signature"+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]+ (let [body "test-body"+ timestamp (str test-timestamp)+ signature "v0=invalid-signature"+ result (wrapped-slack-handler (slack-request body timestamp signature))]+ (is (false? (:slack/validated? result))))))- (testing "No signature header present - request passes through unchanged"- (let [body "test-body"- request (-> (ring.mock/request :post "/anyurl")- (assoc :body (java.io.ByteArrayInputStream. (.getBytes ^String body "UTF-8"))))- result (wrapped-slack-handler request)]- (is (not (contains? result :slack/validated?)))))+ (testing "No signature header present - request passes through unchanged"+ (let [body "test-body"+ request (-> (ring.mock/request :post "/anyurl")+ (assoc :body (java.io.ByteArrayInputStream. (.getBytes ^String body "UTF-8"))))+ result (wrapped-slack-handler request)]+ (is (not (contains? result :slack/validated?)))))- (testing "No signing secret configured"- (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret nil]- (let [body "test-body"- timestamp (str test-timestamp)- signature "v0=some-signature"- result (wrapped-slack-handler (slack-request body timestamp signature))]- (is (nil? (:slack/validated? result)))))))))+ (testing "No signing secret configured"+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret nil]+ (let [body "test-body"+ timestamp (str test-timestamp)+ signature "v0=some-signature"+ result (wrapped-slack-handler (slack-request body timestamp signature))]+ (is (nil? (:slack/validated? result))))))))(defn- validate-request-with-timestamp"Helper to test timestamp validation. Returns :slack/validated? result."@@ -83,20 +82,19 @@(:slack/validated? result)))(deftest verify-slack-request-timestamp-validation-test- (mt/with-premium-features #{:metabot-v3}- (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]- (with-redefs [mw.auth/current-unix-timestamp (constantly test-timestamp)]- (testing "Replay attack prevention - rejects timestamps outside 5 minute window"- (doseq [[expected offset-or-val description]- [[true 0 "current time"]- [true -300 "exactly 5 min ago"]- [true 300 "exactly 5 min ahead"]- [false -301 "5+ min ago"]- [false 301 "5+ min ahead"]- [false "abc" "malformed"]- [false nil "missing"]]]- (testing description- (is (= expected (validate-request-with-timestamp- (if (number? offset-or-val)- (str (+ test-timestamp offset-or-val))- offset-or-val)))))))))))+ (mt/with-temporary-raw-setting-values [metabot-slack-signing-secret test-signing-secret]+ (with-redefs [mw.auth/current-unix-timestamp (constantly test-timestamp)]+ (testing "Replay attack prevention - rejects timestamps outside 5 minute window"+ (doseq [[expected offset-or-val description]+ [[true 0 "current time"]+ [true -300 "exactly 5 min ago"]+ [true 300 "exactly 5 min ahead"]+ [false -301 "5+ min ago"]+ [false 301 "5+ min ahead"]+ [false "abc" "malformed"]+ [false nil "missing"]]]+ (testing description+ (is (= expected (validate-request-with-timestamp+ (if (number? offset-or-val)+ (str (+ test-timestamp offset-or-val))+ offset-or-val))))))))))diff --git a/test/metabase/metabot/native_generation_integration_test.clj b/test/metabase/metabot/native_generation_integration_test.cljindex d2e33ae4712..6e000461bb1 100644--- a/test/metabase/metabot/native_generation_integration_test.clj+++ b/test/metabase/metabot/native_generation_integration_test.clj@@ -37,37 +37,36 @@model-data {:description "Test model":dataset_query (lib/->legacy-MBQL model-source-query):type :model}]- (mt/with-premium-features #{:metabot-v3}- (mt/with-temp [:model/Collection {coll-id :id} {}- :model/Collection {child-id :id} {:location (collection/location-path coll-id)}- :model/Card _ (assoc model-data :name "NativeModel1" :collection_id coll-id)- :model/Card _ (assoc metric-data :name "NativeMetric1" :collection_id child-id)- :model/Metabot {metabot-id :id} {:name "native-test-bot" :collection_id coll-id}]+ (mt/with-temp [:model/Collection {coll-id :id} {}+ :model/Collection {child-id :id} {:location (collection/location-path coll-id)}+ :model/Card _ (assoc model-data :name "NativeModel1" :collection_id coll-id)+ :model/Card _ (assoc metric-data :name "NativeMetric1" :collection_id child-id)+ :model/Metabot {metabot-id :id} {:name "native-test-bot" :collection_id coll-id}]- (let [prompts-by-name {"NativeModel1" ["native q1" "native q2" "native q3" "native q4" "native q5"]- "NativeMetric1" ["native m1" "native m2" "native m3" "native m4" "native m5"]}- native-mock (make-native-prompt-generator prompts-by-name)]+ (let [prompts-by-name {"NativeModel1" ["native q1" "native q2" "native q3" "native q4" "native q5"]+ "NativeMetric1" ["native m1" "native m2" "native m3" "native m4" "native m5"]}+ native-mock (make-native-prompt-generator prompts-by-name)]- (testing "regenerate endpoint works with native path"+ (testing "regenerate endpoint works with native path"+ (with-redefs [native-generator/generate-example-questions native-mock]+ (mt/user-http-request :crowberto :post 204+ (format "metabot/metabot/%d/prompt-suggestions/regenerate" metabot-id)))++ (let [prompts (t2/select [:model/MetabotPrompt :prompt :model [:card.name :model_name]]+ :metabot_id metabot-id+ {:join [[:report_card :card] [:= :card.id :card_id]]+ :order-by [:metabot_prompt.id]})]+ (is (= 10 (count prompts)))+ (is (= (set (mapcat val prompts-by-name))+ (set (map :prompt prompts))))+ (is (= #{:model :metric}+ (set (map :model prompts))))))++ (testing "native path prompts are replaced on re-regenerate"+ (let [old-ids (t2/select-pks-set :model/MetabotPrompt :metabot_id metabot-id)](with-redefs [native-generator/generate-example-questions native-mock](mt/user-http-request :crowberto :post 204(format "metabot/metabot/%d/prompt-suggestions/regenerate" metabot-id)))-- (let [prompts (t2/select [:model/MetabotPrompt :prompt :model [:card.name :model_name]]- :metabot_id metabot-id- {:join [[:report_card :card] [:= :card.id :card_id]]- :order-by [:metabot_prompt.id]})]- (is (= 10 (count prompts)))- (is (= (set (mapcat val prompts-by-name))- (set (map :prompt prompts))))- (is (= #{:model :metric}- (set (map :model prompts))))))-- (testing "native path prompts are replaced on re-regenerate"- (let [old-ids (t2/select-pks-set :model/MetabotPrompt :metabot_id metabot-id)]- (with-redefs [native-generator/generate-example-questions native-mock]- (mt/user-http-request :crowberto :post 204- (format "metabot/metabot/%d/prompt-suggestions/regenerate" metabot-id)))- (let [new-ids (t2/select-pks-set :model/MetabotPrompt :metabot_id metabot-id)]- (is (= 10 (count new-ids)))- (is (empty? (set/intersection old-ids new-ids))))))))))))+ (let [new-ids (t2/select-pks-set :model/MetabotPrompt :metabot_id metabot-id)]+ (is (= 10 (count new-ids)))+ (is (empty? (set/intersection old-ids new-ids)))))))))))diff --git a/test/metabase/metabot/task/suggested_prompts_generator_test.clj b/test/metabase/metabot/task/suggested_prompts_generator_test.cljindex 770f44a366a..6e582a6fdf1 100644--- a/test/metabase/metabot/task/suggested_prompts_generator_test.clj+++ b/test/metabase/metabot/task/suggested_prompts_generator_test.clj@@ -11,7 +11,7 @@[toucan2.core :as t2]))(deftest suggested-prompts-generator-test- (mt/with-premium-features #{:metabot-v3 :content-verification}+ (mt/with-premium-features #{:content-verification}(mt/with-empty-h2-app-db!(let [original-metabot (t2/select-one :model/Metabot:entity_id (get-in metabot.config/metabot-configdiff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.cljindex a14b86a9374..1c06f348da4 100644--- a/test/metabase/metabot/tools/resources_test.clj+++ b/test/metabase/metabot/tools/resources_test.clj@@ -115,7 +115,7 @@(read-resource/read-resource {:uris ["metabase://dashboard/99999"]})))))))(deftest read-transform-resource-test- (mt/with-premium-features #{:metabot-v3 :transforms}+ (mt/with-premium-features #{:transforms}(mt/with-current-user (mt/user->id :crowberto)(mt/with-temp [:model/Transform {transform-id :id transform-name :name}{:name "Gadget Products"