The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1495 Trim all metabot tables to MB_AI_USAGE_MAX_RETENTION_DAYS (#74035)
appleby3fa1bab551a3
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj b/enterprise/backend/src/metabase_enterprise/metabot/settings.cljindex 739f7c2d116..92deb028259 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/settings.clj@@ -2,8 +2,7 @@"Enterprise-only metabot settings for usage limits."(:require[metabase.settings.core :as setting :refer [defsetting]]- [metabase.util.i18n :refer [deferred-tru]]- [metabase.util.log :as log]))+ [metabase.util.i18n :refer [deferred-tru]]))(def ^:private valid-limit-units #{:tokens :messages})@@ -58,49 +57,3 @@:encryption :no:export? true:doc false)--(def ^:private min-retention-days- "Minimum allowed value for `ai-usage-max-retention-days`."- 30)--(def ^:private default-retention-days- "Default value for `ai-usage-max-retention-days` (~6 months)."- 180)--(defn- log-minimum-value-warning- [env-var-value]- (log/warnf "MB_AI_USAGE_MAX_RETENTION_DAYS is set to %d; using the minimum value of %d instead."- env-var-value- min-retention-days))--(defn- -ai-usage-max-retention-days []- (let [env-var-value (setting/get-value-of-type :integer :ai-usage-max-retention-days)]- (cond- (nil? env-var-value)- default-retention-days-- ;; Treat 0 as an alias for infinite retention- (zero? env-var-value)- nil-- (< env-var-value min-retention-days)- (do- (log-minimum-value-warning env-var-value)- min-retention-days)-- :else- env-var-value)))--(defsetting ai-usage-max-retention-days- (deferred-tru "Number of days to retain rows in the ai_usage_log table. Minimum value is 30; set to 0 to retain data indefinitely.")- :type :integer- :visibility :admin- :setter :none- :audit :never- :export? true- :encryption :no- :getter #'-ai-usage-max-retention-days- :doc "Sets the maximum number of days Metabase preserves rows in the `ai_usage_log` table.--Once a day, Metabase deletes rows older than this threshold. The minimum value is 30 days (Metabase will treat entered values of 1 to 29 the same as 30).-If set to 0, Metabase will keep all rows.")diff --git a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.cljindex ff87bb2d8c0..41df1b07091 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj@@ -5,7 +5,7 @@[clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers][java-time.api :as t]- [metabase-enterprise.metabot.settings :as metabot.settings]+ [metabase.metabot.settings :as metabot.settings][metabase.task.core :as task][metabase.util.log :as log][toucan2.core :as t2])diff --git a/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/settings_test.cljdeleted file mode 100644index 20d7e843b71..00000000000--- a/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj+++ /dev/null@@ -1,35 +0,0 @@-(ns metabase-enterprise.metabot.settings-test- (:require- [clojure.test :refer :all]- [metabase-enterprise.metabot.settings :as metabot.settings]- [metabase.settings.core :as setting]- [metabase.test :as mt]))--(set! *warn-on-reflection* true)--(deftest ai-usage-max-retention-days-default-test- (testing "defaults to 180 days when no env var is set"- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days nil]- (is (= 180 (metabot.settings/ai-usage-max-retention-days))))))--(deftest ai-usage-max-retention-days-infinite-test- (testing "0 is an alias for infinite retention"- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]- (is (nil? (metabot.settings/ai-usage-max-retention-days))))))--(deftest ai-usage-max-retention-days-passthrough-test- (testing "values at or above the minimum pass through unchanged"- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 100]- (is (= 100 (metabot.settings/ai-usage-max-retention-days))))))--(deftest ai-usage-max-retention-days-clamp-test- (testing "values below the minimum are clamped up to 30"- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 1]- (is (= 30 (metabot.settings/ai-usage-max-retention-days))))))--(deftest ai-usage-max-retention-days-read-only-test- (testing "the setting is env-var-only and cannot be set at runtime"- (is (thrown-with-msg?- java.lang.UnsupportedOperationException- #"You cannot set ai-usage-max-retention-days"- (setting/set! :ai-usage-max-retention-days 30)))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.cljindex 154509e1c64..b57cdb2f05d 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj@@ -36,9 +36,7 @@(defn- delete-conversations![conversation-ids]- (let [conversation-ids (vec conversation-ids)]- (t2/delete! :model/MetabotMessage {:where [:in :conversation_id conversation-ids]})- (t2/delete! :model/MetabotConversation {:where [:in :id conversation-ids]})))+ (t2/delete! :model/MetabotConversation {:where [:in :id (vec conversation-ids)]}))(defn- insert-feedback![{:keys [message-id user-id positive issue-type freeform created-at updated-at]}]diff --git a/resources/migrations/061/20260511_metabot_message_conversation_delete_cascade.yaml b/resources/migrations/061/20260511_metabot_message_conversation_delete_cascade.yamlnew file mode 100644index 00000000000..5c4d1c78b69--- /dev/null+++ b/resources/migrations/061/20260511_metabot_message_conversation_delete_cascade.yaml@@ -0,0 +1,33 @@+databaseChangeLog:+ - changeSet:+ id: v61.cacxmq+ author: appleby+ comment: "Drop fk_metabot_message_conversation_id in preparation for ON DELETE CASCADE"+ changes:+ - dropForeignKeyConstraint:+ baseTableName: metabot_message+ constraintName: fk_metabot_message_conversation_id+ rollback:+ - addForeignKeyConstraint:+ baseTableName: metabot_message+ baseColumnNames: conversation_id+ referencedTableName: metabot_conversation+ referencedColumnNames: id+ constraintName: fk_metabot_message_conversation_id++ - changeSet:+ id: v61.npxmgz+ author: appleby+ comment: "Re-add fk_metabot_message_conversation_id with ON DELETE CASCADE so trimming metabot_conversation rows removes their metabot_message children"+ changes:+ - addForeignKeyConstraint:+ baseTableName: metabot_message+ baseColumnNames: conversation_id+ referencedTableName: metabot_conversation+ referencedColumnNames: id+ constraintName: fk_metabot_message_conversation_id+ onDelete: CASCADE+ rollback:+ - dropForeignKeyConstraint:+ baseTableName: metabot_message+ constraintName: fk_metabot_message_conversation_iddiff --git a/src/metabase/metabot/init.clj b/src/metabase/metabot/init.cljindex 45d5d592ac3..c2e7042e14d 100644--- a/src/metabase/metabot/init.clj+++ b/src/metabase/metabot/init.clj@@ -2,4 +2,5 @@(:require[metabase.metabot.scope][metabase.metabot.settings]+ [metabase.metabot.task.metabot-conversation-trimmer][metabase.metabot.task.suggested-prompts-generator]))diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.cljindex c15af14746f..132f79c454f 100644--- a/src/metabase/metabot/settings.clj+++ b/src/metabase/metabot/settings.clj@@ -4,7 +4,8 @@[metabase.llm.settings :as llm.settings][metabase.metabot.provider-util :as provider-util][metabase.settings.core :as setting :refer [defsetting]]- [metabase.util.i18n :refer [deferred-tru tru]]))+ [metabase.util.i18n :refer [deferred-tru tru]]+ [metabase.util.log :as log]))(defsetting metabot-id(deferred-tru "Override Metabot ID for agent streaming requests.")@@ -255,3 +256,55 @@:export? false:getter #(llm-provider-configured? (llm-metabot-provider)):doc false)++;;; ------------------------------------------------- AI Data Retention ------------------------------------------------++(def ^:private min-retention-days+ "Minimum allowed value for `ai-usage-max-retention-days`."+ 30)++(def ^:private default-retention-days+ "Default value for `ai-usage-max-retention-days` (~6 months)."+ 180)++(defn- log-minimum-value-warning+ [env-var-value]+ (log/warnf "MB_AI_USAGE_MAX_RETENTION_DAYS is set to %d; using the minimum value of %d instead."+ env-var-value+ min-retention-days))++(defn- -ai-usage-max-retention-days []+ (let [env-var-value (setting/get-value-of-type :integer :ai-usage-max-retention-days)]+ (cond+ (nil? env-var-value)+ default-retention-days++ ;; Treat 0 as an alias for infinite retention+ (zero? env-var-value)+ nil++ (< env-var-value min-retention-days)+ (do+ (log-minimum-value-warning env-var-value)+ min-retention-days)++ :else+ env-var-value)))++(defsetting ai-usage-max-retention-days+ (deferred-tru "Number of days to retain rows in the ai_usage_log, metabot_conversation, and metabot_message tables. Minimum value is 30; set to 0 to retain data indefinitely.")+ :type :integer+ :visibility :admin+ :setter :none+ :audit :never+ :export? true+ :encryption :no+ :getter #'-ai-usage-max-retention-days+ :doc "Sets the maximum number of days Metabase preserves rows for the following application database tables:++- `ai_usage_log`+- `metabot_conversation`+- `metabot_message`++Once a day, Metabase deletes rows older than this threshold. The minimum value is 30 days (Metabase will treat entered values of 1 to 29 the same as 30).+If set to 0, Metabase will keep all rows.")diff --git a/src/metabase/metabot/task/metabot_conversation_trimmer.clj b/src/metabase/metabot/task/metabot_conversation_trimmer.cljnew file mode 100644index 00000000000..f72618a77af--- /dev/null+++ b/src/metabase/metabot/task/metabot_conversation_trimmer.clj@@ -0,0 +1,52 @@+(ns metabase.metabot.task.metabot-conversation-trimmer+ "Scheduled task to delete `metabot_conversation` rows older than the configured+ `ai-usage-max-retention-days`. The FK from `metabot_message.conversation_id` is+ `ON DELETE CASCADE`, so child messages (and, in turn, their `metabot_feedback`+ and `metabot_source_feedback` rows, which cascade off `metabot_message.id`)+ are removed by the database."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t]+ [metabase.metabot.settings :as metabot.settings]+ [metabase.task.core :as task]+ [metabase.util.log :as log]+ [toucan2.core :as t2])+ (:import+ (org.quartz DisallowConcurrentExecution)))++(set! *warn-on-reflection* true)++(def ^:private trimmer-job-key (jobs/key "metabase.task.metabot.conversation-trimmer.job"))+(def ^:private trimmer-trigger-key (triggers/key "metabase.task.metabot.conversation-trimmer.trigger"))++(defn- trim-old-conversations!+ []+ (let [retention-days (metabot.settings/ai-usage-max-retention-days)]+ (if (nil? retention-days)+ (log/info "Skipping metabot conversation cleanup; ai-usage-max-retention-days is 0 (infinite retention).")+ (do+ (log/infof "Trimming metabot_conversation rows older than %d days." (long retention-days))+ (let [cutoff (t/minus (t/offset-date-time) (t/days (long retention-days)))+ deleted (t2/delete! :model/MetabotConversation {:where [:< :created_at cutoff]})]+ (log/infof "Metabot conversation cleanup complete. Deleted %d conversations (messages/feedback removed by ON DELETE CASCADE)."+ (or deleted 0)))))))++(task/defjob ^{DisallowConcurrentExecution true+ :doc "Delete old metabot_conversation rows (cascades to metabot_message)"}+ MetabotConversationTrimmer [_ctx]+ (trim-old-conversations!))++(defmethod task/init! ::MetabotConversationTrimmer+ [_]+ (let [job (jobs/build+ (jobs/of-type MetabotConversationTrimmer)+ (jobs/with-identity trimmer-job-key))+ trigger (triggers/build+ (triggers/with-identity trimmer-trigger-key)+ (triggers/start-now)+ (triggers/with-schedule+ ;; daily at 23:16:37 — 2 minutes after ai-usage-trimmer+ (cron/cron-schedule "37 16 23 * * ?")))]+ (task/schedule-task! job trigger)))diff --git a/src/metabase/testing_api/api.clj b/src/metabase/testing_api/api.cljindex f31f6f38d7c..7654dc88071 100644--- a/src/metabase/testing_api/api.clj+++ b/src/metabase/testing_api/api.clj@@ -353,7 +353,6 @@(defn- delete-seeded-usage-auditing-data- (t2/delete! :model/MetabotMessage {:where [:in :conversation_id e2e-usage-auditing-conversation-ids]})(t2/delete! :model/MetabotConversation {:where [:in :id e2e-usage-auditing-conversation-ids]}))(defn- insert-seeded-usage-auditing-conversation!diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.cljindex 1bfea714311..80230f5685a 100644--- a/test/metabase/metabot/settings_test.clj+++ b/test/metabase/metabot/settings_test.clj@@ -3,6 +3,7 @@[clojure.test :refer [deftest is testing use-fixtures]][metabase.llm.settings :as llm.settings][metabase.metabot.settings :as metabot.settings]+ [metabase.settings.core :as setting][metabase.test :as mt][metabase.test.fixtures :as fixtures]))@@ -187,3 +188,32 @@(mt/with-premium-features #{:metabase-ai-managed}(mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"](is (= "metabase/anthropic/claude-sonnet-4-6" (metabot.settings/llm-metabot-provider)))))))++;;; ------------------------------------------- ai-usage-max-retention-days Tests -------------------------------------------++(deftest ai-usage-max-retention-days-default-test+ (testing "defaults to 180 days when no env var is set"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days nil]+ (is (= 180 (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-infinite-test+ (testing "0 is an alias for infinite retention"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]+ (is (nil? (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-passthrough-test+ (testing "values at or above the minimum pass through unchanged"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 100]+ (is (= 100 (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-clamp-test+ (testing "values below the minimum are clamped up to 30"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 1]+ (is (= 30 (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-read-only-test+ (testing "the setting is env-var-only and cannot be set at runtime"+ (is (thrown-with-msg?+ java.lang.UnsupportedOperationException+ #"You cannot set ai-usage-max-retention-days"+ (setting/set! :ai-usage-max-retention-days 30)))))diff --git a/test/metabase/metabot/task/metabot_conversation_trimmer_test.clj b/test/metabase/metabot/task/metabot_conversation_trimmer_test.cljnew file mode 100644index 00000000000..6e5f2213bde--- /dev/null+++ b/test/metabase/metabot/task/metabot_conversation_trimmer_test.clj@@ -0,0 +1,93 @@+(ns metabase.metabot.task.metabot-conversation-trimmer-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.metabot.task.metabot-conversation-trimmer :as metabot-conversation-trimmer]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))++(set! *warn-on-reflection* true)++(defn- conversation-attrs+ [user-id created-at]+ {:id (str (random-uuid))+ :user_id user-id+ :created_at created-at})++(defn- message-attrs+ [conversation-id created-at]+ {:conversation_id conversation-id+ :role "user"+ :profile_id "trimmer-test"+ :total_tokens 0+ :data []+ :external_id (str (random-uuid))+ :created_at created-at})++(deftest trims-conversations-older-than-default-retention-test+ (testing "with default retention (180 days), only conversations older than 180 days are deleted"+ (let [user-id (mt/user->id :crowberto)+ now (t/offset-date-time)]+ (mt/with-temp+ [:model/MetabotConversation {recent-id :id} (conversation-attrs user-id now)+ :model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))]+ (#'metabot-conversation-trimmer/trim-old-conversations!)+ (is (= #{recent-id}+ (t2/select-fn-set :id :model/MetabotConversation+ {:where [:in :id [recent-id old-id]]})))))))++(deftest trims-conversations-older-than-custom-retention-test+ (testing "with retention set to 30 days, conversations older than 30 days are deleted"+ (let [user-id (mt/user->id :crowberto)+ now (t/offset-date-time)]+ (mt/with-temp+ [:model/MetabotConversation {recent-id :id} (conversation-attrs user-id now)+ :model/MetabotConversation {boundary-id :id} (conversation-attrs user-id (t/minus now (t/days 31)))+ :model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))]+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 30]+ (#'metabot-conversation-trimmer/trim-old-conversations!)+ (is (= #{recent-id}+ (t2/select-fn-set :id :model/MetabotConversation+ {:where [:in :id [recent-id boundary-id old-id]]}))))))))++(deftest skips-deletion-when-retention-is-infinite-test+ (testing "when retention is set to 0 (infinite), nothing is deleted"+ (let [user-id (mt/user->id :crowberto)+ now (t/offset-date-time)]+ (mt/with-temp+ [:model/MetabotConversation {recent-id :id} (conversation-attrs user-id now)+ :model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/years 5)))]+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]+ (#'metabot-conversation-trimmer/trim-old-conversations!)+ (is (= #{recent-id old-id}+ (t2/select-fn-set :id :model/MetabotConversation+ {:where [:in :id [recent-id old-id]]}))))))))++(deftest cascades-to-messages-test+ (testing "deleting an old conversation cascades to its messages via ON DELETE CASCADE"+ (let [user-id (mt/user->id :crowberto)+ now (t/offset-date-time)]+ (mt/with-temp+ [:model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))+ :model/MetabotMessage {msg-id :id} (message-attrs old-id (t/minus now (t/days 200)))]+ (#'metabot-conversation-trimmer/trim-old-conversations!)+ (is (empty? (t2/select-fn-set :id :model/MetabotConversation :id old-id))+ "expired conversation deleted")+ (is (empty? (t2/select-fn-set :id :model/MetabotMessage :id msg-id))+ "messages of expired conversation cascaded away")))))++(deftest mixed-age-messages-in-old-conversation-test+ (testing "an expired conversation is deleted along with all its messages — even fresh ones — because the cascade follows the parent"+ (let [user-id (mt/user->id :crowberto)+ now (t/offset-date-time)]+ (mt/with-temp+ [:model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))+ :model/MetabotMessage {old-msg :id} (message-attrs old-id (t/minus now (t/days 200)))+ :model/MetabotMessage {fresh-msg :id} (message-attrs old-id now)]+ (#'metabot-conversation-trimmer/trim-old-conversations!)+ (is (empty? (t2/select-fn-set :id :model/MetabotConversation :id old-id)))+ (is (empty? (t2/select-fn-set :id :model/MetabotMessage+ {:where [:in :id [old-msg fresh-msg]]})))))))