Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1495 Trim all metabot tables to MB_AI_USAGE_MAX_RETENTION_DAYS (#74035)

appleby3fa1bab551a3
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj b/enterprise/backend/src/metabase_enterprise/metabot/settings.clj
index 739f7c2d116..92deb028259 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot/settings.clj
@@ -2,8 +2,7 @@
"Enterprise-only metabot settings for usage limits."
(:require
[metabase.settings.core :as setting :refer [defsetting]]
- [metabase.util.i18n :refer [deferred-tru]]
- [metabase.util.log :as log]))
+ [metabase.util.i18n :refer [deferred-tru]]))
(def ^:private valid-limit-units #{:tokens :messages})
@@ -58,49 +57,3 @@
:encryption :no
:export? true
:doc false)
-
-(def ^:private min-retention-days
- "Minimum allowed value for `ai-usage-max-retention-days`."
- 30)
-
-(def ^:private default-retention-days
- "Default value for `ai-usage-max-retention-days` (~6 months)."
- 180)
-
-(defn- log-minimum-value-warning
- [env-var-value]
- (log/warnf "MB_AI_USAGE_MAX_RETENTION_DAYS is set to %d; using the minimum value of %d instead."
- env-var-value
- min-retention-days))
-
-(defn- -ai-usage-max-retention-days []
- (let [env-var-value (setting/get-value-of-type :integer :ai-usage-max-retention-days)]
- (cond
- (nil? env-var-value)
- default-retention-days
-
- ;; Treat 0 as an alias for infinite retention
- (zero? env-var-value)
- nil
-
- (< env-var-value min-retention-days)
- (do
- (log-minimum-value-warning env-var-value)
- min-retention-days)
-
- :else
- env-var-value)))
-
-(defsetting ai-usage-max-retention-days
- (deferred-tru "Number of days to retain rows in the ai_usage_log table. Minimum value is 30; set to 0 to retain data indefinitely.")
- :type :integer
- :visibility :admin
- :setter :none
- :audit :never
- :export? true
- :encryption :no
- :getter #'-ai-usage-max-retention-days
- :doc "Sets the maximum number of days Metabase preserves rows in the `ai_usage_log` table.
-
-Once a day, Metabase deletes rows older than this threshold. The minimum value is 30 days (Metabase will treat entered values of 1 to 29 the same as 30).
-If set to 0, Metabase will keep all rows.")
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj
index ff87bb2d8c0..41df1b07091 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj
@@ -5,7 +5,7 @@
[clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
[java-time.api :as t]
- [metabase-enterprise.metabot.settings :as metabot.settings]
+ [metabase.metabot.settings :as metabot.settings]
[metabase.task.core :as task]
[metabase.util.log :as log]
[toucan2.core :as t2])
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj
deleted file mode 100644
index 20d7e843b71..00000000000
--- a/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj
+++ /dev/null
@@ -1,35 +0,0 @@
-(ns metabase-enterprise.metabot.settings-test
- (:require
- [clojure.test :refer :all]
- [metabase-enterprise.metabot.settings :as metabot.settings]
- [metabase.settings.core :as setting]
- [metabase.test :as mt]))
-
-(set! *warn-on-reflection* true)
-
-(deftest ai-usage-max-retention-days-default-test
- (testing "defaults to 180 days when no env var is set"
- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days nil]
- (is (= 180 (metabot.settings/ai-usage-max-retention-days))))))
-
-(deftest ai-usage-max-retention-days-infinite-test
- (testing "0 is an alias for infinite retention"
- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]
- (is (nil? (metabot.settings/ai-usage-max-retention-days))))))
-
-(deftest ai-usage-max-retention-days-passthrough-test
- (testing "values at or above the minimum pass through unchanged"
- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 100]
- (is (= 100 (metabot.settings/ai-usage-max-retention-days))))))
-
-(deftest ai-usage-max-retention-days-clamp-test
- (testing "values below the minimum are clamped up to 30"
- (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 1]
- (is (= 30 (metabot.settings/ai-usage-max-retention-days))))))
-
-(deftest ai-usage-max-retention-days-read-only-test
- (testing "the setting is env-var-only and cannot be set at runtime"
- (is (thrown-with-msg?
- java.lang.UnsupportedOperationException
- #"You cannot set ai-usage-max-retention-days"
- (setting/set! :ai-usage-max-retention-days 30)))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj
index 154509e1c64..b57cdb2f05d 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj
@@ -36,9 +36,7 @@
(defn- delete-conversations!
[conversation-ids]
- (let [conversation-ids (vec conversation-ids)]
- (t2/delete! :model/MetabotMessage {:where [:in :conversation_id conversation-ids]})
- (t2/delete! :model/MetabotConversation {:where [:in :id conversation-ids]})))
+ (t2/delete! :model/MetabotConversation {:where [:in :id (vec conversation-ids)]}))
(defn- insert-feedback!
[{:keys [message-id user-id positive issue-type freeform created-at updated-at]}]
diff --git a/resources/migrations/061/20260511_metabot_message_conversation_delete_cascade.yaml b/resources/migrations/061/20260511_metabot_message_conversation_delete_cascade.yaml
new file mode 100644
index 00000000000..5c4d1c78b69
--- /dev/null
+++ b/resources/migrations/061/20260511_metabot_message_conversation_delete_cascade.yaml
@@ -0,0 +1,33 @@
+databaseChangeLog:
+ - changeSet:
+ id: v61.cacxmq
+ author: appleby
+ comment: "Drop fk_metabot_message_conversation_id in preparation for ON DELETE CASCADE"
+ changes:
+ - dropForeignKeyConstraint:
+ baseTableName: metabot_message
+ constraintName: fk_metabot_message_conversation_id
+ rollback:
+ - addForeignKeyConstraint:
+ baseTableName: metabot_message
+ baseColumnNames: conversation_id
+ referencedTableName: metabot_conversation
+ referencedColumnNames: id
+ constraintName: fk_metabot_message_conversation_id
+
+ - changeSet:
+ id: v61.npxmgz
+ author: appleby
+ comment: "Re-add fk_metabot_message_conversation_id with ON DELETE CASCADE so trimming metabot_conversation rows removes their metabot_message children"
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: metabot_message
+ baseColumnNames: conversation_id
+ referencedTableName: metabot_conversation
+ referencedColumnNames: id
+ constraintName: fk_metabot_message_conversation_id
+ onDelete: CASCADE
+ rollback:
+ - dropForeignKeyConstraint:
+ baseTableName: metabot_message
+ constraintName: fk_metabot_message_conversation_id
diff --git a/src/metabase/metabot/init.clj b/src/metabase/metabot/init.clj
index 45d5d592ac3..c2e7042e14d 100644
--- a/src/metabase/metabot/init.clj
+++ b/src/metabase/metabot/init.clj
@@ -2,4 +2,5 @@
(:require
[metabase.metabot.scope]
[metabase.metabot.settings]
+ [metabase.metabot.task.metabot-conversation-trimmer]
[metabase.metabot.task.suggested-prompts-generator]))
diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.clj
index c15af14746f..132f79c454f 100644
--- a/src/metabase/metabot/settings.clj
+++ b/src/metabase/metabot/settings.clj
@@ -4,7 +4,8 @@
[metabase.llm.settings :as llm.settings]
[metabase.metabot.provider-util :as provider-util]
[metabase.settings.core :as setting :refer [defsetting]]
- [metabase.util.i18n :refer [deferred-tru tru]]))
+ [metabase.util.i18n :refer [deferred-tru tru]]
+ [metabase.util.log :as log]))
(defsetting metabot-id
(deferred-tru "Override Metabot ID for agent streaming requests.")
@@ -255,3 +256,55 @@
:export? false
:getter #(llm-provider-configured? (llm-metabot-provider))
:doc false)
+
+;;; ------------------------------------------------- AI Data Retention ------------------------------------------------
+
+(def ^:private min-retention-days
+ "Minimum allowed value for `ai-usage-max-retention-days`."
+ 30)
+
+(def ^:private default-retention-days
+ "Default value for `ai-usage-max-retention-days` (~6 months)."
+ 180)
+
+(defn- log-minimum-value-warning
+ [env-var-value]
+ (log/warnf "MB_AI_USAGE_MAX_RETENTION_DAYS is set to %d; using the minimum value of %d instead."
+ env-var-value
+ min-retention-days))
+
+(defn- -ai-usage-max-retention-days []
+ (let [env-var-value (setting/get-value-of-type :integer :ai-usage-max-retention-days)]
+ (cond
+ (nil? env-var-value)
+ default-retention-days
+
+ ;; Treat 0 as an alias for infinite retention
+ (zero? env-var-value)
+ nil
+
+ (< env-var-value min-retention-days)
+ (do
+ (log-minimum-value-warning env-var-value)
+ min-retention-days)
+
+ :else
+ env-var-value)))
+
+(defsetting ai-usage-max-retention-days
+ (deferred-tru "Number of days to retain rows in the ai_usage_log, metabot_conversation, and metabot_message tables. Minimum value is 30; set to 0 to retain data indefinitely.")
+ :type :integer
+ :visibility :admin
+ :setter :none
+ :audit :never
+ :export? true
+ :encryption :no
+ :getter #'-ai-usage-max-retention-days
+ :doc "Sets the maximum number of days Metabase preserves rows for the following application database tables:
+
+- `ai_usage_log`
+- `metabot_conversation`
+- `metabot_message`
+
+Once a day, Metabase deletes rows older than this threshold. The minimum value is 30 days (Metabase will treat entered values of 1 to 29 the same as 30).
+If set to 0, Metabase will keep all rows.")
diff --git a/src/metabase/metabot/task/metabot_conversation_trimmer.clj b/src/metabase/metabot/task/metabot_conversation_trimmer.clj
new file mode 100644
index 00000000000..f72618a77af
--- /dev/null
+++ b/src/metabase/metabot/task/metabot_conversation_trimmer.clj
@@ -0,0 +1,52 @@
+(ns metabase.metabot.task.metabot-conversation-trimmer
+ "Scheduled task to delete `metabot_conversation` rows older than the configured
+ `ai-usage-max-retention-days`. The FK from `metabot_message.conversation_id` is
+ `ON DELETE CASCADE`, so child messages (and, in turn, their `metabot_feedback`
+ and `metabot_source_feedback` rows, which cascade off `metabot_message.id`)
+ are removed by the database."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
+ [metabase.metabot.settings :as metabot.settings]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2])
+ (:import
+ (org.quartz DisallowConcurrentExecution)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private trimmer-job-key (jobs/key "metabase.task.metabot.conversation-trimmer.job"))
+(def ^:private trimmer-trigger-key (triggers/key "metabase.task.metabot.conversation-trimmer.trigger"))
+
+(defn- trim-old-conversations!
+ []
+ (let [retention-days (metabot.settings/ai-usage-max-retention-days)]
+ (if (nil? retention-days)
+ (log/info "Skipping metabot conversation cleanup; ai-usage-max-retention-days is 0 (infinite retention).")
+ (do
+ (log/infof "Trimming metabot_conversation rows older than %d days." (long retention-days))
+ (let [cutoff (t/minus (t/offset-date-time) (t/days (long retention-days)))
+ deleted (t2/delete! :model/MetabotConversation {:where [:< :created_at cutoff]})]
+ (log/infof "Metabot conversation cleanup complete. Deleted %d conversations (messages/feedback removed by ON DELETE CASCADE)."
+ (or deleted 0)))))))
+
+(task/defjob ^{DisallowConcurrentExecution true
+ :doc "Delete old metabot_conversation rows (cascades to metabot_message)"}
+ MetabotConversationTrimmer [_ctx]
+ (trim-old-conversations!))
+
+(defmethod task/init! ::MetabotConversationTrimmer
+ [_]
+ (let [job (jobs/build
+ (jobs/of-type MetabotConversationTrimmer)
+ (jobs/with-identity trimmer-job-key))
+ trigger (triggers/build
+ (triggers/with-identity trimmer-trigger-key)
+ (triggers/start-now)
+ (triggers/with-schedule
+ ;; daily at 23:16:37 — 2 minutes after ai-usage-trimmer
+ (cron/cron-schedule "37 16 23 * * ?")))]
+ (task/schedule-task! job trigger)))
diff --git a/src/metabase/testing_api/api.clj b/src/metabase/testing_api/api.clj
index f31f6f38d7c..7654dc88071 100644
--- a/src/metabase/testing_api/api.clj
+++ b/src/metabase/testing_api/api.clj
@@ -353,7 +353,6 @@
(defn- delete-seeded-usage-auditing-data!
[]
(t2/delete! :model/AiUsageLog {:where [:in :conversation_id e2e-usage-auditing-conversation-ids]})
- (t2/delete! :model/MetabotMessage {:where [:in :conversation_id e2e-usage-auditing-conversation-ids]})
(t2/delete! :model/MetabotConversation {:where [:in :id e2e-usage-auditing-conversation-ids]}))
(defn- insert-seeded-usage-auditing-conversation!
diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.clj
index 1bfea714311..80230f5685a 100644
--- a/test/metabase/metabot/settings_test.clj
+++ b/test/metabase/metabot/settings_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer [deftest is testing use-fixtures]]
[metabase.llm.settings :as llm.settings]
[metabase.metabot.settings :as metabot.settings]
+ [metabase.settings.core :as setting]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]))
@@ -187,3 +188,32 @@
(mt/with-premium-features #{:metabase-ai-managed}
(mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]
(is (= "metabase/anthropic/claude-sonnet-4-6" (metabot.settings/llm-metabot-provider)))))))
+
+;;; ------------------------------------------- ai-usage-max-retention-days Tests -------------------------------------------
+
+(deftest ai-usage-max-retention-days-default-test
+ (testing "defaults to 180 days when no env var is set"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days nil]
+ (is (= 180 (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-infinite-test
+ (testing "0 is an alias for infinite retention"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]
+ (is (nil? (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-passthrough-test
+ (testing "values at or above the minimum pass through unchanged"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 100]
+ (is (= 100 (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-clamp-test
+ (testing "values below the minimum are clamped up to 30"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 1]
+ (is (= 30 (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-read-only-test
+ (testing "the setting is env-var-only and cannot be set at runtime"
+ (is (thrown-with-msg?
+ java.lang.UnsupportedOperationException
+ #"You cannot set ai-usage-max-retention-days"
+ (setting/set! :ai-usage-max-retention-days 30)))))
diff --git a/test/metabase/metabot/task/metabot_conversation_trimmer_test.clj b/test/metabase/metabot/task/metabot_conversation_trimmer_test.clj
new file mode 100644
index 00000000000..6e5f2213bde
--- /dev/null
+++ b/test/metabase/metabot/task/metabot_conversation_trimmer_test.clj
@@ -0,0 +1,93 @@
+(ns metabase.metabot.task.metabot-conversation-trimmer-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.metabot.task.metabot-conversation-trimmer :as metabot-conversation-trimmer]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(set! *warn-on-reflection* true)
+
+(defn- conversation-attrs
+ [user-id created-at]
+ {:id (str (random-uuid))
+ :user_id user-id
+ :created_at created-at})
+
+(defn- message-attrs
+ [conversation-id created-at]
+ {:conversation_id conversation-id
+ :role "user"
+ :profile_id "trimmer-test"
+ :total_tokens 0
+ :data []
+ :external_id (str (random-uuid))
+ :created_at created-at})
+
+(deftest trims-conversations-older-than-default-retention-test
+ (testing "with default retention (180 days), only conversations older than 180 days are deleted"
+ (let [user-id (mt/user->id :crowberto)
+ now (t/offset-date-time)]
+ (mt/with-temp
+ [:model/MetabotConversation {recent-id :id} (conversation-attrs user-id now)
+ :model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))]
+ (#'metabot-conversation-trimmer/trim-old-conversations!)
+ (is (= #{recent-id}
+ (t2/select-fn-set :id :model/MetabotConversation
+ {:where [:in :id [recent-id old-id]]})))))))
+
+(deftest trims-conversations-older-than-custom-retention-test
+ (testing "with retention set to 30 days, conversations older than 30 days are deleted"
+ (let [user-id (mt/user->id :crowberto)
+ now (t/offset-date-time)]
+ (mt/with-temp
+ [:model/MetabotConversation {recent-id :id} (conversation-attrs user-id now)
+ :model/MetabotConversation {boundary-id :id} (conversation-attrs user-id (t/minus now (t/days 31)))
+ :model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))]
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 30]
+ (#'metabot-conversation-trimmer/trim-old-conversations!)
+ (is (= #{recent-id}
+ (t2/select-fn-set :id :model/MetabotConversation
+ {:where [:in :id [recent-id boundary-id old-id]]}))))))))
+
+(deftest skips-deletion-when-retention-is-infinite-test
+ (testing "when retention is set to 0 (infinite), nothing is deleted"
+ (let [user-id (mt/user->id :crowberto)
+ now (t/offset-date-time)]
+ (mt/with-temp
+ [:model/MetabotConversation {recent-id :id} (conversation-attrs user-id now)
+ :model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/years 5)))]
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]
+ (#'metabot-conversation-trimmer/trim-old-conversations!)
+ (is (= #{recent-id old-id}
+ (t2/select-fn-set :id :model/MetabotConversation
+ {:where [:in :id [recent-id old-id]]}))))))))
+
+(deftest cascades-to-messages-test
+ (testing "deleting an old conversation cascades to its messages via ON DELETE CASCADE"
+ (let [user-id (mt/user->id :crowberto)
+ now (t/offset-date-time)]
+ (mt/with-temp
+ [:model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))
+ :model/MetabotMessage {msg-id :id} (message-attrs old-id (t/minus now (t/days 200)))]
+ (#'metabot-conversation-trimmer/trim-old-conversations!)
+ (is (empty? (t2/select-fn-set :id :model/MetabotConversation :id old-id))
+ "expired conversation deleted")
+ (is (empty? (t2/select-fn-set :id :model/MetabotMessage :id msg-id))
+ "messages of expired conversation cascaded away")))))
+
+(deftest mixed-age-messages-in-old-conversation-test
+ (testing "an expired conversation is deleted along with all its messages — even fresh ones — because the cascade follows the parent"
+ (let [user-id (mt/user->id :crowberto)
+ now (t/offset-date-time)]
+ (mt/with-temp
+ [:model/MetabotConversation {old-id :id} (conversation-attrs user-id (t/minus now (t/days 200)))
+ :model/MetabotMessage {old-msg :id} (message-attrs old-id (t/minus now (t/days 200)))
+ :model/MetabotMessage {fresh-msg :id} (message-attrs old-id now)]
+ (#'metabot-conversation-trimmer/trim-old-conversations!)
+ (is (empty? (t2/select-fn-set :id :model/MetabotConversation :id old-id)))
+ (is (empty? (t2/select-fn-set :id :model/MetabotMessage
+ {:where [:in :id [old-msg fresh-msg]]})))))))