Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1410 Add ai-usage-max-retention-days setting and use it in ai-usage-trimmer (#73518)

appleby9303855d08d5
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj b/enterprise/backend/src/metabase_enterprise/metabot/settings.clj
index 92deb028259..6dd7213a472 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot/settings.clj
@@ -2,7 +2,8 @@
"Enterprise-only metabot settings for usage limits."
(:require
[metabase.settings.core :as setting :refer [defsetting]]
- [metabase.util.i18n :refer [deferred-tru]]))
+ [metabase.util.i18n :refer [deferred-tru]]
+ [metabase.util.log :as log]))
(def ^:private valid-limit-units #{:tokens :messages})
@@ -57,3 +58,47 @@
:encryption :no
:export? true
:doc false)
+
+(def ^:private min-retention-days
+ "Minimum allowed value for `ai-usage-max-retention-days`."
+ 30)
+
+(def ^:private default-retention-days
+ "Default value for `ai-usage-max-retention-days` (~6 months)."
+ 180)
+
+(defn- log-minimum-value-warning
+ [env-var-value]
+ (log/warnf "MB_AI_USAGE_MAX_RETENTION_DAYS is set to %d; using the minimum value of %d instead."
+ env-var-value
+ min-retention-days))
+
+(defn- -ai-usage-max-retention-days []
+ (let [env-var-value (setting/get-value-of-type :integer :ai-usage-max-retention-days)]
+ (cond
+ (nil? env-var-value)
+ default-retention-days
+
+ ;; Treat 0 as an alias for infinity
+ (zero? env-var-value)
+ ##Inf
+
+ (< env-var-value min-retention-days)
+ (do
+ (log-minimum-value-warning env-var-value)
+ min-retention-days)
+
+ :else
+ env-var-value)))
+
+(defsetting ai-usage-max-retention-days
+ (deferred-tru "Number of days to retain rows in the ai_usage_log table. Minimum value is 30; set to 0 to retain data indefinitely.")
+ :visibility :internal
+ :setter :none
+ :audit :never
+ :export? false
+ :getter #'-ai-usage-max-retention-days
+ :doc "Sets the maximum number of days Metabase preserves rows in the `ai_usage_log` table.
+
+Once a day, Metabase deletes rows older than this threshold. The minimum value is 30 days (Metabase will treat entered values of 1 to 29 the same as 30).
+If set to 0, Metabase will keep all rows.")
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj
index 7c7adfb67d9..8fff23e108c 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj
@@ -1,14 +1,15 @@
(ns metabase-enterprise.metabot.task.ai-usage-trimmer
- "Scheduled task to delete old ai_usage_log rows (older than 6 months)."
+ "Scheduled task to delete `ai_usage_log` rows older than the configured `ai-usage-max-retention-days`."
(:require
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
+ [metabase-enterprise.metabot.settings :as metabot.settings]
[metabase.task.core :as task]
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
- (java.sql Timestamp)
(org.quartz DisallowConcurrentExecution)))
(set! *warn-on-reflection* true)
@@ -16,14 +17,16 @@
(def ^:private trimmer-job-key (jobs/key "metabase.task.metabot.ai-usage-trimmer.job"))
(def ^:private trimmer-trigger-key (triggers/key "metabase.task.metabot.ai-usage-trimmer.trigger"))
-(def ^:private retention-months 6)
-
(defn- trim-old-usage-data!
[]
- (log/info "Trimming old AI usage log data.")
- (let [cutoff (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) retention-months))
- deleted (t2/delete! :model/AiUsageLog {:where [:< :created_at cutoff]})]
- (log/infof "AI usage log cleanup complete. Deleted %d rows." (or deleted 0))))
+ (let [retention-days (metabot.settings/ai-usage-max-retention-days)]
+ (if (infinite? retention-days)
+ (log/info "Skipping AI usage log cleanup; ai-usage-max-retention-days is 0 (infinite retention).")
+ (do
+ (log/infof "Trimming AI usage log rows older than %d days." (long retention-days))
+ (let [cutoff (t/minus (t/offset-date-time) (t/days (long retention-days)))
+ deleted (t2/delete! :model/AiUsageLog {:where [:< :created_at cutoff]})]
+ (log/infof "AI usage log cleanup complete. Deleted %d rows." (or deleted 0)))))))
(task/defjob ^{DisallowConcurrentExecution true
:doc "Delete old ai_usage_log rows"}
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj
new file mode 100644
index 00000000000..f930f0c7197
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj
@@ -0,0 +1,35 @@
+(ns metabase-enterprise.metabot.settings-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.metabot.settings :as metabot.settings]
+ [metabase.settings.core :as setting]
+ [metabase.test :as mt]))
+
+(set! *warn-on-reflection* true)
+
+(deftest ai-usage-max-retention-days-default-test
+ (testing "defaults to 180 days when no env var is set"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days nil]
+ (is (= 180 (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-infinite-test
+ (testing "0 is an alias for infinite retention"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]
+ (is (= ##Inf (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-passthrough-test
+ (testing "values at or above the minimum pass through unchanged"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 100]
+ (is (= 100 (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-clamp-test
+ (testing "values below the minimum are clamped up to 30"
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 1]
+ (is (= 30 (metabot.settings/ai-usage-max-retention-days))))))
+
+(deftest ai-usage-max-retention-days-read-only-test
+ (testing "the setting is env-var-only and cannot be set at runtime"
+ (is (thrown-with-msg?
+ java.lang.UnsupportedOperationException
+ #"You cannot set ai-usage-max-retention-days"
+ (setting/set! :ai-usage-max-retention-days 30)))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/task/ai_usage_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/task/ai_usage_trimmer_test.clj
new file mode 100644
index 00000000000..d46173b76b9
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/metabot/task/ai_usage_trimmer_test.clj
@@ -0,0 +1,57 @@
+(ns metabase-enterprise.metabot.task.ai-usage-trimmer-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase-enterprise.metabot.task.ai-usage-trimmer :as ai-usage-trimmer]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(set! *warn-on-reflection* true)
+
+(defn- usage-log-defaults
+ []
+ {:source "test"
+ :model "test/model"
+ :prompt_tokens 0
+ :completion_tokens 0
+ :total_tokens 0})
+
+(deftest trims-rows-older-than-default-retention-test
+ (testing "with default retention (180 days), only rows older than 180 days are deleted"
+ (mt/with-temp
+ [:model/AiUsageLog {recent-id :id} (assoc (usage-log-defaults)
+ :created_at (t/offset-date-time))
+ :model/AiUsageLog {old-id :id} (assoc (usage-log-defaults)
+ :created_at (t/minus (t/offset-date-time) (t/days 200)))]
+ (#'ai-usage-trimmer/trim-old-usage-data!)
+ (is (= #{recent-id}
+ (t2/select-fn-set :id :model/AiUsageLog {:where [:in :id [recent-id old-id]]}))))))
+
+(deftest trims-rows-older-than-custom-retention-test
+ (testing "with retention set to 30 days, rows older than 30 days are deleted"
+ (mt/with-temp
+ [:model/AiUsageLog {recent-id :id} (assoc (usage-log-defaults)
+ :created_at (t/offset-date-time))
+ :model/AiUsageLog {boundary-id :id} (assoc (usage-log-defaults)
+ :created_at (t/minus (t/offset-date-time) (t/days 31)))
+ :model/AiUsageLog {old-id :id} (assoc (usage-log-defaults)
+ :created_at (t/minus (t/offset-date-time) (t/days 200)))]
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 30]
+ (#'ai-usage-trimmer/trim-old-usage-data!)
+ (is (= #{recent-id}
+ (t2/select-fn-set :id :model/AiUsageLog {:where [:in :id [recent-id boundary-id old-id]]})))))))
+
+(deftest skips-deletion-when-retention-is-infinite-test
+ (testing "when retention is set to 0 (infinite), no rows are deleted"
+ (mt/with-temp
+ [:model/AiUsageLog {recent-id :id} (assoc (usage-log-defaults)
+ :created_at (t/offset-date-time))
+ :model/AiUsageLog {old-id :id} (assoc (usage-log-defaults)
+ :created_at (t/minus (t/offset-date-time) (t/years 5)))]
+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]
+ (#'ai-usage-trimmer/trim-old-usage-data!)
+ (is (= #{recent-id old-id}
+ (t2/select-fn-set :id :model/AiUsageLog {:where [:in :id [recent-id old-id]]})))))))