The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1410 Add ai-usage-max-retention-days setting and use it in ai-usage-trimmer (#73518)
appleby9303855d08d5
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj b/enterprise/backend/src/metabase_enterprise/metabot/settings.cljindex 92deb028259..6dd7213a472 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/settings.clj@@ -2,7 +2,8 @@"Enterprise-only metabot settings for usage limits."(:require[metabase.settings.core :as setting :refer [defsetting]]- [metabase.util.i18n :refer [deferred-tru]]))+ [metabase.util.i18n :refer [deferred-tru]]+ [metabase.util.log :as log]))(def ^:private valid-limit-units #{:tokens :messages})@@ -57,3 +58,47 @@:encryption :no:export? true:doc false)++(def ^:private min-retention-days+ "Minimum allowed value for `ai-usage-max-retention-days`."+ 30)++(def ^:private default-retention-days+ "Default value for `ai-usage-max-retention-days` (~6 months)."+ 180)++(defn- log-minimum-value-warning+ [env-var-value]+ (log/warnf "MB_AI_USAGE_MAX_RETENTION_DAYS is set to %d; using the minimum value of %d instead."+ env-var-value+ min-retention-days))++(defn- -ai-usage-max-retention-days []+ (let [env-var-value (setting/get-value-of-type :integer :ai-usage-max-retention-days)]+ (cond+ (nil? env-var-value)+ default-retention-days++ ;; Treat 0 as an alias for infinity+ (zero? env-var-value)+ ##Inf++ (< env-var-value min-retention-days)+ (do+ (log-minimum-value-warning env-var-value)+ min-retention-days)++ :else+ env-var-value)))++(defsetting ai-usage-max-retention-days+ (deferred-tru "Number of days to retain rows in the ai_usage_log table. Minimum value is 30; set to 0 to retain data indefinitely.")+ :visibility :internal+ :setter :none+ :audit :never+ :export? false+ :getter #'-ai-usage-max-retention-days+ :doc "Sets the maximum number of days Metabase preserves rows in the `ai_usage_log` table.++Once a day, Metabase deletes rows older than this threshold. The minimum value is 30 days (Metabase will treat entered values of 1 to 29 the same as 30).+If set to 0, Metabase will keep all rows.")diff --git a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.cljindex 7c7adfb67d9..8fff23e108c 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/task/ai_usage_trimmer.clj@@ -1,14 +1,15 @@(ns metabase-enterprise.metabot.task.ai-usage-trimmer- "Scheduled task to delete old ai_usage_log rows (older than 6 months)."+ "Scheduled task to delete `ai_usage_log` rows older than the configured `ai-usage-max-retention-days`."(:require[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t]+ [metabase-enterprise.metabot.settings :as metabot.settings][metabase.task.core :as task][metabase.util.log :as log][toucan2.core :as t2])(:import- (java.sql Timestamp)(org.quartz DisallowConcurrentExecution)))(set! *warn-on-reflection* true)@@ -16,14 +17,16 @@(def ^:private trimmer-job-key (jobs/key "metabase.task.metabot.ai-usage-trimmer.job"))(def ^:private trimmer-trigger-key (triggers/key "metabase.task.metabot.ai-usage-trimmer.trigger"))-(def ^:private retention-months 6)-(defn- trim-old-usage-data![]- (log/info "Trimming old AI usage log data.")- (let [cutoff (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) retention-months))- deleted (t2/delete! :model/AiUsageLog {:where [:< :created_at cutoff]})]- (log/infof "AI usage log cleanup complete. Deleted %d rows." (or deleted 0))))+ (let [retention-days (metabot.settings/ai-usage-max-retention-days)]+ (if (infinite? retention-days)+ (log/info "Skipping AI usage log cleanup; ai-usage-max-retention-days is 0 (infinite retention).")+ (do+ (log/infof "Trimming AI usage log rows older than %d days." (long retention-days))+ (let [cutoff (t/minus (t/offset-date-time) (t/days (long retention-days)))+ deleted (t2/delete! :model/AiUsageLog {:where [:< :created_at cutoff]})]+ (log/infof "AI usage log cleanup complete. Deleted %d rows." (or deleted 0)))))))(task/defjob ^{DisallowConcurrentExecution true:doc "Delete old ai_usage_log rows"}diff --git a/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/settings_test.cljnew file mode 100644index 00000000000..f930f0c7197--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/metabot/settings_test.clj@@ -0,0 +1,35 @@+(ns metabase-enterprise.metabot.settings-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.metabot.settings :as metabot.settings]+ [metabase.settings.core :as setting]+ [metabase.test :as mt]))++(set! *warn-on-reflection* true)++(deftest ai-usage-max-retention-days-default-test+ (testing "defaults to 180 days when no env var is set"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days nil]+ (is (= 180 (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-infinite-test+ (testing "0 is an alias for infinite retention"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]+ (is (= ##Inf (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-passthrough-test+ (testing "values at or above the minimum pass through unchanged"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 100]+ (is (= 100 (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-clamp-test+ (testing "values below the minimum are clamped up to 30"+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 1]+ (is (= 30 (metabot.settings/ai-usage-max-retention-days))))))++(deftest ai-usage-max-retention-days-read-only-test+ (testing "the setting is env-var-only and cannot be set at runtime"+ (is (thrown-with-msg?+ java.lang.UnsupportedOperationException+ #"You cannot set ai-usage-max-retention-days"+ (setting/set! :ai-usage-max-retention-days 30)))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot/task/ai_usage_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/task/ai_usage_trimmer_test.cljnew file mode 100644index 00000000000..d46173b76b9--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/metabot/task/ai_usage_trimmer_test.clj@@ -0,0 +1,57 @@+(ns metabase-enterprise.metabot.task.ai-usage-trimmer-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.metabot.task.ai-usage-trimmer :as ai-usage-trimmer]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))++(set! *warn-on-reflection* true)++(defn- usage-log-defaults+ []+ {:source "test"+ :model "test/model"+ :prompt_tokens 0+ :completion_tokens 0+ :total_tokens 0})++(deftest trims-rows-older-than-default-retention-test+ (testing "with default retention (180 days), only rows older than 180 days are deleted"+ (mt/with-temp+ [:model/AiUsageLog {recent-id :id} (assoc (usage-log-defaults)+ :created_at (t/offset-date-time))+ :model/AiUsageLog {old-id :id} (assoc (usage-log-defaults)+ :created_at (t/minus (t/offset-date-time) (t/days 200)))]+ (#'ai-usage-trimmer/trim-old-usage-data!)+ (is (= #{recent-id}+ (t2/select-fn-set :id :model/AiUsageLog {:where [:in :id [recent-id old-id]]}))))))++(deftest trims-rows-older-than-custom-retention-test+ (testing "with retention set to 30 days, rows older than 30 days are deleted"+ (mt/with-temp+ [:model/AiUsageLog {recent-id :id} (assoc (usage-log-defaults)+ :created_at (t/offset-date-time))+ :model/AiUsageLog {boundary-id :id} (assoc (usage-log-defaults)+ :created_at (t/minus (t/offset-date-time) (t/days 31)))+ :model/AiUsageLog {old-id :id} (assoc (usage-log-defaults)+ :created_at (t/minus (t/offset-date-time) (t/days 200)))]+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 30]+ (#'ai-usage-trimmer/trim-old-usage-data!)+ (is (= #{recent-id}+ (t2/select-fn-set :id :model/AiUsageLog {:where [:in :id [recent-id boundary-id old-id]]})))))))++(deftest skips-deletion-when-retention-is-infinite-test+ (testing "when retention is set to 0 (infinite), no rows are deleted"+ (mt/with-temp+ [:model/AiUsageLog {recent-id :id} (assoc (usage-log-defaults)+ :created_at (t/offset-date-time))+ :model/AiUsageLog {old-id :id} (assoc (usage-log-defaults)+ :created_at (t/minus (t/offset-date-time) (t/years 5)))]+ (mt/with-temp-env-var-value! [mb-ai-usage-max-retention-days 0]+ (#'ai-usage-trimmer/trim-old-usage-data!)+ (is (= #{recent-id old-id}+ (t2/select-fn-set :id :model/AiUsageLog {:where [:in :id [recent-id old-id]]})))))))