The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Partition the query_execution table when using Postgres AppDB. (#78497)
technomancyc28f952435ab
diff --git a/bin/lint-migrations-file/src/change_set/strict.clj b/bin/lint-migrations-file/src/change_set/strict.cljindex 10c3029347c..6dce2319558 100644--- a/bin/lint-migrations-file/src/change_set/strict.clj+++ b/bin/lint-migrations-file/src/change_set/strict.clj@@ -47,7 +47,12 @@(s/def ::changes(s/or- ;; only one change allowed per change set for the strict schema.+ ;; only one change allowed per change set for the strict schema:+ ;; "Dumb DBs like MySQL can't do DDL in transactions and auto commit each+ ;; change, so if you do multiple changes in one changeset it can commit the+ ;; first few changes and then fail. The changes that succeeded don't get+ ;; rolled back. Then you're in a state where the change set was partially+ ;; applied but failed and rerunning it is going to be ultra-busted":one-change(s/alt :change :change.strict/change)diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.cljindex b9ccc20a107..de151ca48e6 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.audit-app.task.truncate-audit-tables(:require+ [metabase.app-db.core :as mdb][metabase.premium-features.core :refer [defenterprise]]))(defenterprise audit-models-to-truncate@@ -11,6 +12,10 @@`metabase-enterprise.mcp.task.mcp-usage-trimmer` (keyed on `ai-usage-max-retention-days`).":feature :audit-app[]- [{:model :model/QueryExecution :timestamp-col :started_at}- {:model :model/AuditLog :timestamp-col :timestamp}- {:model :model/ViewLog :timestamp-col :timestamp}])+ ;; postgres has partitioned query_execution; we detach partitions instead of deleting+ (if (= :postgres (mdb/db-type))+ [{:model :model/AuditLog :timestamp-col :timestamp}+ {:model :model/ViewLog :timestamp-col :timestamp}]+ [{:model :model/QueryExecution :timestamp-col :started_at}+ {:model :model/AuditLog :timestamp-col :timestamp}+ {:model :model/ViewLog :timestamp-col :timestamp}]))diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.cljindex fcbf8d1789c..b76d5899d59 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][java-time.api :as t]+ [metabase.app-db.core :as mdb][metabase.audit-app.task.truncate-audit-tables :as task.truncate-audit-tables][metabase.premium-features.core :as premium-features][metabase.query-processor.util :as qp.util]@@ -32,13 +33,18 @@;; 1 year ago:model/QueryExecution {qe3-id :id} (merge (query-execution-defaults){:started_at (t/minus (t/offset-date-time) (t/years 1))})]- ;; Mock a cloud environment so that we can change the setting value via env var- (with-redefs [premium-features/is-hosted? (constantly true)]- (testing "When the threshold is 0 (representing infinity), no rows are deleted"- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0]- (#'task.truncate-audit-tables/truncate-audit-tables!)- (is (= #{qe1-id qe2-id qe3-id}- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))))))))+ (if (= :postgres (mdb/db-type))+ (testing "When on postgres, nothing in query_execution is truncated."+ (#'task.truncate-audit-tables/truncate-audit-tables!)+ (is (= #{qe1-id qe2-id qe3-id}+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))+ ;; Mock a cloud environment so that we can change the setting value via env var+ (with-redefs [premium-features/is-hosted? (constantly true)]+ (testing "When the threshold is 0 (representing infinity), no rows are deleted"+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0]+ (#'task.truncate-audit-tables/truncate-audit-tables!)+ (is (= #{qe1-id qe2-id qe3-id}+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))))))))(defn- audit-log-defaults[]diff --git a/resources/migrations/064/20260723_query_execution_partitioned.yaml b/resources/migrations/064/20260723_query_execution_partitioned.yamlnew file mode 100644index 00000000000..20decb6dca2--- /dev/null+++ b/resources/migrations/064/20260723_query_execution_partitioned.yaml@@ -0,0 +1,142 @@+# The query_execution table needs to be partitioned on postgres.+#+# Attach the query_execution_old table as a partition. This partition covers up+# to the current month, plus one more after, in order to handle the case where+# the migration runs at the end of the month and query executions need to be+# written to disk before the job to create the next partition gets a chance to+# run.+#+# The IDENTITY has to be dropped since this can't be done in a partition; it+# should be handled by the higher level table.+#+# The old query_execution table is now a partition of the new query_execution+# table, but the v_query_log view still references the old one. Re-creating it+# will point it at the new one.+#+# We should use sqlFile for this, but that is blocked because you can't mix sql+# and sqlFile in the same migration because mysql sucks, so sadly we have to+# copy/paste the entire contents of postgres-query_log.sql into here instead.+# Splitting into a separate file doesn't work because the rollback order would+# be wrong; it needs to be after this one for going forward but before this one+# for rollback.+databaseChangeLog:+ - changeSet:+ id: v64.2026-07-23T11:39:56+ author: technomancy+ comment: Create partitioned query_execution table and attach old one+ dbms: postgresql+ changes:+ - sql:+ sql: >-+ ALTER TABLE query_execution DROP CONSTRAINT query_execution_pkey;+ ALTER TABLE query_execution RENAME TO query_execution_old;+ CREATE TABLE query_execution (LIKE query_execution_old INCLUDING ALL)+ PARTITION BY RANGE (started_at);+ SELECT setval(pg_get_serial_sequence('query_execution', 'id'),+ nextval(pg_get_serial_sequence('query_execution_old', 'id')));+ ALTER TABLE query_execution_old ALTER COLUMN id DROP IDENTITY;+ ALTER TABLE query_execution ATTACH PARTITION query_execution_old+ FOR VALUES FROM ('1970-01-01')+ TO (date_trunc('month', current_timestamp + '2 months'::interval));++ DROP VIEW IF EXISTS v_query_log;+ CREATE OR REPLACE VIEW v_query_log AS+ SELECT id AS entity_id,+ started_at,+ cast(running_time AS double precision) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ 'card_' || card_id AS card_qualified_id,+ dashboard_id,+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,+ pulse_id,+ database_id,+ 'database_' || database_id AS database_qualified_id,+ cache_hit,+ action_id,+ 'action_' || action_id AS action_qualified_id,+ transform_id,+ 'transform_' || transform_id AS transform_qualified_id,+ lens_id,+ lens_params,+ query,+ embedding_client,+ embedding_client_identifier,+ embedding_route,+ CASE WHEN embedding_client LIKE '%-preview' THEN TRUE ELSE FALSE END AS is_preview,+ embedding_sdk_version,+ metabase_version,+ auth_method,+ is_sandboxed,+ is_impersonated,+ is_db_routed,+ parameters,+ tenant_id,+ embedding_hostname,+ embedding_path,+ user_agent,+ sanitized_user_agent,+ ip_address+ FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;+ rollback:+ - sql:+ sql: >-+ DROP VIEW IF EXISTS v_query_log;+ ALTER TABLE query_execution DETACH PARTITION query_execution_old;+ ALTER TABLE query_execution_old ALTER COLUMN id+ ADD GENERATED BY DEFAULT AS IDENTITY;+ SELECT setval(pg_get_serial_sequence('query_execution_old', 'id'),+ nextval(pg_get_serial_sequence('query_execution', 'id')));+ DROP TABLE query_execution;+ ALTER TABLE query_execution_old RENAME TO query_execution;+ ALTER TABLE query_execution+ ADD CONSTRAINT query_execution_pkey PRIMARY KEY(ID);++ CREATE OR REPLACE VIEW v_query_log AS+ SELECT id AS entity_id,+ started_at,+ cast(running_time AS double precision) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ 'card_' || card_id AS card_qualified_id,+ dashboard_id,+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,+ pulse_id,+ database_id,+ 'database_' || database_id AS database_qualified_id,+ cache_hit,+ action_id,+ 'action_' || action_id AS action_qualified_id,+ transform_id,+ 'transform_' || transform_id AS transform_qualified_id,+ lens_id,+ lens_params,+ query,+ embedding_client,+ embedding_client_identifier,+ embedding_route,+ CASE WHEN embedding_client LIKE '%-preview' THEN TRUE ELSE FALSE END AS is_preview,+ embedding_sdk_version,+ metabase_version,+ auth_method,+ is_sandboxed,+ is_impersonated,+ is_db_routed,+ parameters,+ tenant_id,+ embedding_hostname,+ embedding_path,+ user_agent,+ sanitized_user_agent,+ ip_address+ FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;diff --git a/resources/migrations/064/20260725_rename_query_execution_partition.yaml b/resources/migrations/064/20260725_rename_query_execution_partition.yamlnew file mode 100644index 00000000000..6c1fb06b38d--- /dev/null+++ b/resources/migrations/064/20260725_rename_query_execution_partition.yaml@@ -0,0 +1,36 @@+# Finally, we want the query_execution_old table to be named after the same+# pattern as the new partitions will have in order to make the logic for+# detaching them simpler: query_execution_2026-08 for example if the migration+# was run in July 2026 (extra month to allow time for jobs to run).+#+# But we don't know when the migration will be run, so that has to be+# calculated dynamically, which requires use of 'EXECUTE'. Because of the+# complication of dynamic calculation, we delay the rename of the table to the+# end so only one query needs to be dynamic.++databaseChangeLog:+ - changeSet:+ id: v64.2026-07-25T00:39:56+ author: technomancy+ comment: Rename old query_execution partition to match naming scheme+ dbms: postgresql+ changes:+ - sql:+ sql: >-+ DO $$+ BEGIN+ EXECUTE format('ALTER TABLE query_execution_old RENAME TO query_execution_%s;',+ TO_CHAR(current_timestamp + '1 month'::interval, 'YYYY_MM'));+ END+ $$ language plpgsql;+ rollback:+ - sql:+ sql: >-+ DO $$+ BEGIN+ EXECUTE format('ALTER TABLE %s RENAME TO query_execution_old;',+ (SELECT table_name FROM information_schema.tables+ WHERE table_name LIKE 'query_execution_%'+ ORDER BY table_name ASC LIMIT 1));+ END+ $$ language plpgsql;diff --git a/src/metabase/audit_app/init.clj b/src/metabase/audit_app/init.cljindex 5c5b57f2b67..1adc38932ee 100644--- a/src/metabase/audit_app/init.clj+++ b/src/metabase/audit_app/init.clj@@ -2,4 +2,5 @@(:require[metabase.audit-app.events.audit-log][metabase.audit-app.settings]+ [metabase.audit-app.task.partitions][metabase.audit-app.task.truncate-audit-tables]))diff --git a/src/metabase/audit_app/task/partitions.clj b/src/metabase/audit_app/task/partitions.cljnew file mode 100644index 00000000000..5536fb0f6de--- /dev/null+++ b/src/metabase/audit_app/task/partitions.clj@@ -0,0 +1,112 @@+(ns metabase.audit-app.task.partitions+ "The query_execution table is partitioned by date for performance.+ As time goes on, we need to ensure that new partitions are added and old ones+ get dropped as they cycle out of the retention window.+ All public functions here are public only for testing purposes."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t]+ [metabase.app-db.core :as mdb]+ [metabase.audit-app.settings :as settings]+ [metabase.task.core :as task]+ [next.jdbc :as next.jdbc]))++(set! *warn-on-reflection* true)++;;; pure functions++(defn- partition-for [date]+ (format "query_execution_%s_%02d" (t/year date) (.getValue (t/month date))))++(defn- start-date-for-partition [partition]+ ;; partitions are named like "query_execution_2026_07"+ (let [[_ year month] (re-find #"query_execution_(\d+)_(\d+)" partition)]+ (t/local-date (parse-long year) (parse-long month))))++(defn partitions-to-create+ "Given current partitions and the current time, what new partitions are needed?+ Returns a sequence of maps with :name, :from, and :to keys."+ [partitions now]+ (let [latest-date (apply t/max (t/local-date 1970)+ (map start-date-for-partition partitions))]+ ;; current month created in case you shut down metabase for over a month+ (for [date [(t/adjust now :first-day-of-month)+ (t/adjust now :first-day-of-next-month)]+ :let [name (partition-for date)]+ :when (t/after? date latest-date)]+ {:name name+ :from (t/format "yyyy-MM-dd" date)+ :to (t/format "yyyy-MM-dd" (t/adjust date :first-day-of-next-month))})))++(defn- keep? [retention-earliest partition]+ (let [end-date (t/adjust (start-date-for-partition partition)+ :first-day-of-next-month)]+ (t/after? end-date retention-earliest)))++(defn partitions-to-drop+ "Given current partitions, time and retention days, which are due to be dropped?"+ [partitions now retention-days]+ (if (or (nil? retention-days) (zero? retention-days))+ []+ (let [retention-earliest (t/minus now (t/days retention-days))]+ (remove (partial keep? retention-earliest) partitions))))++;;; and now for the side-effects...++(defn create-partition+ "Attach a new partition to query_execution."+ [conn {:keys [name from to]}]+ (next.jdbc/execute! conn [(format "CREATE TABLE \"%s\" PARTITION OF query_execution+ FOR VALUES FROM ('%s') TO ('%s')" name from to)]))++(defn- drop-partition [conn partition]+ (next.jdbc/execute! conn [(format "DROP TABLE \"%s\"" partition)]))++(defn current-partitions+ "List the names of the currently-attached query_execution partition tables."+ [conn]+ (map :pg_class/relname+ (next.jdbc/execute! conn ["SELECT c.relname FROM pg_inherits i+ JOIN pg_class p ON i.inhparent = p.oid+ JOIN pg_class c ON i.inhrelid = c.oid+ WHERE p.relname='query_execution'"])))++(defn manage-partitions+ "Create new partitions and drop unnecessary partitions under Postgres."+ [conn now retention-days]+ (let [partitions (current-partitions conn)]+ (doseq [partition (partitions-to-create partitions now)]+ (create-partition conn partition))+ (doseq [partition (partitions-to-drop partitions now retention-days)]+ (drop-partition conn partition))))++;;; scheduling the jobs++(task/defjob ^:private ^{org.quartz.DisallowConcurrentExecution true}+ ManagePartitions [_]+ (when (= :postgres (mdb/db-type)) ; mysql/h2 don't have partitions+ (let [retention-days (settings/audit-max-retention-days)]+ (with-open [conn (.getConnection (mdb/data-source))]+ (manage-partitions conn (t/local-date) retention-days)))))++(defmethod task/init! ::ManagePartitions [_]+ (let [job-key (jobs/key "metabase.app-db.partitions.job")+ trigger-key (triggers/key "metabase.app-db.partitions.trigger")+ job (jobs/build+ (jobs/of-type ManagePartitions)+ (jobs/with-identity job-key)+ (jobs/with-description "Manage DB partitions")+ (jobs/store-durably))+ trigger (triggers/build+ (triggers/with-identity trigger-key)+ (triggers/for-job job-key)+ (triggers/start-now)+ ;; 3:21AM every day+ (triggers/with-schedule+ (cron/schedule+ (cron/cron-schedule "0 21 3 * * ? *")+ ;; we want to make sure it runs if it was shut down for a month+ (cron/with-misfire-handling-instruction-fire-and-proceed))))]+ (task/schedule-task! job trigger)))diff --git a/src/metabase/audit_app/task/truncate_audit_tables.clj b/src/metabase/audit_app/task/truncate_audit_tables.cljindex bfea6734d7d..f0de7e38164 100644--- a/src/metabase/audit_app/task/truncate_audit_tables.clj+++ b/src/metabase/audit_app/task/truncate_audit_tables.clj@@ -65,7 +65,10 @@"List of models to truncate. OSS implementation only truncates `query_execution` table."metabase-enterprise.audit-app.task.truncate-audit-tables[]- [{:model :model/QueryExecution :timestamp-col :started_at}])+ ;; postgres has partitioned query_execution; we detach partitions instead of deleting+ (if (= :postgres (mdb/db-type))+ []+ [{:model :model/QueryExecution :timestamp-col :started_at}]))(defn- truncate-audit-tables![]diff --git a/test/metabase/audit_app/task/partitions_test.clj b/test/metabase/audit_app/task/partitions_test.cljnew file mode 100644index 00000000000..2c9be5561d6--- /dev/null+++ b/test/metabase/audit_app/task/partitions_test.clj@@ -0,0 +1,98 @@+(ns metabase.audit-app.task.partitions-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.app-db.core :as mdb]+ [metabase.audit-app.task.partitions :as partitions]+ [next.jdbc :as next.jdbc]))++(set! *warn-on-reflection* true)++(defn partitions-for-months [& months]+ (set (for [m months] (format "query_execution_2026_%02d" m))))++(def now (t/local-date 2026 07 01))+(def mid-month (t/local-date 2026 07 15))+(def next-month (t/local-date 2026 8 01))+(def nnext-month (t/local-date 2026 9 01))++(deftest partitions-to-create-test+ (is (= [] (partitions/partitions-to-create ["query_execution_2026_08"] now)))+ (let [existing (partitions-for-months 7 8)]+ (is (= [] (partitions/partitions-to-create existing now)))+ (is (= [] (partitions/partitions-to-create existing mid-month)))+ (is (= [{:name "query_execution_2026_09" :from "2026-09-01" :to "2026-10-01"}]+ (partitions/partitions-to-create existing next-month)))+ (is (= [{:name "query_execution_2026_09" :from "2026-09-01" :to "2026-10-01"}+ {:name "query_execution_2026_10" :from "2026-10-01" :to "2026-11-01"}]+ (partitions/partitions-to-create existing nnext-month)))))++(deftest partitions-to-drop-test+ (let [existing (partitions-for-months 4 5 6 7)]+ (is (= [] (partitions/partitions-to-drop existing now 0)))+ (is (= (partitions-for-months 4 5)+ (set (partitions/partitions-to-drop existing now 7))))+ (is (= (partitions-for-months 4 5 6)+ (set (partitions/partitions-to-drop existing mid-month 7))))+ (is (= (partitions-for-months 4 5)+ (set (partitions/partitions-to-drop existing now 30))))+ (is (= [] (partitions/partitions-to-drop existing now 100)))+ (testing "future months"+ (is (= (partitions-for-months 4 5 6)+ (set (partitions/partitions-to-drop existing next-month 7))))+ (is (= (partitions-for-months 4 5 6)+ (set (partitions/partitions-to-drop existing next-month 30))))+ (is (= (partitions-for-months 4)+ (set (partitions/partitions-to-drop existing next-month 90))))+ (is (= (partitions-for-months 4 5 6 7)+ (set (partitions/partitions-to-drop existing nnext-month 30))))+ (is (= (partitions-for-months 4)+ (set (partitions/partitions-to-drop existing nnext-month 100)))))))++(defn drop-all-tables [conn]+ ;; can't use current-partitions because we want to get rid of them regardless+ ;; of whether they're attached currently or not+ (doseq [t (next.jdbc/execute! conn ["SELECT table_name FROM information_schema.tables+ WHERE table_name LIKE 'query_execution_%'"])]+ (next.jdbc/execute! conn [(format "DROP TABLE %s" (:tables/table_name t))])))++(deftest manage-partitions+ (when (= :postgres (mdb/db-type))+ (with-open [conn (.getConnection (mdb/data-source))]+ (try+ (testing "Normal progression of time"+ (drop-all-tables conn)+ (partitions/manage-partitions conn now 25)+ (is (= #{"query_execution_2026_07"+ "query_execution_2026_08"}+ (set (partitions/current-partitions conn))))+ (partitions/manage-partitions conn next-month 25)+ (is (= #{"query_execution_2026_07"+ "query_execution_2026_08"+ "query_execution_2026_09"}+ (set (partitions/current-partitions conn))))+ (partitions/manage-partitions conn nnext-month 45)+ (is (= #{"query_execution_2026_07"+ "query_execution_2026_08"+ "query_execution_2026_09"+ "query_execution_2026_10"}+ (set (partitions/current-partitions conn))))+ (partitions/manage-partitions conn nnext-month 25)+ (is (= #{"query_execution_2026_08"+ "query_execution_2026_09"+ "query_execution_2026_10"}+ (set (partitions/current-partitions conn)))))+ (testing "Initial run with a multi-month legacy partition"+ (drop-all-tables conn)+ (partitions/create-partition conn {:name "query_execution_2026_08"+ :from "1970-01-01"+ :to "2026-08-01"})+ (partitions/manage-partitions conn now 25)+ (is (= #{"query_execution_2026_08"}+ (set (partitions/current-partitions conn)))))+ (finally+ (drop-all-tables conn)+ ;; make sure we end up in a good state for other tests+ (partitions/create-partition conn {:name "query_execution_2026_08"+ :from "1970-01-01"+ :to "2099-01-01"}))))))diff --git a/test/metabase/audit_app/task/truncate_audit_tables_test.clj b/test/metabase/audit_app/task/truncate_audit_tables_test.cljindex 9f77d39b483..8c6898076c4 100644--- a/test/metabase/audit_app/task/truncate_audit_tables_test.clj+++ b/test/metabase/audit_app/task/truncate_audit_tables_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][java-time.api :as t]+ [metabase.app-db.core :as mdb][metabase.audit-app.task.truncate-audit-tables :as task.truncate-audit-tables][metabase.premium-features.core :as premium-features][metabase.query-processor.util :as qp.util]@@ -33,41 +34,47 @@;; 1 year ago:model/QueryExecution {qe3-id :id} (merge (query-execution-defaults){:started_at (t/minus (t/offset-date-time) (t/years 1))})]- ;; Mock a cloud environment so that we can change the setting value via env var- (mt/with-dynamic-fn-redefs [premium-features/is-hosted? (constantly true)]- (testing "When the threshold is 0 (representing infinity), no rows are deleted"- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0]- (#'task.truncate-audit-tables/truncate-audit-tables!)- (is (= #{qe1-id qe2-id qe3-id}- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))- (testing "When the threshold is 100 days, one row is deleted"- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 100]+ (if (= :postgres (mdb/db-type))+ (testing "When on postgres, nothing is truncated."+ (#'task.truncate-audit-tables/truncate-audit-tables!)+ (is (= #{qe1-id qe2-id qe3-id}+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))+ ;; Mock a cloud environment so that we can change the setting value via env var+ (mt/with-dynamic-fn-redefs [premium-features/is-hosted? (constantly true)]+ (testing "When the threshold is 0 (representing infinity), no rows are deleted"+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0](#'task.truncate-audit-tables/truncate-audit-tables!)- (is (= #{qe1-id qe2-id}- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))- (testing "When the threshold is 30 days, two rows are deleted"- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 30]- (#'task.truncate-audit-tables/truncate-audit-tables!)- (is (= #{qe1-id}- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))- (testing "When the threshold set to 1 day, the remaining row is not deleted because the minimum threshold is 30"- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 1]- (#'task.truncate-audit-tables/truncate-audit-tables!)- (is (= #{qe1-id}- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))))))))+ (is (= #{qe1-id qe2-id qe3-id}+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))+ (testing "When the threshold is 100 days, one row is deleted"+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 100]+ (#'task.truncate-audit-tables/truncate-audit-tables!)+ (is (= #{qe1-id qe2-id}+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))+ (testing "When the threshold is 30 days, two rows are deleted"+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 30]+ (#'task.truncate-audit-tables/truncate-audit-tables!)+ (is (= #{qe1-id}+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))+ (testing "When the threshold set to 1 day, the remaining row is not deleted because the minimum threshold is 30"+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 1]+ (#'task.truncate-audit-tables/truncate-audit-tables!)+ (is (= #{qe1-id}+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))))))))))(deftest batched-deletion-test(testing "Deletion is batched into multiple queries"- (mt/with-temporary-setting-values [audit-table-truncation-batch-size 2]- (mt/with-temp- [:model/QueryExecution {qe1-id :id} (merge (query-execution-defaults)- {:started_at (t/minus (t/offset-date-time) (t/years 4))})- :model/QueryExecution {qe2-id :id} (merge (query-execution-defaults)- {:started_at (t/minus (t/offset-date-time) (t/years 4))})- :model/QueryExecution {qe3-id :id} (merge (query-execution-defaults)- {:started_at (t/minus (t/offset-date-time) (t/years 4))})]- (t2/with-call-count [call-count]- (#'task.truncate-audit-tables/truncate-table! :model/QueryExecution :started_at)- ;; Should make deletion queries in two batches (2 rows, then 1 row)- (is (= 2 (call-count)))- (is (nil? (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))))))+ (when (not= :postgres (mdb/db-type))+ (mt/with-temporary-setting-values [audit-table-truncation-batch-size 2]+ (mt/with-temp+ [:model/QueryExecution {qe1-id :id} (merge (query-execution-defaults)+ {:started_at (t/minus (t/offset-date-time) (t/years 4))})+ :model/QueryExecution {qe2-id :id} (merge (query-execution-defaults)+ {:started_at (t/minus (t/offset-date-time) (t/years 4))})+ :model/QueryExecution {qe3-id :id} (merge (query-execution-defaults)+ {:started_at (t/minus (t/offset-date-time) (t/years 4))})]+ (t2/with-call-count [call-count]+ (#'task.truncate-audit-tables/truncate-table! :model/QueryExecution :started_at)+ ;; Should make deletion queries in two batches (2 rows, then 1 row)+ (is (= 2 (call-count)))+ (is (nil? (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))))))