Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Partition the query_execution table when using Postgres AppDB. (#78497)

technomancyc28f952435ab
diff --git a/bin/lint-migrations-file/src/change_set/strict.clj b/bin/lint-migrations-file/src/change_set/strict.clj
index 10c3029347c..6dce2319558 100644
--- a/bin/lint-migrations-file/src/change_set/strict.clj
+++ b/bin/lint-migrations-file/src/change_set/strict.clj
@@ -47,7 +47,12 @@
(s/def ::changes
(s/or
- ;; only one change allowed per change set for the strict schema.
+ ;; only one change allowed per change set for the strict schema:
+ ;; "Dumb DBs like MySQL can't do DDL in transactions and auto commit each
+ ;; change, so if you do multiple changes in one changeset it can commit the
+ ;; first few changes and then fail. The changes that succeeded don't get
+ ;; rolled back. Then you're in a state where the change set was partially
+ ;; applied but failed and rerunning it is going to be ultra-busted"
:one-change
(s/alt :change :change.strict/change)
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj
index b9ccc20a107..de151ca48e6 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.audit-app.task.truncate-audit-tables
(:require
+ [metabase.app-db.core :as mdb]
[metabase.premium-features.core :refer [defenterprise]]))
(defenterprise audit-models-to-truncate
@@ -11,6 +12,10 @@
`metabase-enterprise.mcp.task.mcp-usage-trimmer` (keyed on `ai-usage-max-retention-days`)."
:feature :audit-app
[]
- [{:model :model/QueryExecution :timestamp-col :started_at}
- {:model :model/AuditLog :timestamp-col :timestamp}
- {:model :model/ViewLog :timestamp-col :timestamp}])
+ ;; postgres has partitioned query_execution; we detach partitions instead of deleting
+ (if (= :postgres (mdb/db-type))
+ [{:model :model/AuditLog :timestamp-col :timestamp}
+ {:model :model/ViewLog :timestamp-col :timestamp}]
+ [{:model :model/QueryExecution :timestamp-col :started_at}
+ {:model :model/AuditLog :timestamp-col :timestamp}
+ {:model :model/ViewLog :timestamp-col :timestamp}]))
diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.clj
index fcbf8d1789c..b76d5899d59 100644
--- a/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/audit_app/task/truncate_audit_tables_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase.app-db.core :as mdb]
[metabase.audit-app.task.truncate-audit-tables :as task.truncate-audit-tables]
[metabase.premium-features.core :as premium-features]
[metabase.query-processor.util :as qp.util]
@@ -32,13 +33,18 @@
;; 1 year ago
:model/QueryExecution {qe3-id :id} (merge (query-execution-defaults)
{:started_at (t/minus (t/offset-date-time) (t/years 1))})]
- ;; Mock a cloud environment so that we can change the setting value via env var
- (with-redefs [premium-features/is-hosted? (constantly true)]
- (testing "When the threshold is 0 (representing infinity), no rows are deleted"
- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0]
- (#'task.truncate-audit-tables/truncate-audit-tables!)
- (is (= #{qe1-id qe2-id qe3-id}
- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))))))))
+ (if (= :postgres (mdb/db-type))
+ (testing "When on postgres, nothing in query_execution is truncated."
+ (#'task.truncate-audit-tables/truncate-audit-tables!)
+ (is (= #{qe1-id qe2-id qe3-id}
+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))
+ ;; Mock a cloud environment so that we can change the setting value via env var
+ (with-redefs [premium-features/is-hosted? (constantly true)]
+ (testing "When the threshold is 0 (representing infinity), no rows are deleted"
+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0]
+ (#'task.truncate-audit-tables/truncate-audit-tables!)
+ (is (= #{qe1-id qe2-id qe3-id}
+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))))))))
(defn- audit-log-defaults
[]
diff --git a/resources/migrations/064/20260723_query_execution_partitioned.yaml b/resources/migrations/064/20260723_query_execution_partitioned.yaml
new file mode 100644
index 00000000000..20decb6dca2
--- /dev/null
+++ b/resources/migrations/064/20260723_query_execution_partitioned.yaml
@@ -0,0 +1,142 @@
+# The query_execution table needs to be partitioned on postgres.
+#
+# Attach the query_execution_old table as a partition. This partition covers up
+# to the current month, plus one more after, in order to handle the case where
+# the migration runs at the end of the month and query executions need to be
+# written to disk before the job to create the next partition gets a chance to
+# run.
+#
+# The IDENTITY has to be dropped since this can't be done in a partition; it
+# should be handled by the higher level table.
+#
+# The old query_execution table is now a partition of the new query_execution
+# table, but the v_query_log view still references the old one. Re-creating it
+# will point it at the new one.
+#
+# We should use sqlFile for this, but that is blocked because you can't mix sql
+# and sqlFile in the same migration because mysql sucks, so sadly we have to
+# copy/paste the entire contents of postgres-query_log.sql into here instead.
+# Splitting into a separate file doesn't work because the rollback order would
+# be wrong; it needs to be after this one for going forward but before this one
+# for rollback.
+databaseChangeLog:
+ - changeSet:
+ id: v64.2026-07-23T11:39:56
+ author: technomancy
+ comment: Create partitioned query_execution table and attach old one
+ dbms: postgresql
+ changes:
+ - sql:
+ sql: >-
+ ALTER TABLE query_execution DROP CONSTRAINT query_execution_pkey;
+ ALTER TABLE query_execution RENAME TO query_execution_old;
+ CREATE TABLE query_execution (LIKE query_execution_old INCLUDING ALL)
+ PARTITION BY RANGE (started_at);
+ SELECT setval(pg_get_serial_sequence('query_execution', 'id'),
+ nextval(pg_get_serial_sequence('query_execution_old', 'id')));
+ ALTER TABLE query_execution_old ALTER COLUMN id DROP IDENTITY;
+ ALTER TABLE query_execution ATTACH PARTITION query_execution_old
+ FOR VALUES FROM ('1970-01-01')
+ TO (date_trunc('month', current_timestamp + '2 months'::interval));
+
+ DROP VIEW IF EXISTS v_query_log;
+ CREATE OR REPLACE VIEW v_query_log AS
+ SELECT id AS entity_id,
+ started_at,
+ cast(running_time AS double precision) / 1000 AS running_time_seconds,
+ result_rows,
+ native AS is_native,
+ context AS query_source,
+ error,
+ coalesce(executor_id, 0) AS user_id,
+ card_id,
+ 'card_' || card_id AS card_qualified_id,
+ dashboard_id,
+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,
+ pulse_id,
+ database_id,
+ 'database_' || database_id AS database_qualified_id,
+ cache_hit,
+ action_id,
+ 'action_' || action_id AS action_qualified_id,
+ transform_id,
+ 'transform_' || transform_id AS transform_qualified_id,
+ lens_id,
+ lens_params,
+ query,
+ embedding_client,
+ embedding_client_identifier,
+ embedding_route,
+ CASE WHEN embedding_client LIKE '%-preview' THEN TRUE ELSE FALSE END AS is_preview,
+ embedding_sdk_version,
+ metabase_version,
+ auth_method,
+ is_sandboxed,
+ is_impersonated,
+ is_db_routed,
+ parameters,
+ tenant_id,
+ embedding_hostname,
+ embedding_path,
+ user_agent,
+ sanitized_user_agent,
+ ip_address
+ FROM query_execution
+ LEFT JOIN query ON query_execution.hash = query.query_hash;
+ rollback:
+ - sql:
+ sql: >-
+ DROP VIEW IF EXISTS v_query_log;
+ ALTER TABLE query_execution DETACH PARTITION query_execution_old;
+ ALTER TABLE query_execution_old ALTER COLUMN id
+ ADD GENERATED BY DEFAULT AS IDENTITY;
+ SELECT setval(pg_get_serial_sequence('query_execution_old', 'id'),
+ nextval(pg_get_serial_sequence('query_execution', 'id')));
+ DROP TABLE query_execution;
+ ALTER TABLE query_execution_old RENAME TO query_execution;
+ ALTER TABLE query_execution
+ ADD CONSTRAINT query_execution_pkey PRIMARY KEY(ID);
+
+ CREATE OR REPLACE VIEW v_query_log AS
+ SELECT id AS entity_id,
+ started_at,
+ cast(running_time AS double precision) / 1000 AS running_time_seconds,
+ result_rows,
+ native AS is_native,
+ context AS query_source,
+ error,
+ coalesce(executor_id, 0) AS user_id,
+ card_id,
+ 'card_' || card_id AS card_qualified_id,
+ dashboard_id,
+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,
+ pulse_id,
+ database_id,
+ 'database_' || database_id AS database_qualified_id,
+ cache_hit,
+ action_id,
+ 'action_' || action_id AS action_qualified_id,
+ transform_id,
+ 'transform_' || transform_id AS transform_qualified_id,
+ lens_id,
+ lens_params,
+ query,
+ embedding_client,
+ embedding_client_identifier,
+ embedding_route,
+ CASE WHEN embedding_client LIKE '%-preview' THEN TRUE ELSE FALSE END AS is_preview,
+ embedding_sdk_version,
+ metabase_version,
+ auth_method,
+ is_sandboxed,
+ is_impersonated,
+ is_db_routed,
+ parameters,
+ tenant_id,
+ embedding_hostname,
+ embedding_path,
+ user_agent,
+ sanitized_user_agent,
+ ip_address
+ FROM query_execution
+ LEFT JOIN query ON query_execution.hash = query.query_hash;
diff --git a/resources/migrations/064/20260725_rename_query_execution_partition.yaml b/resources/migrations/064/20260725_rename_query_execution_partition.yaml
new file mode 100644
index 00000000000..6c1fb06b38d
--- /dev/null
+++ b/resources/migrations/064/20260725_rename_query_execution_partition.yaml
@@ -0,0 +1,36 @@
+# Finally, we want the query_execution_old table to be named after the same
+# pattern as the new partitions will have in order to make the logic for
+# detaching them simpler: query_execution_2026-08 for example if the migration
+# was run in July 2026 (extra month to allow time for jobs to run).
+#
+# But we don't know when the migration will be run, so that has to be
+# calculated dynamically, which requires use of 'EXECUTE'. Because of the
+# complication of dynamic calculation, we delay the rename of the table to the
+# end so only one query needs to be dynamic.
+
+databaseChangeLog:
+ - changeSet:
+ id: v64.2026-07-25T00:39:56
+ author: technomancy
+ comment: Rename old query_execution partition to match naming scheme
+ dbms: postgresql
+ changes:
+ - sql:
+ sql: >-
+ DO $$
+ BEGIN
+ EXECUTE format('ALTER TABLE query_execution_old RENAME TO query_execution_%s;',
+ TO_CHAR(current_timestamp + '1 month'::interval, 'YYYY_MM'));
+ END
+ $$ language plpgsql;
+ rollback:
+ - sql:
+ sql: >-
+ DO $$
+ BEGIN
+ EXECUTE format('ALTER TABLE %s RENAME TO query_execution_old;',
+ (SELECT table_name FROM information_schema.tables
+ WHERE table_name LIKE 'query_execution_%'
+ ORDER BY table_name ASC LIMIT 1));
+ END
+ $$ language plpgsql;
diff --git a/src/metabase/audit_app/init.clj b/src/metabase/audit_app/init.clj
index 5c5b57f2b67..1adc38932ee 100644
--- a/src/metabase/audit_app/init.clj
+++ b/src/metabase/audit_app/init.clj
@@ -2,4 +2,5 @@
(:require
[metabase.audit-app.events.audit-log]
[metabase.audit-app.settings]
+ [metabase.audit-app.task.partitions]
[metabase.audit-app.task.truncate-audit-tables]))
diff --git a/src/metabase/audit_app/task/partitions.clj b/src/metabase/audit_app/task/partitions.clj
new file mode 100644
index 00000000000..5536fb0f6de
--- /dev/null
+++ b/src/metabase/audit_app/task/partitions.clj
@@ -0,0 +1,112 @@
+(ns metabase.audit-app.task.partitions
+ "The query_execution table is partitioned by date for performance.
+ As time goes on, we need to ensure that new partitions are added and old ones
+ get dropped as they cycle out of the retention window.
+ All public functions here are public only for testing purposes."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
+ [metabase.app-db.core :as mdb]
+ [metabase.audit-app.settings :as settings]
+ [metabase.task.core :as task]
+ [next.jdbc :as next.jdbc]))
+
+(set! *warn-on-reflection* true)
+
+;;; pure functions
+
+(defn- partition-for [date]
+ (format "query_execution_%s_%02d" (t/year date) (.getValue (t/month date))))
+
+(defn- start-date-for-partition [partition]
+ ;; partitions are named like "query_execution_2026_07"
+ (let [[_ year month] (re-find #"query_execution_(\d+)_(\d+)" partition)]
+ (t/local-date (parse-long year) (parse-long month))))
+
+(defn partitions-to-create
+ "Given current partitions and the current time, what new partitions are needed?
+ Returns a sequence of maps with :name, :from, and :to keys."
+ [partitions now]
+ (let [latest-date (apply t/max (t/local-date 1970)
+ (map start-date-for-partition partitions))]
+ ;; current month created in case you shut down metabase for over a month
+ (for [date [(t/adjust now :first-day-of-month)
+ (t/adjust now :first-day-of-next-month)]
+ :let [name (partition-for date)]
+ :when (t/after? date latest-date)]
+ {:name name
+ :from (t/format "yyyy-MM-dd" date)
+ :to (t/format "yyyy-MM-dd" (t/adjust date :first-day-of-next-month))})))
+
+(defn- keep? [retention-earliest partition]
+ (let [end-date (t/adjust (start-date-for-partition partition)
+ :first-day-of-next-month)]
+ (t/after? end-date retention-earliest)))
+
+(defn partitions-to-drop
+ "Given current partitions, time and retention days, which are due to be dropped?"
+ [partitions now retention-days]
+ (if (or (nil? retention-days) (zero? retention-days))
+ []
+ (let [retention-earliest (t/minus now (t/days retention-days))]
+ (remove (partial keep? retention-earliest) partitions))))
+
+;;; and now for the side-effects...
+
+(defn create-partition
+ "Attach a new partition to query_execution."
+ [conn {:keys [name from to]}]
+ (next.jdbc/execute! conn [(format "CREATE TABLE \"%s\" PARTITION OF query_execution
+ FOR VALUES FROM ('%s') TO ('%s')" name from to)]))
+
+(defn- drop-partition [conn partition]
+ (next.jdbc/execute! conn [(format "DROP TABLE \"%s\"" partition)]))
+
+(defn current-partitions
+ "List the names of the currently-attached query_execution partition tables."
+ [conn]
+ (map :pg_class/relname
+ (next.jdbc/execute! conn ["SELECT c.relname FROM pg_inherits i
+ JOIN pg_class p ON i.inhparent = p.oid
+ JOIN pg_class c ON i.inhrelid = c.oid
+ WHERE p.relname='query_execution'"])))
+
+(defn manage-partitions
+ "Create new partitions and drop unnecessary partitions under Postgres."
+ [conn now retention-days]
+ (let [partitions (current-partitions conn)]
+ (doseq [partition (partitions-to-create partitions now)]
+ (create-partition conn partition))
+ (doseq [partition (partitions-to-drop partitions now retention-days)]
+ (drop-partition conn partition))))
+
+;;; scheduling the jobs
+
+(task/defjob ^:private ^{org.quartz.DisallowConcurrentExecution true}
+ ManagePartitions [_]
+ (when (= :postgres (mdb/db-type)) ; mysql/h2 don't have partitions
+ (let [retention-days (settings/audit-max-retention-days)]
+ (with-open [conn (.getConnection (mdb/data-source))]
+ (manage-partitions conn (t/local-date) retention-days)))))
+
+(defmethod task/init! ::ManagePartitions [_]
+ (let [job-key (jobs/key "metabase.app-db.partitions.job")
+ trigger-key (triggers/key "metabase.app-db.partitions.trigger")
+ job (jobs/build
+ (jobs/of-type ManagePartitions)
+ (jobs/with-identity job-key)
+ (jobs/with-description "Manage DB partitions")
+ (jobs/store-durably))
+ trigger (triggers/build
+ (triggers/with-identity trigger-key)
+ (triggers/for-job job-key)
+ (triggers/start-now)
+ ;; 3:21AM every day
+ (triggers/with-schedule
+ (cron/schedule
+ (cron/cron-schedule "0 21 3 * * ? *")
+ ;; we want to make sure it runs if it was shut down for a month
+ (cron/with-misfire-handling-instruction-fire-and-proceed))))]
+ (task/schedule-task! job trigger)))
diff --git a/src/metabase/audit_app/task/truncate_audit_tables.clj b/src/metabase/audit_app/task/truncate_audit_tables.clj
index bfea6734d7d..f0de7e38164 100644
--- a/src/metabase/audit_app/task/truncate_audit_tables.clj
+++ b/src/metabase/audit_app/task/truncate_audit_tables.clj
@@ -65,7 +65,10 @@
"List of models to truncate. OSS implementation only truncates `query_execution` table."
metabase-enterprise.audit-app.task.truncate-audit-tables
[]
- [{:model :model/QueryExecution :timestamp-col :started_at}])
+ ;; postgres has partitioned query_execution; we detach partitions instead of deleting
+ (if (= :postgres (mdb/db-type))
+ []
+ [{:model :model/QueryExecution :timestamp-col :started_at}]))
(defn- truncate-audit-tables!
[]
diff --git a/test/metabase/audit_app/task/partitions_test.clj b/test/metabase/audit_app/task/partitions_test.clj
new file mode 100644
index 00000000000..2c9be5561d6
--- /dev/null
+++ b/test/metabase/audit_app/task/partitions_test.clj
@@ -0,0 +1,98 @@
+(ns metabase.audit-app.task.partitions-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.app-db.core :as mdb]
+ [metabase.audit-app.task.partitions :as partitions]
+ [next.jdbc :as next.jdbc]))
+
+(set! *warn-on-reflection* true)
+
+(defn partitions-for-months [& months]
+ (set (for [m months] (format "query_execution_2026_%02d" m))))
+
+(def now (t/local-date 2026 07 01))
+(def mid-month (t/local-date 2026 07 15))
+(def next-month (t/local-date 2026 8 01))
+(def nnext-month (t/local-date 2026 9 01))
+
+(deftest partitions-to-create-test
+ (is (= [] (partitions/partitions-to-create ["query_execution_2026_08"] now)))
+ (let [existing (partitions-for-months 7 8)]
+ (is (= [] (partitions/partitions-to-create existing now)))
+ (is (= [] (partitions/partitions-to-create existing mid-month)))
+ (is (= [{:name "query_execution_2026_09" :from "2026-09-01" :to "2026-10-01"}]
+ (partitions/partitions-to-create existing next-month)))
+ (is (= [{:name "query_execution_2026_09" :from "2026-09-01" :to "2026-10-01"}
+ {:name "query_execution_2026_10" :from "2026-10-01" :to "2026-11-01"}]
+ (partitions/partitions-to-create existing nnext-month)))))
+
+(deftest partitions-to-drop-test
+ (let [existing (partitions-for-months 4 5 6 7)]
+ (is (= [] (partitions/partitions-to-drop existing now 0)))
+ (is (= (partitions-for-months 4 5)
+ (set (partitions/partitions-to-drop existing now 7))))
+ (is (= (partitions-for-months 4 5 6)
+ (set (partitions/partitions-to-drop existing mid-month 7))))
+ (is (= (partitions-for-months 4 5)
+ (set (partitions/partitions-to-drop existing now 30))))
+ (is (= [] (partitions/partitions-to-drop existing now 100)))
+ (testing "future months"
+ (is (= (partitions-for-months 4 5 6)
+ (set (partitions/partitions-to-drop existing next-month 7))))
+ (is (= (partitions-for-months 4 5 6)
+ (set (partitions/partitions-to-drop existing next-month 30))))
+ (is (= (partitions-for-months 4)
+ (set (partitions/partitions-to-drop existing next-month 90))))
+ (is (= (partitions-for-months 4 5 6 7)
+ (set (partitions/partitions-to-drop existing nnext-month 30))))
+ (is (= (partitions-for-months 4)
+ (set (partitions/partitions-to-drop existing nnext-month 100)))))))
+
+(defn drop-all-tables [conn]
+ ;; can't use current-partitions because we want to get rid of them regardless
+ ;; of whether they're attached currently or not
+ (doseq [t (next.jdbc/execute! conn ["SELECT table_name FROM information_schema.tables
+ WHERE table_name LIKE 'query_execution_%'"])]
+ (next.jdbc/execute! conn [(format "DROP TABLE %s" (:tables/table_name t))])))
+
+(deftest manage-partitions
+ (when (= :postgres (mdb/db-type))
+ (with-open [conn (.getConnection (mdb/data-source))]
+ (try
+ (testing "Normal progression of time"
+ (drop-all-tables conn)
+ (partitions/manage-partitions conn now 25)
+ (is (= #{"query_execution_2026_07"
+ "query_execution_2026_08"}
+ (set (partitions/current-partitions conn))))
+ (partitions/manage-partitions conn next-month 25)
+ (is (= #{"query_execution_2026_07"
+ "query_execution_2026_08"
+ "query_execution_2026_09"}
+ (set (partitions/current-partitions conn))))
+ (partitions/manage-partitions conn nnext-month 45)
+ (is (= #{"query_execution_2026_07"
+ "query_execution_2026_08"
+ "query_execution_2026_09"
+ "query_execution_2026_10"}
+ (set (partitions/current-partitions conn))))
+ (partitions/manage-partitions conn nnext-month 25)
+ (is (= #{"query_execution_2026_08"
+ "query_execution_2026_09"
+ "query_execution_2026_10"}
+ (set (partitions/current-partitions conn)))))
+ (testing "Initial run with a multi-month legacy partition"
+ (drop-all-tables conn)
+ (partitions/create-partition conn {:name "query_execution_2026_08"
+ :from "1970-01-01"
+ :to "2026-08-01"})
+ (partitions/manage-partitions conn now 25)
+ (is (= #{"query_execution_2026_08"}
+ (set (partitions/current-partitions conn)))))
+ (finally
+ (drop-all-tables conn)
+ ;; make sure we end up in a good state for other tests
+ (partitions/create-partition conn {:name "query_execution_2026_08"
+ :from "1970-01-01"
+ :to "2099-01-01"}))))))
diff --git a/test/metabase/audit_app/task/truncate_audit_tables_test.clj b/test/metabase/audit_app/task/truncate_audit_tables_test.clj
index 9f77d39b483..8c6898076c4 100644
--- a/test/metabase/audit_app/task/truncate_audit_tables_test.clj
+++ b/test/metabase/audit_app/task/truncate_audit_tables_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase.app-db.core :as mdb]
[metabase.audit-app.task.truncate-audit-tables :as task.truncate-audit-tables]
[metabase.premium-features.core :as premium-features]
[metabase.query-processor.util :as qp.util]
@@ -33,41 +34,47 @@
;; 1 year ago
:model/QueryExecution {qe3-id :id} (merge (query-execution-defaults)
{:started_at (t/minus (t/offset-date-time) (t/years 1))})]
- ;; Mock a cloud environment so that we can change the setting value via env var
- (mt/with-dynamic-fn-redefs [premium-features/is-hosted? (constantly true)]
- (testing "When the threshold is 0 (representing infinity), no rows are deleted"
- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0]
- (#'task.truncate-audit-tables/truncate-audit-tables!)
- (is (= #{qe1-id qe2-id qe3-id}
- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))
- (testing "When the threshold is 100 days, one row is deleted"
- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 100]
+ (if (= :postgres (mdb/db-type))
+ (testing "When on postgres, nothing is truncated."
+ (#'task.truncate-audit-tables/truncate-audit-tables!)
+ (is (= #{qe1-id qe2-id qe3-id}
+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))
+ ;; Mock a cloud environment so that we can change the setting value via env var
+ (mt/with-dynamic-fn-redefs [premium-features/is-hosted? (constantly true)]
+ (testing "When the threshold is 0 (representing infinity), no rows are deleted"
+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 0]
(#'task.truncate-audit-tables/truncate-audit-tables!)
- (is (= #{qe1-id qe2-id}
- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))
- (testing "When the threshold is 30 days, two rows are deleted"
- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 30]
- (#'task.truncate-audit-tables/truncate-audit-tables!)
- (is (= #{qe1-id}
- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))
- (testing "When the threshold set to 1 day, the remaining row is not deleted because the minimum threshold is 30"
- (mt/with-temp-env-var-value! [mb-audit-max-retention-days 1]
- (#'task.truncate-audit-tables/truncate-audit-tables!)
- (is (= #{qe1-id}
- (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))))))))
+ (is (= #{qe1-id qe2-id qe3-id}
+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))
+ (testing "When the threshold is 100 days, one row is deleted"
+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 100]
+ (#'task.truncate-audit-tables/truncate-audit-tables!)
+ (is (= #{qe1-id qe2-id}
+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))
+ (testing "When the threshold is 30 days, two rows are deleted"
+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 30]
+ (#'task.truncate-audit-tables/truncate-audit-tables!)
+ (is (= #{qe1-id}
+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))
+ (testing "When the threshold set to 1 day, the remaining row is not deleted because the minimum threshold is 30"
+ (mt/with-temp-env-var-value! [mb-audit-max-retention-days 1]
+ (#'task.truncate-audit-tables/truncate-audit-tables!)
+ (is (= #{qe1-id}
+ (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))))))))))
(deftest batched-deletion-test
(testing "Deletion is batched into multiple queries"
- (mt/with-temporary-setting-values [audit-table-truncation-batch-size 2]
- (mt/with-temp
- [:model/QueryExecution {qe1-id :id} (merge (query-execution-defaults)
- {:started_at (t/minus (t/offset-date-time) (t/years 4))})
- :model/QueryExecution {qe2-id :id} (merge (query-execution-defaults)
- {:started_at (t/minus (t/offset-date-time) (t/years 4))})
- :model/QueryExecution {qe3-id :id} (merge (query-execution-defaults)
- {:started_at (t/minus (t/offset-date-time) (t/years 4))})]
- (t2/with-call-count [call-count]
- (#'task.truncate-audit-tables/truncate-table! :model/QueryExecution :started_at)
- ;; Should make deletion queries in two batches (2 rows, then 1 row)
- (is (= 2 (call-count)))
- (is (nil? (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]}))))))))
+ (when (not= :postgres (mdb/db-type))
+ (mt/with-temporary-setting-values [audit-table-truncation-batch-size 2]
+ (mt/with-temp
+ [:model/QueryExecution {qe1-id :id} (merge (query-execution-defaults)
+ {:started_at (t/minus (t/offset-date-time) (t/years 4))})
+ :model/QueryExecution {qe2-id :id} (merge (query-execution-defaults)
+ {:started_at (t/minus (t/offset-date-time) (t/years 4))})
+ :model/QueryExecution {qe3-id :id} (merge (query-execution-defaults)
+ {:started_at (t/minus (t/offset-date-time) (t/years 4))})]
+ (t2/with-call-count [call-count]
+ (#'task.truncate-audit-tables/truncate-table! :model/QueryExecution :started_at)
+ ;; Should make deletion queries in two batches (2 rows, then 1 row)
+ (is (= 2 (call-count)))
+ (is (nil? (t2/select-fn-set :id :model/QueryExecution {:where [:in :id [qe1-id qe2-id qe3-id]]})))))))))