Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Continue legacy truncation for partitioned tables until fully switched (#82790)

technomancyb19ae810463c
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj
index de151ca48e6..ee6b8883cb1 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj
@@ -1,6 +1,6 @@
(ns metabase-enterprise.audit-app.task.truncate-audit-tables
(:require
- [metabase.app-db.core :as mdb]
+ [metabase.audit-app.task.partitions :as partitions]
[metabase.premium-features.core :refer [defenterprise]]))
(defenterprise audit-models-to-truncate
@@ -13,7 +13,7 @@
:feature :audit-app
[]
;; postgres has partitioned query_execution; we detach partitions instead of deleting
- (if (= :postgres (mdb/db-type))
+ (if (partitions/fully-on-partitions?)
[{:model :model/AuditLog :timestamp-col :timestamp}
{:model :model/ViewLog :timestamp-col :timestamp}]
[{:model :model/QueryExecution :timestamp-col :started_at}
diff --git a/src/metabase/audit_app/task/partitions.clj b/src/metabase/audit_app/task/partitions.clj
index 5536fb0f6de..904cb77d8db 100644
--- a/src/metabase/audit_app/task/partitions.clj
+++ b/src/metabase/audit_app/task/partitions.clj
@@ -48,11 +48,37 @@
(defn partitions-to-drop
"Given current partitions, time and retention days, which are due to be dropped?"
[partitions now retention-days]
- (if (or (nil? retention-days) (zero? retention-days))
+ (if (or (nil? retention-days) (zero? retention-days) (infinite? retention-days))
[]
(let [retention-earliest (t/minus now (t/days retention-days))]
(remove (partial keep? retention-earliest) partitions))))
+(defn current-partitions
+ "List the names of the currently-attached query_execution partition tables."
+ [conn]
+ (map :pg_class/relname
+ (next.jdbc/execute! conn ["SELECT c.relname FROM pg_inherits i
+ JOIN pg_class p ON i.inhparent = p.oid
+ JOIN pg_class c ON i.inhrelid = c.oid
+ WHERE p.relname='query_execution'"])))
+
+(defn fully-on-partitions?
+ "Has the database fully switched to managing its truncation with partitions?
+ If the database is using partitions but they're still in the first partition
+ then we need the legacy truncation to continue to operate, because the first
+ partition *is* the old table and contains rows going back much further than
+ a single partition's worth."
+ []
+ (and (= :postgres (mdb/db-type))
+ (with-open [conn (.getConnection (mdb/data-source))]
+ (let [[oldest-partition] (sort (current-partitions conn))
+ [oldest] (next.jdbc/execute! conn [(str "SELECT min(started_at) AS min FROM "
+ oldest-partition)])
+ oldest-date-time (some-> (:min oldest) t/instant
+ (t/local-date-time "UTC"))]
+ (or (nil? oldest-date-time)
+ (= oldest-partition (partition-for oldest-date-time)))))))
+
;;; and now for the side-effects...
(defn create-partition
@@ -64,15 +90,6 @@
(defn- drop-partition [conn partition]
(next.jdbc/execute! conn [(format "DROP TABLE \"%s\"" partition)]))
-(defn current-partitions
- "List the names of the currently-attached query_execution partition tables."
- [conn]
- (map :pg_class/relname
- (next.jdbc/execute! conn ["SELECT c.relname FROM pg_inherits i
- JOIN pg_class p ON i.inhparent = p.oid
- JOIN pg_class c ON i.inhrelid = c.oid
- WHERE p.relname='query_execution'"])))
-
(defn manage-partitions
"Create new partitions and drop unnecessary partitions under Postgres."
[conn now retention-days]
diff --git a/src/metabase/audit_app/task/truncate_audit_tables.clj b/src/metabase/audit_app/task/truncate_audit_tables.clj
index 8a0844c5adb..e351a7ab8e1 100644
--- a/src/metabase/audit_app/task/truncate_audit_tables.clj
+++ b/src/metabase/audit_app/task/truncate_audit_tables.clj
@@ -9,6 +9,7 @@
[metabase.app-db.core :as mdb]
[metabase.audit-app.db :as audit-app.db]
[metabase.audit-app.settings :as audit-app.settings]
+ [metabase.audit-app.task.partitions :as partitions]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.task-history.core :as task-history]
[metabase.task.core :as task]
@@ -55,7 +56,7 @@
metabase-enterprise.audit-app.task.truncate-audit-tables
[]
;; postgres has partitioned query_execution; we detach partitions instead of deleting
- (if (= :postgres (mdb/db-type))
+ (if (partitions/fully-on-partitions?)
[]
[{:model :model/QueryExecution :timestamp-col :started_at}]))
diff --git a/test/metabase/audit_app/task/partitions_test.clj b/test/metabase/audit_app/task/partitions_test.clj
index 2c9be5561d6..9f313f61188 100644
--- a/test/metabase/audit_app/task/partitions_test.clj
+++ b/test/metabase/audit_app/task/partitions_test.clj
@@ -4,7 +4,8 @@
[java-time.api :as t]
[metabase.app-db.core :as mdb]
[metabase.audit-app.task.partitions :as partitions]
- [next.jdbc :as next.jdbc]))
+ [next.jdbc :as next.jdbc]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -30,6 +31,7 @@
(deftest partitions-to-drop-test
(let [existing (partitions-for-months 4 5 6 7)]
(is (= [] (partitions/partitions-to-drop existing now 0)))
+ (is (= [] (partitions/partitions-to-drop existing now ##Inf)))
(is (= (partitions-for-months 4 5)
(set (partitions/partitions-to-drop existing now 7))))
(is (= (partitions-for-months 4 5 6)
@@ -96,3 +98,54 @@
(partitions/create-partition conn {:name "query_execution_2026_08"
:from "1970-01-01"
:to "2099-01-01"}))))))
+
+(defn insert-execution! [started-at]
+ (t2/insert! :model/QueryExecution {:started_at started-at
+ :hash (.getBytes "lol")
+ :result_rows 0
+ :running_time 2
+ :context :exploration
+ :native false}))
+
+;; this simulates an old install that has a bunch of query_execution rows that
+;; switches over to partitioned tables. we want to make sure that we can detect
+;; fully-on-partitions? in these cases.
+(deftest initial-partition-to-fully-managed
+ (when (= :postgres (mdb/db-type))
+ (with-open [conn (.getConnection (mdb/data-source))]
+ (try
+ (testing "initial partition"
+ (drop-all-tables conn)
+ ;; initial partition, upper bound is exclusive
+ (partitions/create-partition conn {:name "query_execution_2026_09"
+ :from "1970-01-01"
+ :to "2026-10-01"})
+ ;; successive partition
+ (partitions/create-partition conn {:name "query_execution_2026_10"
+ :from "2026-10-01"
+ :to "2026-11-01"})
+ ;; initial data
+ (insert-execution! #t "2026-02-01")
+ (insert-execution! #t "2026-07-01")
+ (insert-execution! #t "2026-08-01")
+ (insert-execution! #t "2026-09-01")
+ (insert-execution! #t "2026-10-02")
+ (is (not (partitions/fully-on-partitions?))))
+ (testing "after second partition is added"
+ (partitions/manage-partitions conn #t "2026-09-01" 25)
+ (is (not (partitions/fully-on-partitions?))))
+ (testing "after third partition is added"
+ ;; we won't drop the initial partition yet because its end date is
+ ;; still within the 25-day retention window
+ (partitions/manage-partitions conn #t "2026-10-01" 25)
+ (is (not (partitions/fully-on-partitions?))))
+ (testing "after fourth partition is added"
+ ;; by now the rows from 2026-10-02 have cycled out of retention
+ (partitions/manage-partitions conn #t "2026-11-01" 25)
+ (is (partitions/fully-on-partitions?)))
+ (finally
+ (drop-all-tables conn)
+ ;; make sure we end up in a good state for other tests
+ (partitions/create-partition conn {:name "query_execution_2026_08"
+ :from "1970-01-01"
+ :to "2099-01-01"}))))))