The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Continue legacy truncation for partitioned tables until fully switched (#82790)
technomancyb19ae810463c
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.cljindex de151ca48e6..ee6b8883cb1 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/task/truncate_audit_tables.clj@@ -1,6 +1,6 @@(ns metabase-enterprise.audit-app.task.truncate-audit-tables(:require- [metabase.app-db.core :as mdb]+ [metabase.audit-app.task.partitions :as partitions][metabase.premium-features.core :refer [defenterprise]]))(defenterprise audit-models-to-truncate@@ -13,7 +13,7 @@:feature :audit-app[];; postgres has partitioned query_execution; we detach partitions instead of deleting- (if (= :postgres (mdb/db-type))+ (if (partitions/fully-on-partitions?)[{:model :model/AuditLog :timestamp-col :timestamp}{:model :model/ViewLog :timestamp-col :timestamp}][{:model :model/QueryExecution :timestamp-col :started_at}diff --git a/src/metabase/audit_app/task/partitions.clj b/src/metabase/audit_app/task/partitions.cljindex 5536fb0f6de..904cb77d8db 100644--- a/src/metabase/audit_app/task/partitions.clj+++ b/src/metabase/audit_app/task/partitions.clj@@ -48,11 +48,37 @@(defn partitions-to-drop"Given current partitions, time and retention days, which are due to be dropped?"[partitions now retention-days]- (if (or (nil? retention-days) (zero? retention-days))+ (if (or (nil? retention-days) (zero? retention-days) (infinite? retention-days))[](let [retention-earliest (t/minus now (t/days retention-days))](remove (partial keep? retention-earliest) partitions))))+(defn current-partitions+ "List the names of the currently-attached query_execution partition tables."+ [conn]+ (map :pg_class/relname+ (next.jdbc/execute! conn ["SELECT c.relname FROM pg_inherits i+ JOIN pg_class p ON i.inhparent = p.oid+ JOIN pg_class c ON i.inhrelid = c.oid+ WHERE p.relname='query_execution'"])))++(defn fully-on-partitions?+ "Has the database fully switched to managing its truncation with partitions?+ If the database is using partitions but they're still in the first partition+ then we need the legacy truncation to continue to operate, because the first+ partition *is* the old table and contains rows going back much further than+ a single partition's worth."+ []+ (and (= :postgres (mdb/db-type))+ (with-open [conn (.getConnection (mdb/data-source))]+ (let [[oldest-partition] (sort (current-partitions conn))+ [oldest] (next.jdbc/execute! conn [(str "SELECT min(started_at) AS min FROM "+ oldest-partition)])+ oldest-date-time (some-> (:min oldest) t/instant+ (t/local-date-time "UTC"))]+ (or (nil? oldest-date-time)+ (= oldest-partition (partition-for oldest-date-time)))))))+;;; and now for the side-effects...(defn create-partition@@ -64,15 +90,6 @@(defn- drop-partition [conn partition](next.jdbc/execute! conn [(format "DROP TABLE \"%s\"" partition)]))-(defn current-partitions- "List the names of the currently-attached query_execution partition tables."- [conn]- (map :pg_class/relname- (next.jdbc/execute! conn ["SELECT c.relname FROM pg_inherits i- JOIN pg_class p ON i.inhparent = p.oid- JOIN pg_class c ON i.inhrelid = c.oid- WHERE p.relname='query_execution'"])))-(defn manage-partitions"Create new partitions and drop unnecessary partitions under Postgres."[conn now retention-days]diff --git a/src/metabase/audit_app/task/truncate_audit_tables.clj b/src/metabase/audit_app/task/truncate_audit_tables.cljindex 8a0844c5adb..e351a7ab8e1 100644--- a/src/metabase/audit_app/task/truncate_audit_tables.clj+++ b/src/metabase/audit_app/task/truncate_audit_tables.clj@@ -9,6 +9,7 @@[metabase.app-db.core :as mdb][metabase.audit-app.db :as audit-app.db][metabase.audit-app.settings :as audit-app.settings]+ [metabase.audit-app.task.partitions :as partitions][metabase.premium-features.core :refer [defenterprise]][metabase.task-history.core :as task-history][metabase.task.core :as task]@@ -55,7 +56,7 @@metabase-enterprise.audit-app.task.truncate-audit-tables[];; postgres has partitioned query_execution; we detach partitions instead of deleting- (if (= :postgres (mdb/db-type))+ (if (partitions/fully-on-partitions?)[][{:model :model/QueryExecution :timestamp-col :started_at}]))diff --git a/test/metabase/audit_app/task/partitions_test.clj b/test/metabase/audit_app/task/partitions_test.cljindex 2c9be5561d6..9f313f61188 100644--- a/test/metabase/audit_app/task/partitions_test.clj+++ b/test/metabase/audit_app/task/partitions_test.clj@@ -4,7 +4,8 @@[java-time.api :as t][metabase.app-db.core :as mdb][metabase.audit-app.task.partitions :as partitions]- [next.jdbc :as next.jdbc]))+ [next.jdbc :as next.jdbc]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -30,6 +31,7 @@(deftest partitions-to-drop-test(let [existing (partitions-for-months 4 5 6 7)](is (= [] (partitions/partitions-to-drop existing now 0)))+ (is (= [] (partitions/partitions-to-drop existing now ##Inf)))(is (= (partitions-for-months 4 5)(set (partitions/partitions-to-drop existing now 7))))(is (= (partitions-for-months 4 5 6)@@ -96,3 +98,54 @@(partitions/create-partition conn {:name "query_execution_2026_08":from "1970-01-01":to "2099-01-01"}))))))++(defn insert-execution! [started-at]+ (t2/insert! :model/QueryExecution {:started_at started-at+ :hash (.getBytes "lol")+ :result_rows 0+ :running_time 2+ :context :exploration+ :native false}))++;; this simulates an old install that has a bunch of query_execution rows that+;; switches over to partitioned tables. we want to make sure that we can detect+;; fully-on-partitions? in these cases.+(deftest initial-partition-to-fully-managed+ (when (= :postgres (mdb/db-type))+ (with-open [conn (.getConnection (mdb/data-source))]+ (try+ (testing "initial partition"+ (drop-all-tables conn)+ ;; initial partition, upper bound is exclusive+ (partitions/create-partition conn {:name "query_execution_2026_09"+ :from "1970-01-01"+ :to "2026-10-01"})+ ;; successive partition+ (partitions/create-partition conn {:name "query_execution_2026_10"+ :from "2026-10-01"+ :to "2026-11-01"})+ ;; initial data+ (insert-execution! #t "2026-02-01")+ (insert-execution! #t "2026-07-01")+ (insert-execution! #t "2026-08-01")+ (insert-execution! #t "2026-09-01")+ (insert-execution! #t "2026-10-02")+ (is (not (partitions/fully-on-partitions?))))+ (testing "after second partition is added"+ (partitions/manage-partitions conn #t "2026-09-01" 25)+ (is (not (partitions/fully-on-partitions?))))+ (testing "after third partition is added"+ ;; we won't drop the initial partition yet because its end date is+ ;; still within the 25-day retention window+ (partitions/manage-partitions conn #t "2026-10-01" 25)+ (is (not (partitions/fully-on-partitions?))))+ (testing "after fourth partition is added"+ ;; by now the rows from 2026-10-02 have cycled out of retention+ (partitions/manage-partitions conn #t "2026-11-01" 25)+ (is (partitions/fully-on-partitions?)))+ (finally+ (drop-all-tables conn)+ ;; make sure we end up in a good state for other tests+ (partitions/create-partition conn {:name "query_execution_2026_08"+ :from "1970-01-01"+ :to "2099-01-01"}))))))