Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Replace Bigquery/Snowflake dataset tracking table; track by name instead (#81655)

technomancy4be1fd71c9ca
diff --git a/.github/workflows/test.cleanup-dwh-data.yml b/.github/workflows/test.cleanup-dwh-data.yml
index 25a4c893e32..afbf3d9cd64 100644
--- a/.github/workflows/test.cleanup-dwh-data.yml
+++ b/.github/workflows/test.cleanup-dwh-data.yml
@@ -10,21 +10,15 @@ on:
required: false
default: 'snowflake,bigquery-cloud-sdk,redshift'
type: string
- temp-data-hours:
- description: 'TTL in hours for per-run garbage (minimum 2)'
+ hours:
+ description: 'TTL in hours for per-run garbage (minimum 4)'
required: false
- default: '2'
- type: string
- fixture-hours:
- description: 'TTL in hours for datasets runs share (minimum 24)'
- required: false
- default: '96'
+ default: '12'
type: string
# `inputs.*` is empty on the scheduled run, so each falls back to the nightly value.
env:
- GC_TEMP_HOURS: ${{ inputs.temp-data-hours || 2 }} # for temp data
- GC_FIXTURE_HOURS: ${{ inputs.fixture-hours || 96 }} # for reusable data
+ GC_HOURS: ${{ inputs.hours || 12 }} # for temp data
GC_DRIVERS: ${{ inputs.drivers || 'snowflake,bigquery-cloud-sdk,redshift' }}
concurrency:
@@ -66,8 +60,7 @@ jobs:
run: |
clojure -X:dev:drivers:drivers-dev:test metabase.test.data.gc/gc-orphans! \
:drivers "\"${{ env.GC_DRIVERS }}\"" \
- :temp-data-hours "${{ env.GC_TEMP_HOURS }}" \
- :fixture-hours "${{ env.GC_FIXTURE_HOURS }}" \
+ :hours "${{ env.GC_HOURS }}" \
:report-dir '"target/gc-report"'
# The report is most wanted on the runs that fail, so upload it regardless of the sweep's outcome.
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index fdfe7e76ea9..b9f8ae8e23c 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -1314,24 +1314,25 @@
(deftest later-page-fetch-returns-nil-test
(mt/test-driver :bigquery-cloud-sdk
- (testing "BigQuery query whose later page fetch returns nil is caught, not silently truncated"
- ;; The query path pages via `query-results-page` (`.getQueryResults`), so simulate BigQuery returning nil
- ;; for a later page even though the page token reported there was more.
- (let [page-counter (atom 3)
- orig-fetch (mt/original-fn #'bigquery/query-results-page)]
- (mt/with-dynamic-fn-redefs [bigquery/query-results-page (fn [job options]
- (if (zero? @page-counter)
- nil
- (orig-fetch job options)))]
- (binding [bigquery/*page-size* 10 ; small pages so there are several
- bigquery/*page-callback* (fn []
- (let [pages (swap! page-counter #(max (dec %) 0))]
- (log/debugf "*page-callback counting down: %d to go" pages)))]
- (mt/dataset test-data
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Cannot get next page from BigQuery"
- (mt/process-query (mt/query orders)))))))))))
+ (mt/dataset test-data
+ (testing "BigQuery query whose later page fetch returns nil is caught, not silently truncated"
+ ;; The query path pages via `query-results-page` (`.getQueryResults`), so simulate BigQuery returning nil
+ ;; for a later page even though the page token reported there was more.
+ (let [page-counter (atom 3)
+ orig-fetch (mt/original-fn #'bigquery/query-results-page)]
+ (mt/with-dynamic-fn-redefs [bigquery/query-results-page (fn [job options]
+ (if (zero? @page-counter)
+ nil
+ (orig-fetch job options)))]
+ (binding [bigquery/*page-size* 10 ; small pages so there are several
+ bigquery/*page-callback* (fn []
+ (let [pages (swap! page-counter #(max (dec %) 0))]
+ (log/debugf "*page-callback counting down: %d to go" pages)))]
+ (mt/dataset test-data
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Cannot get next page from BigQuery"
+ (mt/process-query (mt/query orders))))))))))))
(deftest later-page-fetch-throws-test
(mt/test-driver :bigquery-cloud-sdk
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
index 3871113a706..2ec1f6be7c7 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
@@ -7,6 +7,7 @@
[metabase.driver :as driver]
[metabase.driver.bigquery-cloud-sdk :as bigquery]
[metabase.driver.ddl.interface :as ddl.i]
+ [metabase.driver.sql.test-util.unique-prefix :as sql.tu.unique-prefix]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.test.data.impl :as data.impl]
[metabase.test.data.interface :as tx]
@@ -38,8 +39,7 @@
Schema
StandardTableDefinition
TableId
- TableInfo)
- (java.time Duration)))
+ TableInfo)))
(set! *warn-on-reflection* true)
@@ -59,17 +59,24 @@
{:error/message "Dataset IDs must be alphanumeric (plus underscores)"}
#"^[\w_]+$"]])
+(defn- already-qualified? [database-name]
+ (and (string? database-name)
+ (or (str/starts-with? database-name "temp_")
+ (str/starts-with? database-name "sha_"))))
+
+(def ^:private to-cleanup (atom #{}))
+
(mu/defn test-dataset-id :- ::dataset-id
"Prepend `database-name` with the hash of the db-def so we don't stomp on any other jobs running at the same
time."
- [{:keys [database-name] :as db-def}]
- (cond (str/starts-with? database-name "sha_")
- database-name
- ;; releases get their own isolated datasets
- (tx/on-master-or-release-branch?)
- (str "sha_rel_" (tx/hash-dataset db-def) "_" (normalize-name database-name))
- :else
- (str "sha__" (tx/hash-dataset db-def) "_" (normalize-name database-name))))
+ [{:keys [database-name options] :as db-def}]
+ (cond (already-qualified? database-name) database-name
+ (:static options) (str "sha_" (tx/hash-dataset (update db-def :options
+ dissoc :static))
+ "_" (normalize-name database-name))
+ :else (let [name (sql.tu.unique-prefix/unique-prefix (normalize-name database-name))]
+ (swap! to-cleanup conj name)
+ name)))
(defn- test-db-details []
(if tx/*use-routing-details*
@@ -159,6 +166,7 @@
(.delete (bigquery) dataset-id (u/varargs
BigQuery$DatasetDeleteOption
[(BigQuery$DatasetDeleteOption/deleteContents)]))
+ ;; TODO: drop the test tracking table once 58 and 64 are EOL
(execute-params!
(format "DELETE FROM `%s.metabase_test_tracking.datasets` WHERE `name` = ?"
(project-id))
@@ -348,135 +356,43 @@
(recur (dec num-retries))
(throw e))))))))
-(def ^:private ^Duration reap-after
- "How long [[delete-old-datasets!]] lets a dataset sit before deleting it - since its last recorded use if it is
- tracked, since its creation if it is not."
- (t/duration 14 :days))
-
-(def ^:private ^Duration track-debounce
- "How stale a dataset's `accessed_at` may be before another use is worth recording.
-
- Half of [[reap-after]], so a dataset that keeps being used is re-recorded with a full reap interval to spare.
- Anything past that halfway point leaves a dataset that is still in use eligible for deletion."
- (.dividedBy reap-after 2))
-
-(defn- track-debounce-seconds
- "[[track-debounce]] shortened by a random margin.
-
- Without the margin, every job using a given dataset crosses the threshold at the same moment and writes at once.
- [[tx/track-dataset]] writes to one table shared by every branch and release stream, and BigQuery runs only two
- mutating statements per table concurrently, queueing twenty more before it starts rejecting them."
- []
- (let [seconds (.toSeconds track-debounce)]
- (- seconds (rand-int (quot seconds 4)))))
-
-(def ^:private recently-tracked-hashes
- "Hashes of datasets used recently enough that recording another use would be redundant, read once per process.
-
- Nothing reads `accessed_at` during a test run, so a snapshot going stale mid-run costs at worst a redundant write.
- Nil if the tracking table cannot be read: on a fresh project nothing has been tracked, which is the same answer."
- (delay
- (u/ignore-exceptions
- (into #{}
- (map first)
- (execute! (str "SELECT `hash` FROM `%s.metabase_test_tracking.datasets`"
- " WHERE `accessed_at` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL %d SECOND)")
- (project-id)
- (track-debounce-seconds))))))
-
-(def ^:private old-dataset-name-pattern "sha_%")
-
-(defn- old-dataset-names
- "Names of test datasets older than `hours`: tracked ones nothing has accessed in that long, plus untracked ones
- created that long ago. Excludes the current `test-data`, which [[destroy-dataset!]] refuses to delete anyway.
-
- Shared by [[delete-old-datasets!]] and the nightly sweep ([[tx/gc-orphans!]]), which differ only in `hours`."
- [hours]
- (let [current-test-data (test-dataset-id (tx/get-dataset-definition
- (data.impl/resolve-dataset-definition *ns* 'test-data)))]
- (into []
- (comp (map first)
- (remove #(= % current-test-data)))
- (execute! (str "(SELECT `name` FROM `%1$s.metabase_test_tracking.datasets`"
- " WHERE `name` LIKE '%2$s'"
- " AND `accessed_at` < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%3$d hour))"
- " UNION ALL "
- "(select schema_name from `%1$s`.INFORMATION_SCHEMA.SCHEMATA d
+(defn- drop-orphan! [server dry-run? name]
+ (try
+ (when-not dry-run?
+ (destroy-dataset! name))
+ {:server server :name name :status :deleted}
+ (catch Exception e
+ {:server server :name name :status :error :error (ex-message e)})))
+
+(defn- gc-tracked-datasets!
+ "Datasets created by CI runs from older versions still use the tracking table. Until
+ we stop supporting version 58 and 63 we'll have to keep GCing these, but we handle them
+ in a separate function so they'll be easier to delete later."
+ [hours dry-run?]
+ (mapv (fn [[dataset-name]] (drop-orphan! (project-id) dry-run? dataset-name))
+ (execute! (str "(SELECT `name` FROM `%s.metabase_test_tracking.datasets`"
+ " WHERE `name` LIKE '%2$s'"
+ " AND `accessed_at` < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%d hour))"
+ " UNION ALL "
+ "(select schema_name from `%1$s`.INFORMATION_SCHEMA.SCHEMATA d
where d.schema_name not in (select name from `%1$s.metabase_test_tracking.datasets`)
- and d.schema_name like '%2$s'
- and creation_time < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%3$d hour))")
- (project-id)
- old-dataset-name-pattern
- hours))))
-
-(defn- drop-datasets!
- "Delete each named dataset, reporting per dataset whether it went. See [[tx/gc-orphans!]] for the shape."
- [dataset-ids]
- (mapv (fn [dataset-id]
- (try
- (destroy-dataset! dataset-id)
- {:name dataset-id, :status :deleted}
- ;; usually just another job deleting the same dataset at the same time
- (catch Exception e
- {:name dataset-id, :status :failed, :error (ex-message e)})))
- dataset-ids))
-
-(defn delete-old-datasets! []
- (drop-datasets! (old-dataset-names (.toHours reap-after))))
-
-(defonce ^:private deleted-old-datasets?
- (atom false))
-
-(defn- delete-old-datasets-if-needed!
- "Call [[delete-old-datasets!]], only if we haven't done so already."
- []
- (when (compare-and-set! deleted-old-datasets? false true)
- (delete-old-datasets!)))
+ and creation_time < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%d hour))")
+ (project-id) hours hours)))
-;; Nightly garbage collection of orphaned test datasets.
(defmethod tx/gc-orphans! :bigquery-cloud-sdk
- [_driver {:keys [fixture-hours]}]
- (let [project (project-id)]
- (mapv #(assoc % :server project) (drop-datasets! (old-dataset-names fixture-hours)))))
+ [_driver {:keys [hours tracked? dry-run?]}]
+ (->> (execute-params!
+ (format "select schema_name from `%s`.%s where schema_name like 'temp_%%'"
+ (project-id) "INFORMATION_SCHEMA.SCHEMATA") [])
+ (filter (partial sql.tu.unique-prefix/old-temp-dataset? hours))
+ (mapv (partial drop-orphan! (project-id) dry-run?))
+ (concat (and tracked? (gc-tracked-datasets! hours dry-run?)))))
(defmethod tx/count-datasets :bigquery-cloud-sdk
[_driver]
(let [project (project-id)]
- {project (ffirst (execute! "SELECT COUNT(*) FROM `%s`.INFORMATION_SCHEMA.SCHEMATA" project))}))
-
-(defn- setup-tracking-dataset!
- "Idempotently create test tracking database"
- []
- (let [dataset-id "metabase_test_tracking"]
- (try
- (create-dataset! dataset-id)
- (catch BigQueryException e
- ;; Already exists, ignore
- (when-not (= (.getCode e) 409)
- (throw e))))
- (try
- (create-table*! dataset-id "datasets" [{:field-name "hash"
- :base-type :type/Text}
- {:field-name "name"
- :base-type :type/Text}
- {:field-name "accessed_at"
- :base-type :type/DateTimeWithTZ}
- {:field-name "access_note"
- :base-type :type/Text}])
- (catch BigQueryException e
- ;; Already exists, ignore
- (when-not (= (.getCode e) 409)
- (throw e))))))
-
-(defn- dataset-tracked?!
- [db-def]
- (->
- (execute-params!
- (format "SELECT true FROM `%s.metabase_test_tracking.datasets` WHERE `hash` = ? and `name` = ?"
- (project-id))
- [(tx/hash-dataset db-def)
- (test-dataset-id db-def)])
- ffirst))
+ {project (ffirst (execute! "SELECT COUNT(*) FROM `%s`.INFORMATION_SCHEMA.SCHEMATA"
+ project))}))
(defn database-exists?!
[db-def]
@@ -498,26 +414,6 @@
(set/subset? (set (map :table-name (:table-definitions db-def)))
(set (get-existing-tables (test-dataset-id db-def))))))
-(defmethod tx/track-dataset :bigquery-cloud-sdk
- [_driver db-def]
- ;; BigQuery has a limit of 20 pending DML statements per table.
- ;; https://cloud.google.com/blog/products/data-analytics/dml-without-limits-now-in-bigquery
- ;; Repeatedly tracking the same dataset each time it's touched causes us to run up against
- ;; that limit in CI. Debounce tracking recently tracked datasets so we don't create nearly as much
- ;; dataset tracking noise.
- (when-not (contains? @recently-tracked-hashes (tx/hash-dataset db-def))
- (setup-tracking-dataset!)
- ; ignore exceptions because of https://cloud.google.com/bigquery/docs/troubleshoot-queries#could_not_serialize
- (u/ignore-exceptions
- (execute-params!
- (format (str "MERGE INTO `%s.metabase_test_tracking.datasets` d"
- " USING (select ? as `hash`, ? as `name`, current_timestamp() as accessed_at, ? as access_note) as n on d.`hash` = n.`hash`"
- " WHEN MATCHED THEN UPDATE SET d.accessed_at = n.accessed_at, d.access_note = n.access_note"
- " WHEN NOT MATCHED THEN INSERT (`hash`,`name`, accessed_at, access_note) VALUES (n.`hash`, n.`name`, n.accessed_at, n.access_note)") (project-id))
- [(tx/hash-dataset db-def)
- (test-dataset-id db-def)
- (tx/tracking-access-note)]))))
-
(defmethod tx/create-db! :bigquery-cloud-sdk
[driver {:keys [database-name table-definitions options] :as db-def} & _]
{:pre [(seq database-name) (sequential? table-definitions)]}
@@ -577,13 +473,7 @@
(comment
"REPL utilities for static datasets"
- (setup-tracking-dataset!)
- (destroy-dataset! "metabase_test_tracking")
(destroy-dataset! (test-dataset-id (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))
- (tx/track-dataset :bigquery-cloud-sdk (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data)))
- (dataset-tracked?! (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'attempted-murders)))
-
- (execute! "select name from `%s`.metabase_test_tracking.datasets order by accessed_at" (project-id))
(database-exists?! (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))
(defn ^:private get-test-data-name
diff --git a/modules/drivers/redshift/test/metabase/test/data/redshift.clj b/modules/drivers/redshift/test/metabase/test/data/redshift.clj
index 269d1e74ba3..61bbad874af 100644
--- a/modules/drivers/redshift/test/metabase/test/data/redshift.clj
+++ b/modules/drivers/redshift/test/metabase/test/data/redshift.clj
@@ -64,7 +64,7 @@
(throw (UnsupportedOperationException. "Redshift does not have a TIME data type.")))
(defn unique-session-schema []
- (str (sql.tu.unique-prefix/unique-prefix) "schema"))
+ (sql.tu.unique-prefix/unique-prefix "schema"))
;;; `MB_REDSHIFT_TEST_HOSTS`
;;;
@@ -151,10 +151,6 @@
init
(reducible-result-set (.. conn getMetaData getSchemas))))))
-(def ^Long ^:private hours-before-expired-threshold
- "Number of hours that elapse before a persisted schema is considered expired."
- 1)
-
(defn- classify-cache-schemas
"Classifies the persistence cache schemas. Returns a map with where each value is a (possibly empty) sequence of
schemas:
@@ -164,8 +160,8 @@
:expired `cache_info` table and created [[hours-before-expired-threshold]] ago
:lacking-created-at should never happen, but if they lack an entry for `created-at`
:unknown-error if an error was thrown while classifying the schema}"
- [^java.sql.Connection conn schemas]
- (let [threshold (t/minus (t/instant) (t/hours hours-before-expired-threshold))]
+ [^java.sql.Connection conn schemas hours]
+ (let [threshold (t/minus (t/instant) (t/hours hours))]
(with-open [stmt (.createStatement conn)]
(let [classify (fn [schema-name]
(try (let [sql (format "select value from %s.cache_info where key = 'created-at'"
@@ -213,7 +209,7 @@
[^java.sql.Connection conn hours-threshold]
(let [{old-convention :old
caches-with-info :cache} (reduce (fn [acc s]
- (cond (sql.tu.unique-prefix/old-dataset-name? s hours-threshold)
+ (cond (sql.tu.unique-prefix/old-temp-dataset? hours-threshold s)
(update acc :old conj s)
(str/starts-with? s "metabase_cache_")
(update acc :cache conj s)
@@ -222,7 +218,8 @@
(fetch-schemas conn))
{expired-cache :expired
old-style-cache :old-style-cache
- lacking-created-at :lacking-created-at} (classify-cache-schemas conn caches-with-info)]
+ lacking-created-at :lacking-created-at} (classify-cache-schemas
+ conn caches-with-info hours-threshold)]
{:old (vec old-convention)
:expired-cache (vec expired-cache)
:old-style-cache (vec old-style-cache)
@@ -237,11 +234,12 @@
Takes the orphan-map directly so callers can preview-then-drop without
re-querying. Caller owns the Statement."
- [^java.sql.Statement stmt orphans]
+ [^java.sql.Statement stmt dry-run? orphans]
(mapv (fn [[fmt-str schema]]
(tx/print-progress! :redshift fmt-str schema)
(try
- (.execute stmt (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema))
+ (when-not dry-run?
+ (.execute stmt (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema)))
{:name schema, :status :deleted}
(catch Exception e
{:name schema, :status :failed, :error (ex-message e)})))
@@ -260,9 +258,9 @@
Glue: thin wrapper that calls the enumerator + dropper in order. To preview
from a REPL, call [[orphan-schemas]] directly."
[^java.sql.Connection conn]
- (let [orphans (orphan-schemas conn nil)]
+ (let [orphans (orphan-schemas conn 12)]
(with-open [stmt (.createStatement conn)]
- (drop-orphan-schemas! stmt orphans))))
+ (drop-orphan-schemas! stmt false orphans))))
(defn- gc-hosts
"Every cluster tests run against, not just the one a run would pick. `MB_REDSHIFT_TEST_HOSTS` is the fleet
@@ -314,7 +312,7 @@
(doall (cp/pmap pool f servers)))))
(defmethod tx/gc-orphans! :redshift
- [driver {:keys [temp-data-hours]}]
+ [driver {:keys [hours dry-run?]}]
;; Redshift schema names carry their own creation time, so we scan for age via the name
(into []
cat
@@ -326,7 +324,7 @@
(fn [^java.sql.Connection conn]
(with-open [stmt (.createStatement conn)]
(mapv #(assoc % :server server)
- (drop-orphan-schemas! stmt (orphan-schemas conn temp-data-hours)))))
+ (drop-orphan-schemas! stmt dry-run? (orphan-schemas conn hours)))))
(fn [e]
[{:server server, :name nil, :status :failed, :error (ex-message e)}])))))))
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index 0ae15fa74ed..7ae0a1c6166 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -443,7 +443,6 @@
[{:field-name "name" :base-type :type/Text}]
[["mb_qnkhuat"]]]])
(let [{{db-name :db, :as details} :details} (mt/db)]
- (tx/track-dataset :snowflake data.impl/*dbdef-used-to-create-db*)
;; TARGET_LAG = DOWNSTREAM instead of a time interval: nothing reads this table, so it never actually
;; needs to refresh. With a time-based lag, a test DB that leaks (e.g. a cancelled CI job skips
;; [[metabase.test.data.snowflake/after-run]]) keeps refreshing on that schedule forever.
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 2be4d2e17c3..e008eac723b 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -2,10 +2,10 @@
(:require
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
- [com.climate.claypoole :as cp]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+ [metabase.driver.sql.test-util.unique-prefix :as sql.tu.unique-prefix]
[metabase.test :as mt]
[metabase.test.data.impl :as data.impl]
[metabase.test.data.interface :as tx]
@@ -43,22 +43,23 @@
:type/Time "TIME(3)"}]
(defmethod sql.tx/field-base-type->sql-type [:snowflake base-type] [_ _] sql-type))
-;; in CI use a completely different set of databases for each run and tear down
-;; all of them when the job completes; see after-run below.
-(defonce dataset-prefix (str (rand-int 9999999)))
-
(defn- already-qualified? [database-name]
(and (string? database-name)
- (or (str/starts-with? database-name "isolate_")
+ (or (str/starts-with? database-name "temp_")
(str/starts-with? database-name "sha_"))))
+(def ^:private to-cleanup (atom #{}))
+
(defn qualified-db-name
"Isolate db name so we don't stomp on any other jobs running at the same time."
- [{:keys [database-name] :as db-def}]
+ [{:keys [database-name options] :as db-def}]
(cond (already-qualified? database-name) database-name
- ;; isolate if we are in a CI job
- (System/getenv "GITHUB_REF_NAME") (str "isolate_" dataset-prefix database-name)
- :else (str "sha_" (tx/hash-dataset db-def) "_" database-name)))
+ (:static options) (str "sha_" (tx/hash-dataset (update db-def :options
+ dissoc :static))
+ "_" database-name)
+ :else (let [name (sql.tu.unique-prefix/unique-prefix database-name)]
+ (swap! to-cleanup conj name)
+ name)))
(defmethod tx/dbdef->connection-details :snowflake
[_driver context dbdef]
@@ -107,126 +108,10 @@
;;; --------------------------------- Cleanup ----------------------------------
(defmethod tx/after-run :snowflake [_driver]
- (let [spec (no-db-connection-spec)
- query "select name from metabase_test_tracking.PUBLIC.datasets
- where name like 'isolate_%s%%'"]
- (doseq [{:keys [name]} (jdbc/query spec [(format query dataset-prefix)])]
- (jdbc/query spec
- ["DELETE FROM metabase_test_tracking.PUBLIC.datasets where name = ?" name])
+ (let [spec (no-db-connection-spec)]
+ (doseq [name @to-cleanup]
(jdbc/execute! spec [(format "DROP DATABASE \"%s\";" name)]))))
-(defn- old-dataset-names
- "Names of test databases old enough to delete, oldest first.
-
- [[qualified-db-name]] gives CI runs an `isolate_` name built from a random int -- per-run, never reused, so they
- age from `created`. Everything else is a `sha_` name from a local run, content-addressed and reused, so those age
- from `accessed_at` when the tracking table knows them.
-
- `accessed_at` is only trustworthy for `sha_`: the tracking table is keyed on dataset hash and its MERGE never
- updates `name`, so an `isolate_` row keeps the first run's name while later runs keep refreshing its `accessed_at`.
-
- Compares timestamps directly; `timestampdiff` counts boundary crossings, so an hour bucket calls a 10:59 database
- two hours old at 12:01."
- [{:keys [temp-data-hours fixture-hours]}]
- (into []
- (map :database_name)
- (jdbc/query (no-db-connection-spec)
- [(format
- "select d.database_name
- from metabase_test_tracking.information_schema.databases d
- left join metabase_test_tracking.PUBLIC.datasets t on t.name = d.database_name
- where (startswith(d.database_name, 'isolate_')
- and d.created < dateadd(hour, -%d, current_timestamp()))
- or (startswith(d.database_name, 'sha_')
- and coalesce(t.accessed_at, d.created) < dateadd(hour, -%d, current_timestamp()))
- order by d.created"
- temp-data-hours
- fixture-hours)])))
-
-;;; --------------------------------- Destruction ----------------------------------
-
-(defn- with-write-stmt!
- "Open a write-capable Snowflake connection + Statement, call `f` with the stmt,
- close everything. Centralizes the boilerplate so the per-resource drop fns
- don't repeat it."
- [f & args]
- (sql-jdbc.execute/do-with-connection-with-options
- :snowflake
- (no-db-connection-spec)
- {:write? true}
- (fn [^java.sql.Connection conn]
- (with-open [stmt (.createStatement conn)]
- (apply f stmt args)))))
-
-(def ^:private drop-workers
- "Connections used to drop databases at once. Each `DROP DATABASE` is a round trip of roughly half a second and a
- backlog runs to thousands, which is what overran the job's first real run. Snowflake caps a session at 8 concurrent
- statements by default, so more workers than this buys queueing, not throughput."
- 8)
-
-(def ^:private untrack-batch-size
- "Names per `DELETE` when clearing tracking rows. Snowflake locks the whole table for DML, so this is one statement
- at a time by design -- the win is round trips, not concurrency."
- 500)
-
-(defn- drop-one!
- [^java.sql.Statement stmt dataset-name]
- (tx/print-progress! :snowflake "deleting %s" dataset-name)
- (try
- (.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))
- {:name dataset-name, :status :deleted}
- ;; usually just another job deleting the same dataset at the same time
- (catch Exception e
- {:name dataset-name, :status :failed, :error (ex-message e)})))
-
-(defn- drop-chunk!
- "Drop one worker's share on a connection of its own: a JDBC Statement cannot be shared across threads, and
- reconnecting per database would cost more than the drop does."
- [dataset-names]
- (with-write-stmt!
- (fn [^java.sql.Statement stmt]
- (mapv #(drop-one! stmt %) dataset-names))))
-
-(defn- untrack!
- "Forget these databases. Kept out of the workers: every row lives in one table, and Snowflake serializes DML on a
- table, so per-database deletes would have undone the parallelism they ran alongside."
- [dataset-names]
- (doseq [batch (partition-all untrack-batch-size dataset-names)]
- (jdbc/execute! (no-db-connection-spec)
- (into [(format "delete from metabase_test_tracking.PUBLIC.datasets where name in (%s)"
- (str/join "," (repeat (count batch) "?")))]
- batch))))
-
-(defn- drop-datasets!
- "Un-track each named test database and then drop it, reporting per database whether it went. See [[tx/gc-orphans!]]
- for the shape.
-
- Un-tracking comes first so that being killed partway -- the GitHub job hitting its timeout -- leaves recoverable
- state. A database that is present but untracked is collected again by the next sweep, which ages an untracked
- `sha_` database from `created`, necessarily older than the `accessed_at` that made it eligible here. Dropping
- first would instead strand tracking rows for databases that no longer exist, and the sweep enumerates from
- `information_schema`, so it would never revisit them and the table would grow without bound.
-
- Each `DROP DATABASE IF EXISTS` is atomic and idempotent on its own, so no individual delete can be torn in half
- and the parallelism costs nothing in recoverability."
- [dataset-names]
- ;; nothing to drop is the common case on a healthy night; don't open a connection to discover that
- (if (empty? dataset-names)
- []
- (if-let [untrack-error (try
- (untrack! dataset-names)
- nil
- (catch Exception e (ex-message e)))]
- ;; dropping anyway would strand exactly the rows we failed to clear, so drop nothing
- [{:name nil
- :status :failed
- :error (format "could not clear tracking rows for %d database(s), so dropped none of them: %s"
- (count dataset-names) untrack-error)}]
- (let [chunks (partition-all (max 1 (long (Math/ceil (/ (count dataset-names) (double drop-workers)))))
- dataset-names)]
- (cp/with-shutdown! [pool (cp/threadpool (min drop-workers (count chunks)))]
- (into [] cat (doall (cp/pmap pool drop-chunk! chunks))))))))
-
;;; --------------------------------- Orphan GC ----------------------------------
;;;
;;; Nightly sweep (`.github/workflows/test.cleanup-dwh-data.yml`). This replaces the old in-process cleanup, which
@@ -237,10 +122,64 @@
[]
(tx/db-test-env-var-or-throw :snowflake :account))
+(defn- drop-orphan [conn server dry-run? name]
+ (try
+ (when-not dry-run?
+ (jdbc/execute! conn [(format "DROP DATABASE \"%s\";" name)]))
+ {:server server :name name :status :deleted}
+ (catch Exception e
+ {:server server :name name :status :error :error (ex-message e)})))
+
+(defn- gc-orphans! [conn {:keys [hours dry-run?]}]
+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement
+ :snowflake conn
+ "SHOW TERSE DATABASES STARTS WITH 'temp_'" [])
+ ^ResultSet rs (sql-jdbc.execute/execute-prepared-statement! :snowflake stmt)]
+ (->> (resultset-seq rs)
+ (filter (partial sql.tu.unique-prefix/old-temp-dataset? hours))
+ (mapv (partial drop-orphan conn (account) dry-run?)))))
+
+(defn- drop-tracked-dataset [conn server dry-run? {:keys [database_name]}]
+ (try
+ (when-not dry-run?
+ (jdbc/execute! conn [(format "DROP DATABASE IF EXISTS \"%s\";" database_name)])
+ (jdbc/execute! conn ["DELETE FROM metabase_test_tracking.PUBLIC.datasets where name = ?"
+ database_name]))
+ {:server server :name database_name :status :deleted}
+ (catch Exception e
+ {:server server :name database_name :status :error :error (ex-message e)})))
+
+(def ^:private tracked-sql
+ (str "select d.database_name
+ from metabase_test_tracking.information_schema.databases d
+ left join metabase_test_tracking.PUBLIC.datasets t on t.name = d.database_name
+ where (startswith(d.database_name, 'isolate_')
+ and d.created < dateadd(hour, 0 - ?, current_timestamp()))
+ or (startswith(d.database_name, 'sha_')
+ and coalesce(t.accessed_at, d.created) < dateadd(hour, 0 - ?, current_timestamp()))
+ order by d.created"))
+
+(defn- gc-tracked-datasets!
+ "Datasets created by CI runs from older versions still use the tracking table. Until
+ we stop supporting version 58 and 63 we'll have to keep GCing these, but we handle them
+ in a separate function so they'll be easier to delete later."
+ [conn {:keys [hours dry-run?]}]
+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement
+ :snowflake conn
+ tracked-sql [hours hours])
+ ^ResultSet rs (sql-jdbc.execute/execute-prepared-statement! :snowflake stmt)]
+ (->> (resultset-seq rs)
+ (mapv (partial drop-tracked-dataset conn (account) dry-run?)))))
+
(defmethod tx/gc-orphans! :snowflake
[_driver options]
- (let [server (account)]
- (mapv #(assoc % :server server) (drop-datasets! (old-dataset-names options)))))
+ (sql-jdbc.execute/do-with-connection-with-options
+ :snowflake
+ (no-db-connection-spec)
+ {:write? true}
+ (fn [conn]
+ (concat (gc-orphans! conn options)
+ (and (:tracked? options) (gc-tracked-datasets! conn options))))))
(defmethod tx/count-datasets :snowflake
[_driver]
@@ -279,8 +218,6 @@
;; test-harness cleanup output goes to the CI console, not the app log
#_{:clj-kondo/ignore [:discouraged-var]}
(println "[Snowflake] destroy database " database-name (:database-name dbdef))
- (jdbc/query (no-db-connection-spec)
- ["DELETE FROM metabase_test_tracking.PUBLIC.datasets where name = ?" database-name])
(jdbc/execute! (no-db-connection-spec) [sql])))
;; For reasons I don't understand the Snowflake JDBC driver doesn't seem to work when trying to use parameterized
@@ -314,36 +251,6 @@
(defmethod sql.tx/generated-column-sql :snowflake [_ expr]
(format "AS (%s)" expr))
-(defn- setup-tracking-db!
- "Idempotently create test tracking database"
- [conn driver]
- (with-open [^PreparedStatement setup-1 (sql-jdbc.execute/prepared-statement
- driver
- conn
- "CREATE DATABASE IF NOT EXISTS metabase_test_tracking;"
- [])
- ^PreparedStatement setup-2 (sql-jdbc.execute/prepared-statement
- driver
- conn
- "CREATE TABLE IF NOT EXISTS metabase_test_tracking.PUBLIC.datasets (hash TEXT, name TEXT, accessed_at TIMESTAMP_TZ, access_note TEXT)"
- [])
- ^ResultSet _ (sql-jdbc.execute/execute-prepared-statement! driver setup-1)
- ^ResultSet _ (sql-jdbc.execute/execute-prepared-statement! driver setup-2)]
- nil))
-
-(defonce ^:private set-up-tracking-db?
- (atom false))
-
-(defn- setup-tracking-db-if-needed!
- "Call [[setup-tracking-db!]], only if we haven't done so already.
-
- Both of its statements are server round trips, and nothing drops `metabase_test_tracking` while a run is in
- progress, so once they have succeeded they only need repeating in the next process."
- [conn driver]
- (when-not @set-up-tracking-db?
- (setup-tracking-db! conn driver)
- (reset! set-up-tracking-db? true)))
-
(defn- database-exists?!
[conn driver db-def]
(with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement
@@ -384,27 +291,6 @@
(and (database-exists?! conn driver db-def)
(dataset-rows-ok?! conn db-def)))))
-(defmethod tx/track-dataset :snowflake
- [driver db-def]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- (sql-jdbc.conn/connection-details->spec driver (tx/dbdef->connection-details driver :server db-def))
- {:write? false}
- (fn [^java.sql.Connection conn]
- (setup-tracking-db-if-needed! conn driver)
- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement
- driver
- conn
- (str "MERGE INTO metabase_test_tracking.PUBLIC.datasets d"
- " USING (select ? as hash, ? as name, current_timestamp() as accessed_at, ? as access_note) as n on d.hash = n.hash"
- " WHEN MATCHED THEN UPDATE SET d.accessed_at = n.accessed_at, d.access_note = n.access_note"
- " WHEN NOT MATCHED THEN INSERT (hash,name, accessed_at, access_note) VALUES (n.hash, n.name, n.accessed_at, n.access_note)")
- [(tx/hash-dataset db-def)
- (qualified-db-name db-def)
- (tx/tracking-access-note)])
- ^ResultSet rs (sql-jdbc.execute/execute-prepared-statement! driver stmt)]
- (some-> rs resultset-seq doall)))))
-
(defn drop-if-exists-and-create-roles!
[driver details roles]
(let [spec (sql-jdbc.conn/connection-details->spec driver details)]
@@ -482,33 +368,7 @@
(jdbc/query (no-db-connection-spec) ["SELECT query_text, end_time
FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
WHERE query_text LIKE 'DROP DATABASE %'
- ORDER BY end_time DESC limit 64"])
- ;; preview what the nightly sweep would collect, at its own thresholds
- (old-dataset-names {:temp-data-hours 2, :fixture-hours 72})
- (into [] (jdbc/reducible-query (no-db-connection-spec) ["select * from metabase_test_tracking.PUBLIC.datasets"]))
- ;; Tracked databases ordered by age
- (->> ["select d.name, d.accessed_at, i.created, timestampdiff('minute', i.created, d.accessed_at) as diff, timestampdiff('minute', i.created, current_timestamp()) as age
- from metabase_test_tracking.PUBLIC.datasets d
- inner join metabase_test_tracking.information_schema.databases i on i.database_name = d.name
- order by 5 asc"]
- (jdbc/reducible-query (no-db-connection-spec))
- (into [] (map (juxt :name :diff :age :accessed_at :created))))
-
- ;; Tracked DBs that are not in snowflake
- (->> ["select name, accessed_at from metabase_test_tracking.PUBLIC.datasets d
- where d.name not in (select database_name from metabase_test_tracking.information_schema.databases)
- order by accessed_at"]
- (jdbc/reducible-query (no-db-connection-spec))
- (into [] (map (juxt :name :accessed_at))))
-
- ;; DBs in snowflake that are not tracked
- (->> ["select database_name, created from metabase_test_tracking.information_schema.databases d
- where d.database_name not in (select name from metabase_test_tracking.PUBLIC.datasets)
- and d.database_name like 'sha_%'
- -- and created < dateadd(day, -2, current_timestamp())
- order by created"]
- (jdbc/reducible-query (no-db-connection-spec))
- (into [] (map (juxt :database_name :created)))))
+ ORDER BY end_time DESC limit 64"]))
(defmethod sql.tx/session-schema :snowflake [_driver] "PUBLIC")
diff --git a/test/metabase/driver/sql/test_util/unique_prefix.clj b/test/metabase/driver/sql/test_util/unique_prefix.clj
index 72100d03522..375dbf7c4b1 100644
--- a/test/metabase/driver/sql/test_util/unique_prefix.clj
+++ b/test/metabase/driver/sql/test_util/unique_prefix.clj
@@ -1,29 +1,28 @@
(ns metabase.driver.sql.test-util.unique-prefix
"Tooling for testing Cloud-based SQL databases, creating unique schema names for every test run and 'garbage
- collecting' old ones.
+ collecting' old ones. (Note that Athena and Databricks are Cloud-based DBs, but they don't use this because
+ they don't support creating databases during CI, so this is only used by Redshift, Snowflake, and Bigquery.)
In the past we had one shared prefix for everybody, and everybody was expected to play nice and not screw with it.
This eventually led to BIG problems when one CI job would think see that a dataset did not exist, and try to
recreate it, and then another CI job would do the same thing at the same time, and eventually we'd end up with a
half-created dataset that was missing a bunch of rows. So here is the new strategy going forward:
- 1. Every instance of Metabase gets their own prefix like `<current-date-utc>_<hour>_<site-uuid>_` e.g. `test-data`
+ 1. Static datasets don't use this; they get checksummed and prefixed with `sha_` and aren't subject to
+ automated cleanup.
+
+ 2. Other datasets get their own prefix like `temp_<current-date-utc>_<hour>_<site-uuid>_` e.g. `transform-abc`
becomes something like
- 2025_12_18_20_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data
+ temp_2025_12_18_20_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_transform-abc
This will prevent jobs from running at the same time from stomping on each other's work.
- 2. To avoid filling our Snowflake/Redshift/etc. accounts up with ephemeral data that never gets deleted, we will
- delete datasets following this pattern when they are more than 3 hours old. This allows aggressive cleanup while
- still being safe for long-running test suites. Old-format datasets (without hour) are deleted after more than 1
- day for backwards compatibility. Cleanup is done once the first time we create a test dataset in this process,
- i.e. done in [[metabase.test.data.interface/before-run]].
- See [[metabase.test.data.snowflake/delete-old-datasets-if-needed!]] for example.
+ 3. To avoid filling our Snowflake/Redshift/etc. accounts up with ephemeral data that never gets deleted, we will
+ delete datasets following this pattern when they are too old. This allows aggressive cleanup while
+ still being safe for long-running test suites.
- See this Slack thread for more info
- https://metaboat.slack.com/archives/CKZEMT1MJ/p1676659086280609?thread_ts=1676656964.624609&cid=CKZEMT1MJ or ask
- me (Cam) if you have any questions."
+ 4. Cleanup is done in the nightly `metabase.test.data.gc/gc-orphans!` job."
(:require
[clojure.string :as str]
[clojure.test :refer :all]
@@ -32,43 +31,33 @@
[metabase.test.initialize :as initialize]
[metabase.util.date-2 :as u.date]))
-(defn- utc-date
- "`LocalDate` in UTC time."
- []
- (t/local-date (t/instant) (t/zone-id "UTC")))
-
(defn- utc-date-time
"`LocalDateTime` in UTC time."
[]
(t/local-date-time (t/instant) (t/zone-id "UTC")))
-(defn- unique-prefix*
- ([]
- (unique-prefix* (utc-date-time)))
- ([local-date-time]
- {:pre [(instance? java.time.LocalDateTime local-date-time)]}
- ;; app DB has to be initialized to get settings
- (initialize/initialize-if-needed! :db)
- ;; Format: YYYY_MM_DD_HH_<site-uuid>_ (hour precision allows cleanup after ~3 hours)
- (-> (format "%s_%02d_%s_"
- (t/local-date local-date-time)
- (t/as local-date-time :hour-of-day)
- (system/site-uuid))
- (str/replace #"-" "_"))))
-
-(def ^{:arglists '([])} unique-prefix
- "Unique prefix for test datasets for this instance. Format is `<current-date-utc>_<hour-utc>_<site-uuid>_`. See comments above.
- Example:
-
- 2025_12_18_20_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_"
+(defn- unique-prefix* [suffix & [local-dt]]
+ ;; app DB has to be initialized to get settings
+ (initialize/initialize-if-needed! :db)
+ ;; Format: YYYY_MM_DD_HH_<site-uuid>_
+ (let [local-date-time (or local-dt (utc-date-time))]
+ (-> (format "temp_%s_%02d_%s_%s"
+ (t/local-date local-date-time)
+ (t/as local-date-time :hour-of-day)
+ (system/site-uuid)
+ suffix)
+ (str/replace #"-" "_"))))
+
+(def ^{:arglists '([suffix])} unique-prefix
+ "Uniquely prefix the non-static dataset name to allow it to be GC'd later."
(memoize unique-prefix*))
-(def ^:private old-dataset-hours-threshold
- "Number of hours after which a dataset is considered old and can be deleted. Set this high enough that any running
- test won't have its schema deleted mid-run."
- 3)
+(defn- parse-name [dataset-name]
+ (if-let [[_ year month day hour] (re-matches #"^temp_(\d{4})_(\d{2})_(\d{2})_(\d{2})_.*$" dataset-name)]
+ (t/local-date-time (parse-long year) (parse-long month) (parse-long day) (parse-long hour) 0)
+ false))
-(defn old-dataset-name?
+(defn old-temp-dataset?
"Is this dataset name old enough to be deleted?
For new-format names (with hour): more than `hours-threshold` hours old, defaulting to
@@ -80,83 +69,35 @@
If the date/time is invalid, we return false (not old) to be safe - we only want to delete
datasets that match our known format."
- ([dataset-name]
- (old-dataset-name? dataset-name nil))
- ([dataset-name hours-threshold]
- ;; Try new format first: YYYY_MM_DD_HH_<uuid>_...
- (if-let [[_ year month day hour] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_(\d{2})_.*$" dataset-name)]
- (let [dataset-date-time (try
- (t/local-date-time (parse-long year) (parse-long month) (parse-long day) (parse-long hour) 0)
- (catch Throwable _ nil))]
- (if-not dataset-date-time
- false
- ;; a name stamped 10 was created in [10:00, 11:00), so age from the end of the hour -- otherwise an N hour
- ;; threshold collects things only N-1 hours old
- (t/before? (u.date/add dataset-date-time :hour 1)
- (u.date/add (utc-date-time)
- :hour (- (or hours-threshold old-dataset-hours-threshold))))))
- ;; TODO (bryan 12-23-25) Remove old-format checks when this has been running for a few days.
- ;; Fall back to old format: YYYY_MM_DD_<uuid>_...
- (when-let [[_ year month day] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_.*$" dataset-name)]
- (let [dataset-date (try
- (t/local-date (parse-long year) (parse-long month) (parse-long day))
- (catch Throwable _ nil))]
- (if-not dataset-date
- false
- (t/before? dataset-date (u.date/add (utc-date) :day -1))))))))
+ [hours dataset-name]
+ (try
+ (if-let [dataset-date-time (parse-name dataset-name)]
+ (t/before? (u.date/add dataset-date-time :hour 1)
+ ;; a name stamped 10 was created in [10:00, 11:00), so age from
+ ;; the end of the hour -- otherwise an N hour threshold collects
+ ;; things only N-1 hours old
+ (u.date/add (utc-date-time) :hour (- hours)))
+ false)
+ (catch Exception _)))
(deftest ^:parallel old-dataset-name?-test
- (testing "old-format names (date only) - more than 1 day old"
- (let [two-days-ago (str/replace (str (u.date/add (utc-date) :day -2)) "-" "_")
- yesterday (str/replace (str (u.date/add (utc-date) :day -1)) "-" "_")
- today (str/replace (str (utc-date)) "-" "_")]
- (are [s] (old-dataset-name? s)
- "2023_02_01_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2023_01_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2022_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- ;; 2 days ago is old
- (str two-days-ago "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"))
- (are [s] (not (old-dataset-name? s))
- "2050_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "v4_test-data"
- ;; yesterday is not old (must be MORE than 1 day)
- (str yesterday "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")
- ;; today is not old
- (str today "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")
- ;; invalid dates are not old - only delete datasets matching our known format
- "2022_00_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2022_13_01_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2022_02_31_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")))
- (testing "new-format names (with hour) - more than 3 hours old"
- (are [s] (old-dataset-name? s)
+ (testing "names - more than 3 hours old"
+ (are [s] (old-temp-dataset? 3 s)
;; Ancient dates are old
- "2023_02_01_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2023_02_01_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "temp_2023_02_01_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "temp_2023_02_01_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
;; 5 hours ago is old
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -5)) "test-data"))
- (are [s] (not (old-dataset-name? s))
+ (unique-prefix* "test-data" (u.date/add (utc-date-time) :hour -5)))
+ (are [s] (not (old-temp-dataset? 3 s))
;; Current time is not old
- (str (unique-prefix*) "test-data")
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -1)) "test-data")
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -2)) "test-data")
+ (unique-prefix* "test-data")
+ (unique-prefix* "test-data" (u.date/add (utc-date-time) :hour -1))
+ (unique-prefix* "test-data" (u.date/add (utc-date-time) :hour -2))
;; 3 hours ago by hour is NOT old: the name records only the hour, so it may have been stamped as late as
;; :59 and be barely over 2 hours old. Collecting it here is what let a 2 hour threshold reach inside a
;; running driver job.
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -3)) "test-data")
+ (unique-prefix* "test-data" (u.date/add (utc-date-time) :hour -3))
;; Future dates are not old
"2050_02_17_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
;; invalid hour is not old - only delete datasets matching our known format
- "2023_02_01_25_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"))
- (testing "explicit hours-threshold, as passed by the nightly orphan sweep"
- (are [s] (old-dataset-name? s 2)
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -4)) "test-data")
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -5)) "test-data"))
- (are [s] (not (old-dataset-name? s 2))
- (str (unique-prefix*) "test-data")
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -1)) "test-data")
- ;; the guarantee the sweep relies on: nothing is collected before a full 2 hours have elapsed, no matter where
- ;; in its hour the name was stamped
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -2)) "test-data"))
- (testing "a smaller threshold does not make old-format (date-only) names collectable any sooner"
- (let [today (str/replace (str (utc-date)) "-" "_")]
- (is (not (old-dataset-name? (str today "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data") 2)))))))
+ "2023_02_01_25_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")))
diff --git a/test/metabase/test/data/gc.clj b/test/metabase/test/data/gc.clj
index 7b44d000364..17555f638e3 100644
--- a/test/metabase/test/data/gc.clj
+++ b/test/metabase/test/data/gc.clj
@@ -9,15 +9,6 @@
(set! *warn-on-reflection* true)
-(def ^:private min-temp-data-hours
- "Floor on `:temp-data-hours`. Must stay above the longest driver job (90 min, in `drivers-stress-test.yml`) or we
- delete a live run's data."
- 2)
-
-(def ^:private min-fixture-hours
- "Floor on `:fixture-hours`. Runs share these datasets; collecting them hourly would make every run rebuild its data."
- 24)
-
(def ^:private default-drivers
"Drivers implementing [[tx/gc-orphans!]]. Athena and Databricks are excluded: their datasets are preloaded, not
created by tests."
@@ -76,15 +67,16 @@
(let [results (try
(vec (tx/gc-orphans! driver options))
(catch Exception e
+ (def ee e)
[{:name nil, :status :failed, :error (ex-message e)}]))
remaining (census! driver)
- {deleted :deleted, failed :failed} (group-by :status results)]
+ {:keys [deleted failed]} (group-by :status results)]
(tx/print-progress! driver "%d deleted, %d failed" (count deleted) (count failed))
(doseq [{dataset-name :name, :keys [error]} failed]
- (tx/print-progress! driver "FAILED %s: %s" (or dataset-name "<server unreachable>") error))
+ (tx/print-progress! driver "FAILED %s: %s" (or dataset-name "<unknown>") error))
{:driver (name driver)
- :deleted (vec deleted)
- :failed (vec failed)
+ :deleted deleted
+ :failed failed
:counts {:deleted (count deleted), :failed (count failed)}
:before before
:remaining remaining})))
@@ -134,9 +126,9 @@
(render-names "Failed" failed render-object)))
(defn- render-markdown [{:keys [generated-at options totals drivers]}]
- (str (format "## DWH test data sweep\n\n**%d deleted, %d failed** — temp-data-hours %s, fixture-hours %s, %s\n\n"
+ (str (format "## DWH test data sweep\n\n**%d deleted, %d failed** — hours %s, %s\n\n"
(:deleted totals) (:failed totals)
- (:temp-data-hours options) (:fixture-hours options) generated-at)
+ (:hours options) generated-at)
(str/join (map render-driver drivers))))
(defn- write-report!
@@ -156,23 +148,17 @@
`:drivers` is a comma-separated string, defaulting to [[default-drivers]].
- `:temp-data-hours` (default 2) is the TTL for per-run garbage, `:fixture-hours` (default 72) for datasets runs
- share. Floored at [[min-temp-data-hours]] and [[min-fixture-hours]].
+ `:hours` is how old a dataset needs to be before it's considered orphaned. Defaults to 12.
+
+ `:tracked?` (default: nil) whether or not to also GC the legacy tracked datasets.
+ off by default to avoid disrupting backport branches but easy to run manually.
- `:report-dir` (default [[default-report-dir]]) receives `report.json` and `report.md`.
+ `:report-dir` (default: target/gc-report) receives `report.json` and `report.md`.
Failures don't stop the sweep. They are reported per object, written to the report, and fail the job at the end."
- [{:keys [drivers temp-data-hours fixture-hours report-dir]}]
- (let [options {:temp-data-hours (or temp-data-hours min-temp-data-hours)
- :fixture-hours (or fixture-hours 72)}]
- (doseq [[k floor] {:temp-data-hours min-temp-data-hours, :fixture-hours min-fixture-hours}
- :let [v (get options k)]]
- ;; whole hours, not merely numeric: the drivers interpolate these with `%d`, and a dispatch box lets someone
- ;; hand us 2.5
- (when-not (and (int? v) (>= v floor))
- (throw (ex-info (format "%s must be a whole number of hours >= %d; refusing to sweep with %s"
- k floor (pr-str v))
- {:option k, :value v}))))
+ [{:keys [drivers hours dry-run? tracked? report-dir]}]
+ (let [options {:hours (or hours 12) :dry-run? dry-run? :tracked? tracked?}]
+ (when (< (:hours options) 4) (throw (Exception. "Must specify at least 4 hours.")))
(let [swept (let [ds (parse-drivers drivers)]
;; drivers share nothing, so the job costs the slowest one rather than the sum of all three
(cp/with-shutdown! [pool (cp/threadpool (count ds))]
diff --git a/test/metabase/test/data/impl/get_or_create.clj b/test/metabase/test/data/impl/get_or_create.clj
index d0403172fa7..2b11bbad9e7 100644
--- a/test/metabase/test/data/impl/get_or_create.clj
+++ b/test/metabase/test/data/impl/get_or_create.clj
@@ -417,8 +417,7 @@
(u/with-timeout create-database-timeout-ms
;; ALWAYS CREATE DATABASE AND LOAD DATA AS UTC! Unless you like broken tests.
(test.tz/with-system-timezone-id! "UTC"
- (tx/create-db! driver dbdef)))))
- (tx/track-dataset driver dbdef))
+ (tx/create-db! driver dbdef))))))
(mu/defn- create-and-sync-Database!
"Add DB object to Metabase DB. Return an instance of `:model/Database`."
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index 7d4158d287a..6975f03e2f5 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -113,7 +113,9 @@
[:native-ddl {:optional true} [:sequential :any]]
;; When true, drivers that support it (e.g., MySQL) will disable FK checks during data loading.
;; Useful for datasets with self-referencing FKs that need to be inserted in a single batch.
- [:disable-fk-checks {:optional true} :boolean]]]]
+ [:disable-fk-checks {:optional true} :boolean]
+ ;; static datasets are not subject to periodic GC
+ [:static {:optional true} :boolean]]]]
(ms/InstanceOfClass DatabaseDefinition)])
;; TODO - this should probably be a protocol instead
@@ -398,7 +400,7 @@
what was found. Runs nightly (`.github/workflows/test.cleanup-dwh-data.yml`) because [[after-run]] never fires when
a CI job is cancelled.
- `options` are `:temp-data-hours` (per-run garbage) and `:fixture-hours` (datasets runs share).
+ `options` are `:hours` and `:dry-run?`
Returns one map per object it tried to delete:
@@ -698,17 +700,6 @@
(original-db-supports?# driver-arg# feature-arg# db-arg#)))]
~@body)))
-(defmulti track-dataset
- "Track the creation or the usage of the database.
- This is useful for cloud databases with shared state to ensure that stale datasets can be deleted and dataset loading is not done more than necessary. Pairs well with [[dataset-already-loaded?]]"
- {:arglists '([driver dbdef]) :added "0.56.0"}
- dispatch-on-driver-with-test-extensions
- :hierarchy #'driver/hierarchy)
-
-(defmethod track-dataset ::test-extensions
- [_driver _dbdef]
- nil)
-
(defmulti create-db!
"Create a new database from `database-definition`, including adding tables, fields, and foreign key constraints,
and load the appropriate data. (This refers to creating the actual *DBMS* database itself, *not* a Metabase
@@ -904,9 +895,12 @@
`(defdataset ~dataset-name nil ~definition))
([dataset-name docstring definition]
+ `(defdataset ~dataset-name ~docstring ~definition {:static true}))
+ ([dataset-name docstring definition options]
{:pre [(symbol? dataset-name)]}
- `(~(if config/is-dev? 'def 'defonce) ~(vary-meta dataset-name assoc :doc docstring, :tag `DatabaseDefinition)
- (dataset-definition ~(name dataset-name) ~definition))))
+ `(~(if config/is-dev? 'def 'defonce)
+ ~(vary-meta dataset-name assoc :doc docstring, :tag `DatabaseDefinition)
+ (dataset-definition ~(name dataset-name) ~definition ~options))))
(p.types/deftype+ ^:private NativeDatasetDefinition [dataset-name table-definitions native-ddl]
pretty/PrettyPrintable
@@ -947,7 +941,7 @@
(p.types/deftype+ ^:private EDNDatasetDefinition [dataset-name def]
pretty/PrettyPrintable
(pretty [_]
- (list `edn-dataset-definition dataset-name)))
+ (list `edn-dataset-definition dataset-name {})))
(defmethod get-dataset-definition EDNDatasetDefinition
[^EDNDatasetDefinition this]
@@ -956,12 +950,12 @@
(mu/defn edn-dataset-definition
"Define a new test dataset using the definition in an EDN file in the `test/metabase/test/data/dataset_definitions/`
directory. (Filename should be `dataset-name` + `.edn`.)"
- [dataset-name :- ms/NonBlankString]
+ [dataset-name :- ms/NonBlankString options :- map?]
(let [get-def (delay
(let [file-contents (edn/read-string
{:eof nil, :readers {'t #'u.date/parse}}
(slurp (str edn-definitions-dir dataset-name ".edn")))]
- (dataset-definition dataset-name file-contents)))]
+ (dataset-definition dataset-name file-contents options)))]
(EDNDatasetDefinition. dataset-name get-def)))
(defmacro defdataset-edn
@@ -969,7 +963,7 @@
directory. (Filename should be `dataset-name` + `.edn`.)"
[dataset-name & [docstring]]
`(defonce ~(vary-meta dataset-name assoc :doc docstring, :tag `EDNDatasetDefinition)
- (edn-dataset-definition ~(name dataset-name))))
+ (edn-dataset-definition ~(name dataset-name) {:static true})))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Transformed Dataset Definitions |