Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Cleanup data warehouses nightly (#79849)

Ryan Laurie7e3ab7a682f2
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 312af2cf532..06a0b7df6d6 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -15,7 +15,7 @@
:deprecated-var 86
:discouraged-java-method 4
:discouraged-namespace 81
- :discouraged-var 132
+ :discouraged-var 130
:duplicate-require 1
:equals-true 1
:main-without-gen-class 1
diff --git a/.github/workflows/reset-cloud-test-dataset.yml b/.github/workflows/reset-cloud-test-dataset.yml
deleted file mode 100644
index 947a3c497f6..00000000000
--- a/.github/workflows/reset-cloud-test-dataset.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-name: Reset Cloud Test Dataset
-
-on:
- workflow_dispatch:
- inputs:
- driver:
- description: 'Driver to reset dataset for'
- required: true
- default: 'snowflake'
- type: choice
- options:
- - snowflake
- - bigquery-cloud-sdk
- - redshift
- - databricks
- dataset-name:
- description: 'Dataset name to drop (e.g., test-data)'
- required: true
- default: 'test-data'
- type: string
-
-jobs:
- reset-dataset:
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 10
- env:
- CI: 'true'
- DRIVERS: ${{ inputs.driver }}
- # Snowflake
- MB_SNOWFLAKE_TEST_USER: METABASE CI
- MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}
- MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}
- MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}
- MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}
- # BigQuery
- MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}
- MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}
- MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}
- MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}
- MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
- MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
- MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
- # Redshift
- MB_REDSHIFT_TEST_USER: metabase_ci
- MB_REDSHIFT_TEST_DB: testdb
- MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }}
- MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}
- # Databricks
- MB_DATABRICKS_TEST_HOST: ${{ secrets.MB_DATABRICKS_JDBC_TEST_HOST }}
- MB_DATABRICKS_TEST_HTTP_PATH: ${{ secrets.MB_DATABRICKS_JDBC_TEST_HTTP_PATH }}
- MB_DATABRICKS_TEST_TOKEN: ${{ secrets.MB_DATABRICKS_JDBC_TEST_TOKEN }}
- MB_DATABRICKS_TEST_CATALOG: metabase_ci
- MB_DATABRICKS_TEST_CLIENT_ID: ${{ secrets.MB_DATABRICKS_JDBC_TEST_CLIENT_ID }}
- MB_DATABRICKS_TEST_OAUTH_SECRET: ${{ secrets.MB_DATABRICKS_JDBC_TEST_OAUTH_SECRET }}
- steps:
- - uses: actions/checkout@v6
- - uses: ./.github/actions/prepare-backend
- - name: Drop dataset
- run: >-
- clojure -X:dev:drivers:drivers-dev:test
- metabase.test.data.impl/drop-dataset!
- :driver '"${{ inputs.driver }}"'
- :dataset-name '"${{ inputs.dataset-name }}"'
diff --git a/.github/workflows/test.cleanup-dwh-data.yml b/.github/workflows/test.cleanup-dwh-data.yml
new file mode 100644
index 00000000000..66db86572de
--- /dev/null
+++ b/.github/workflows/test.cleanup-dwh-data.yml
@@ -0,0 +1,81 @@
+name: Cleanup DWH Test Data
+
+on:
+ schedule:
+ - cron: '0 7 * * *'
+ workflow_dispatch:
+ inputs:
+ drivers:
+ description: 'Drivers to sweep (comma-separated)'
+ required: false
+ default: 'snowflake,bigquery-cloud-sdk,redshift'
+ type: string
+ temp-data-hours:
+ description: 'TTL in hours for per-run garbage (minimum 2)'
+ required: false
+ default: '2'
+ type: string
+ fixture-hours:
+ description: 'TTL in hours for datasets runs share (minimum 24)'
+ required: false
+ default: '96'
+ type: string
+
+# `inputs.*` is empty on the scheduled run, so each falls back to the nightly value.
+env:
+ GC_TEMP_HOURS: ${{ inputs.temp-data-hours || 2 }} # for temp data
+ GC_FIXTURE_HOURS: ${{ inputs.fixture-hours || 96 }} # for reusable data
+ GC_DRIVERS: ${{ inputs.drivers || 'snowflake,bigquery-cloud-sdk,redshift' }}
+
+concurrency:
+ group: test-cleanup-dwh-data
+ cancel-in-progress: false
+
+permissions:
+ contents: read
+
+jobs:
+ gc:
+ name: Sweep orphaned test data
+ if: ${{ github.repository == 'metabase/metabase' }}
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 20
+ env:
+ CI: 'true'
+ # Snowflake
+ MB_SNOWFLAKE_TEST_USER: METABASE CI
+ MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}
+ MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}
+ MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}
+ # BigQuery
+ MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
+ # Redshift
+ MB_REDSHIFT_TEST_USER: metabase_ci
+ MB_REDSHIFT_TEST_DB: testdb
+ MB_REDSHIFT_TEST_DB_ROUTING: dev
+ MB_REDSHIFT_TEST_HOSTS: ${{ secrets.MB_REDSHIFT_TEST_HOSTS }}
+ MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }}
+ MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}
+ steps:
+ - uses: actions/checkout@v6
+ - uses: ./.github/actions/prepare-backend
+ - name: Sweep orphaned test data
+ run: |
+ clojure -X:dev:drivers:drivers-dev:test metabase.test.data.gc/gc-orphans! \
+ :drivers "\"${{ env.GC_DRIVERS }}\"" \
+ :temp-data-hours "${{ env.GC_TEMP_HOURS }}" \
+ :fixture-hours "${{ env.GC_FIXTURE_HOURS }}" \
+ :report-dir '"target/gc-report"'
+
+ # The report is most wanted on the runs that fail, so upload it regardless of the sweep's outcome.
+ - name: Upload sweep report
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses: actions/upload-artifact@v7
+ with:
+ name: dwh-gc-report-${{ github.run_id }}-attempt-${{ github.run_attempt }}
+ path: target/gc-report
+ if-no-files-found: warn
diff --git a/dev/src/dev/portal.clj b/dev/src/dev/portal.clj
index a1a2f4e2582..fba45b89c75 100644
--- a/dev/src/dev/portal.clj
+++ b/dev/src/dev/portal.clj
@@ -69,7 +69,7 @@
:dev.debug-qp/transformed-result, :dev.debug-qp/transformed-row}
tag)
(send-log (with-meta [before after]
- {:portal.viewer/default :portal.viewer/diff})
+ {:portal.viewer/default :portal.viewer/diff})
(update (meta middleware-var) :ns #(.name %)))
(send-log event)))
@@ -91,10 +91,10 @@
~@(when (seq args)
(mapcat identity args))
~after-sym ~(if (seq args)
- ;; Has args: a list with their new symbols!
- `(-> ~before-sym ~(list* (first form) (map first args)))
- ;; No args: just inline the form.
- `(-> ~before-sym ~form))]
+ ;; Has args: a list with their new symbols!
+ `(-> ~before-sym ~(list* (first form) (map first args)))
+ ;; No args: just inline the form.
+ `(-> ~before-sym ~form))]
(tap> [~(list `quote form)
~@(when (seq args)
`[(into {} [~@(for [[sym form] args]
diff --git a/mage/src/mage/jar_download.clj b/mage/src/mage/jar_download.clj
index b725191dcc4..bec96f539b4 100644
--- a/mage/src/mage/jar_download.clj
+++ b/mage/src/mage/jar_download.clj
@@ -190,7 +190,6 @@
(when-not (fs/exists? (io/file dir))
(println (c/blue "Creating directory " dir))
(fs/create-dirs dir))
-
(if (fs/exists? jar-path)
(println (str "Already downloaded " (c/green dl-url) " to " (c/green jar-path)
", size: " (c/red (c/bold (jar-size-mb (fs/size jar-path))))))
@@ -276,7 +275,6 @@
(println "env:")
(doseq [[k v] extra-env]
(println (str " " (c/yellow k) "=" (c/green v))))
-
(when (not (get extra-env "MB_PREMIUM_EMBEDDING_TOKEN"))
;; Check that the embedding token is set:
(u/env "MB_PREMIUM_EMBEDDING_TOKEN"
@@ -286,7 +284,6 @@
:latest-version latest-version
:jar-path jar-path
:babashka/exit 1}))))
-
(println (str "Socket repl will open on " (c/green socket-port) ". "
"See: https://lambdaisland.com/guides/clojure-repls/clojure-repls#org259d775"))
(println (str "\n\n Open in browser: " (c/magenta "http://localhost:" port) "\n"))
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
index 00deaad3804..eb4f6263389 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
@@ -385,20 +385,45 @@
(project-id)
(track-debounce-seconds))))))
+(def ^:private old-dataset-name-pattern "sha_%")
+
+(defn- old-dataset-names
+ "Names of test datasets older than `hours`: tracked ones nothing has accessed in that long, plus untracked ones
+ created that long ago. Excludes the current `test-data`, which [[destroy-dataset!]] refuses to delete anyway.
+
+ Shared by [[delete-old-datasets!]] and the nightly sweep ([[tx/gc-orphans!]]), which differ only in `hours`."
+ [hours]
+ (let [current-test-data (test-dataset-id (tx/get-dataset-definition
+ (data.impl/resolve-dataset-definition *ns* 'test-data)))]
+ (into []
+ (comp (map first)
+ (remove #(= % current-test-data)))
+ (execute! (str "(SELECT `name` FROM `%1$s.metabase_test_tracking.datasets`"
+ " WHERE `name` LIKE '%2$s'"
+ " AND `accessed_at` < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%3$d hour))"
+ " UNION ALL "
+ "(select schema_name from `%1$s`.INFORMATION_SCHEMA.SCHEMATA d
+ where d.schema_name not in (select name from `%1$s.metabase_test_tracking.datasets`)
+ and d.schema_name like '%2$s'
+ and creation_time < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%3$d hour))")
+ (project-id)
+ old-dataset-name-pattern
+ hours))))
+
+(defn- drop-datasets!
+ "Delete each named dataset, reporting per dataset whether it went. See [[tx/gc-orphans!]] for the shape."
+ [dataset-ids]
+ (mapv (fn [dataset-id]
+ (try
+ (destroy-dataset! dataset-id)
+ {:name dataset-id, :status :deleted}
+ ;; usually just another job deleting the same dataset at the same time
+ (catch Exception e
+ {:name dataset-id, :status :failed, :error (ex-message e)})))
+ dataset-ids))
+
(defn delete-old-datasets! []
- (let [all-outdated (execute!
- (str "(SELECT `name` FROM `%1$s.metabase_test_tracking.datasets`"
- " WHERE `accessed_at` < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL %2$d SECOND))"
- " UNION ALL "
- "(select schema_name from `%1$s`.INFORMATION_SCHEMA.SCHEMATA d
- where d.schema_name not in (select name from `%1$s.metabase_test_tracking.datasets`)
- and d.schema_name like 'sha_%%'
- and creation_time < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL %2$d SECOND))")
- (project-id)
- (.toSeconds reap-after))]
- (doseq [outdated (map first all-outdated)]
- (log/info (u/format-color 'blue "Deleting temporary dataset: %s`." outdated))
- (destroy-dataset! outdated))))
+ (drop-datasets! (old-dataset-names (.toHours reap-after))))
(defonce ^:private deleted-old-datasets?
(atom false))
@@ -409,6 +434,17 @@
(when (compare-and-set! deleted-old-datasets? false true)
(delete-old-datasets!)))
+;; Nightly garbage collection of orphaned test datasets.
+(defmethod tx/gc-orphans! :bigquery-cloud-sdk
+ [_driver {:keys [fixture-hours]}]
+ (let [project (project-id)]
+ (mapv #(assoc % :server project) (drop-datasets! (old-dataset-names fixture-hours)))))
+
+(defmethod tx/count-datasets :bigquery-cloud-sdk
+ [_driver]
+ (let [project (project-id)]
+ {project (ffirst (execute! "SELECT COUNT(*) FROM `%s`.INFORMATION_SCHEMA.SCHEMATA" project))}))
+
(defn- setup-tracking-dataset!
"Idempotently create test tracking database"
[]
diff --git a/modules/drivers/redshift/test/metabase/test/data/redshift.clj b/modules/drivers/redshift/test/metabase/test/data/redshift.clj
index e28ac13b7f8..67d464717fc 100644
--- a/modules/drivers/redshift/test/metabase/test/data/redshift.clj
+++ b/modules/drivers/redshift/test/metabase/test/data/redshift.clj
@@ -205,11 +205,14 @@
:old-style-cache -- cache schemas without a `cache_info` table at all
Pure: makes 1-2 catalog queries but does NOT drop anything. Use the
- `drop-orphan-*!` fns to act on the result."
- [^java.sql.Connection conn]
+ `drop-orphan-*!` fns to act on the result.
+
+ `hours-threshold` is how old a test data schema must be to count as `:old`; nil for the usual default. Cache
+ schemas are classified by their own TTL and are unaffected by it."
+ [^java.sql.Connection conn hours-threshold]
(let [{old-convention :old
caches-with-info :cache} (reduce (fn [acc s]
- (cond (sql.tu.unique-prefix/old-dataset-name? s)
+ (cond (sql.tu.unique-prefix/old-dataset-name? s hours-threshold)
(update acc :old conj s)
(str/starts-with? s "metabase_cache_")
(update acc :cache conj s)
@@ -227,23 +230,26 @@
;;; --------------------------------- Destruction ----------------------------------
(defn- drop-orphan-schemas!
- "Drop every schema classified by [[orphan-schemas]] as expired/old. Per-entry
- try/catch: never let one orphan block the rest.
+ "Drop every schema classified by [[orphan-schemas]] as expired/old, reporting per schema whether it went. See
+ [[tx/gc-orphans!]] for the shape; the caller adds `:server`, since a Statement does not know which cluster it is
+ on. Never let one orphan block the rest.
Takes the orphan-map directly so callers can preview-then-drop without
re-querying. Caller owns the Statement."
[^java.sql.Statement stmt orphans]
- (let [drop-sql (fn [schema-name] (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema-name))]
- (doseq [[k fmt-str] [[:old "Dropping old data schema: %s"]
- [:expired-cache "Dropping expired cache schema: %s"]
- [:lacking-created-at "Dropping cache without created-at info: %s"]
- [:old-style-cache "Dropping old cache schema without `cache_info` table: %s"]]
- schema (get orphans k)]
- (log/infof fmt-str schema)
- (try
- (.execute stmt (drop-sql schema))
- (catch Throwable e
- (log/infof "Failed to drop %s, skipping: %s" schema (ex-message e)))))))
+ (mapv (fn [[fmt-str schema]]
+ (log/infof fmt-str schema)
+ (try
+ (.execute stmt (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema))
+ {:name schema, :status :deleted}
+ (catch Exception e
+ {:name schema, :status :failed, :error (ex-message e)})))
+ (for [[k fmt-str] [[:old "Dropping old data schema: %s"]
+ [:expired-cache "Dropping expired cache schema: %s"]
+ [:lacking-created-at "Dropping cache without created-at info: %s"]
+ [:old-style-cache "Dropping old cache schema without `cache_info` table: %s"]]
+ schema (get orphans k)]
+ [fmt-str schema])))
(defn- delete-old-schemas!
"Remove unneeded schemas from redshift. Local databases are thrown away after
@@ -253,10 +259,83 @@
Glue: thin wrapper that calls the enumerator + dropper in order. To preview
from a REPL, call [[orphan-schemas]] directly."
[^java.sql.Connection conn]
- (let [orphans (orphan-schemas conn)]
+ (let [orphans (orphan-schemas conn nil)]
(with-open [stmt (.createStatement conn)]
(drop-orphan-schemas! stmt orphans))))
+(defn- gc-hosts
+ "Every cluster tests run against, not just the one a run would pick. `MB_REDSHIFT_TEST_HOSTS` is the fleet
+ `drivers.yml` uses; `MB_REDSHIFT_TEST_HOST` is the cluster the stress-test workflows use, and nothing sets both.
+ Taking either alone -- as [[random-host]] does, correctly, for a single run -- leaves the other to grow into the
+ max-tables limit this sweep exists to prevent."
+ []
+ (or (not-empty (distinct (concat @hosts (some-> (tx/db-test-env-var :redshift :host) vector))))
+ (throw (ex-info "no Redshift hosts configured: set MB_REDSHIFT_TEST_HOSTS or MB_REDSHIFT_TEST_HOST" {}))))
+
+(defn- gc-connection-details
+ "Every cluster and database a leaked schema could be in. Built from env vars rather than
+ [[db-connection-details]], which pins one random host for the process (fine for tests, but `MB_REDSHIFT_TEST_HOSTS`
+ is several clusters) and computes schema filters we don't need via [[unique-session-schema]]."
+ []
+ (for [host (gc-hosts)
+ db [(tx/db-test-env-var :redshift :db "testdb")
+ (tx/db-test-env-var :redshift :db-routing "dev")]]
+ {:host host
+ :port (parse-long (tx/db-test-env-var :redshift :port "5439"))
+ :db db
+ :user (tx/db-test-env-var :redshift :user "metabase_ci")
+ :password (tx/db-test-env-var-or-throw :redshift :password)}))
+
+(defn- server-label
+ "`host/db`, the `:server` key identifying one cluster+database pair in the nightly report."
+ [{:keys [host db]}]
+ (str host "/" db))
+
+(defn- with-gc-connection
+ "Call `f` with a write-capable Connection to one cluster+database, or return `fallback` built from the exception: a
+ cluster that is down, or a database that does not exist on it, must not cost us the others."
+ [driver details f fallback]
+ (try
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ (sql-jdbc.conn/connection-details->spec driver details)
+ {:write? true}
+ f)
+ (catch Exception e
+ (fallback e))))
+
+(defmethod tx/gc-orphans! :redshift
+ [driver {:keys [temp-data-hours]}]
+ ;; Redshift schema names carry their own creation time, so we scan for age via the name
+ (into []
+ (mapcat (fn [details]
+ (let [server (server-label details)]
+ (with-gc-connection
+ driver details
+ (fn [^java.sql.Connection conn]
+ (with-open [stmt (.createStatement conn)]
+ (mapv #(assoc % :server server)
+ (drop-orphan-schemas! stmt (orphan-schemas conn temp-data-hours)))))
+ (fn [e]
+ [{:server server, :name nil, :status :failed, :error (ex-message e)}])))))
+ (gc-connection-details)))
+
+(defmethod tx/count-datasets :redshift
+ [driver]
+ (into {}
+ (map (fn [details]
+ (let [server (server-label details)]
+ [server (with-gc-connection
+ driver details
+ ;; every schema on the cluster, catalog schemas included: the number worth watching is
+ ;; total pressure toward the max-tables limit, not our share of it
+ (fn [^java.sql.Connection conn]
+ (reduce (fn [n _] (inc n)) 0 (fetch-schemas conn)))
+ (fn [e]
+ (log/errorf "[redshift] could not count schemas on %s: %s" server (ex-message e))
+ nil))])))
+ (gc-connection-details)))
+
(defn- create-session-schema! [^java.sql.Connection conn]
(with-open [stmt (.createStatement conn)]
(doseq [^String sql [(format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" (unique-session-schema))
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 123674275ac..91bd9519ebf 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -115,22 +115,34 @@
(jdbc/execute! spec [(format "DROP DATABASE \"%s\";" name)]))))
(defn- old-dataset-names
- "Return a collection of all dataset names that are old
- -- tracked that haven't been touched in a while or are not tracked and too old"
- []
- (let [days-ago -5
- ;; tracked UNION ALL untracked
- ;; NB. currently appears that the second half never shows anything; all
- ;; datasets currently appear to be tracked.
- query "select name from metabase_test_tracking.PUBLIC.datasets
- where accessed_at < dateadd(day, ?, current_timestamp())"]
- (into [] (map :name) (jdbc/reducible-query (no-db-connection-spec)
- [query days-ago]))))
+ "Names of test databases old enough to delete, oldest first.
+
+ [[qualified-db-name]] gives CI runs an `isolate_` name built from a random int -- per-run, never reused, so they
+ age from `created`. Everything else is a `sha_` name from a local run, content-addressed and reused, so those age
+ from `accessed_at` when the tracking table knows them.
+
+ `accessed_at` is only trustworthy for `sha_`: the tracking table is keyed on dataset hash and its MERGE never
+ updates `name`, so an `isolate_` row keeps the first run's name while later runs keep refreshing its `accessed_at`.
+
+ Compares timestamps directly; `timestampdiff` counts boundary crossings, so an hour bucket calls a 10:59 database
+ two hours old at 12:01."
+ [{:keys [temp-data-hours fixture-hours]}]
+ (into []
+ (map :database_name)
+ (jdbc/query (no-db-connection-spec)
+ [(format
+ "select d.database_name
+ from metabase_test_tracking.information_schema.databases d
+ left join metabase_test_tracking.PUBLIC.datasets t on t.name = d.database_name
+ where (startswith(d.database_name, 'isolate_')
+ and d.created < dateadd(hour, -%d, current_timestamp()))
+ or (startswith(d.database_name, 'sha_')
+ and coalesce(t.accessed_at, d.created) < dateadd(hour, -%d, current_timestamp()))
+ order by d.created"
+ temp-data-hours
+ fixture-hours)])))
;;; --------------------------------- Destruction ----------------------------------
-;;;
-;;; The whole-orchestration entry point ([[delete-old-test-data!]]) is the glue
-;;; and gets called from `tx/create-db!`.
(defn- with-write-stmt!
"Open a write-capable Snowflake connection + Statement, call `f` with the stmt,
@@ -145,55 +157,48 @@
(with-open [stmt (.createStatement conn)]
(apply f stmt args)))))
-(defn- drop-old-datasets!
- "Drop test datasets (databases) prefixed by `sha_` that too old."
- []
- (when-let [old-datasets (not-empty (old-dataset-names))]
+(defn- drop-datasets!
+ "Drop each named test database and un-track it, reporting per database whether it went. See [[tx/gc-orphans!]] for
+ the shape."
+ [dataset-names]
+ ;; nothing to drop is the common case on a healthy night; don't open a connection to discover that
+ (if (empty? dataset-names)
+ []
(with-write-stmt!
(fn [^java.sql.Statement stmt]
- (doseq [dataset-name old-datasets]
- ;; test-harness cleanup output goes to the CI console, not the app log
- #_{:clj-kondo/ignore [:discouraged-var]}
- (println "[Snowflake] Deleting old dataset:" dataset-name)
- (try
- (.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))
- (.execute stmt (format "delete from metabase_test_tracking.PUBLIC.datasets where name = '%s';"
- dataset-name))
- ;; if this fails for some reason it's probably just because some other job tried to delete the dataset at the
- ;; same time. No big deal. Just log this and carry on trying to delete the other datasets. If we don't end up
- ;; deleting anything it's not the end of the world because it won't affect our ability to run our tests
- (catch Throwable e
- ;; test-harness cleanup output goes to the CI console, not the app log
- #_{:clj-kondo/ignore [:discouraged-var]}
- (println "[Snowflake] Error deleting old dataset:" (ex-message e)))))))))
-
-(defn- delete-old-test-data!
- "Delete old test data: datasets (databases) prefixed by sha_ that haven't been
- accessed in a while."
- []
- ;; the printlns are on purpose because we want them to show up when running tests, even on CI, to make sure this
- ;; stuff is working correctly. We can change it to `log` in the future when we're satisfied everything is working as
- ;; intended -- Cam
- #_{:clj-kondo/ignore [:discouraged-var]}
- (println "[Snowflake] deleting old test data...")
- ;; disabling this temporarily as it has caused very difficult-to-debug failures
- ;; in CI. even tho the datasets *have* been accessed recently, they are still
- ;; being deleted and reinserted, with race conditions that cause some data to be
- ;; inserted three times. this does mean that if datasets change, old versions
- ;; will not be cleaned up automatically and will need to be manually GCed.
- ;; local testing shows that identifying old datasets works correctly, but
- ;; sometimes randomly in CI it seems to decide that datasets are old and
- ;; deletes them even tho they are not old.
- #_(drop-old-datasets!))
-
-(defonce ^:private deleted-old-test-data?
- (atom false))
+ (mapv (fn [dataset-name]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (println "[Snowflake] Deleting old dataset:" dataset-name)
+ (try
+ (.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))
+ (.execute stmt (format "delete from metabase_test_tracking.PUBLIC.datasets where name = '%s';"
+ dataset-name))
+ {:name dataset-name, :status :deleted}
+ ;; usually just another job deleting the same dataset at the same time
+ (catch Exception e
+ {:name dataset-name, :status :failed, :error (ex-message e)})))
+ dataset-names)))))
+
+;;; --------------------------------- Orphan GC ----------------------------------
+;;;
+;;; Nightly sweep (`.github/workflows/test.cleanup-dwh-data.yml`). This replaces the old in-process cleanup, which
+;;; ran on every job and was disabled for causing hard-to-debug CI failures.
-(defn- delete-old-test-data-if-needed!
- "Call [[delete-old-test-data!]], only if we haven't done so already."
+(defn- account
+ "Label for the Snowflake account under sweep, used as the `:server` key in the nightly report."
[]
- (when (compare-and-set! deleted-old-test-data? false true)
- (delete-old-test-data!)))
+ (tx/db-test-env-var-or-throw :snowflake :account))
+
+(defmethod tx/gc-orphans! :snowflake
+ [_driver options]
+ (let [server (account)]
+ (mapv #(assoc % :server server) (drop-datasets! (old-dataset-names options)))))
+
+(defmethod tx/count-datasets :snowflake
+ [_driver]
+ {(account) (:count (first (jdbc/query (no-db-connection-spec)
+ ["select count(*) as count
+ from metabase_test_tracking.information_schema.databases"])))})
(defn- set-current-user-timezone!
[timezone]
@@ -209,8 +214,6 @@
[driver db-def & options]
;; qualify the DB name with the unique prefix
(let [db-def (assoc db-def :database-name (qualified-db-name db-def))]
- ;; clean up any old test data (datasets)
- (delete-old-test-data-if-needed!)
;; Snowflake by default uses America/Los_Angeles timezone. See https://docs.snowflake.com/en/sql-reference/parameters#timezone.
;; We expect UTC in tests. Hence fixing [[metabase.query-processor.timezone/database-timezone-id]] (PR #36413)
;; produced lot of failures. Following expression addresses that, setting timezone for the test user.
@@ -432,8 +435,8 @@
FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
WHERE query_text LIKE 'DROP DATABASE %'
ORDER BY end_time DESC limit 64"])
- (old-dataset-names)
- (drop-old-datasets!)
+ ;; preview what the nightly sweep would collect, at its own thresholds
+ (old-dataset-names {:temp-data-hours 2, :fixture-hours 72})
(into [] (jdbc/reducible-query (no-db-connection-spec) ["select * from metabase_test_tracking.PUBLIC.datasets"]))
;; Tracked databases ordered by age
(->> ["select d.name, d.accessed_at, i.created, timestampdiff('minute', i.created, d.accessed_at) as diff, timestampdiff('minute', i.created, current_timestamp()) as age
diff --git a/test/metabase/driver/sql/test_util/unique_prefix.clj b/test/metabase/driver/sql/test_util/unique_prefix.clj
index bfebb8138a0..72100d03522 100644
--- a/test/metabase/driver/sql/test_util/unique_prefix.clj
+++ b/test/metabase/driver/sql/test_util/unique_prefix.clj
@@ -64,36 +64,46 @@
(memoize unique-prefix*))
(def ^:private old-dataset-hours-threshold
- "Number of hours after which a dataset is considered old and can be deleted. Because schema names only include the
- hour (not minutes/seconds), actual deletion can occur anywhere from N to N+1 hours after creation. Set this high
- enough that any running test won't have its schema deleted mid-run."
+ "Number of hours after which a dataset is considered old and can be deleted. Set this high enough that any running
+ test won't have its schema deleted mid-run."
3)
(defn old-dataset-name?
"Is this dataset name old enough to be deleted?
- For new-format names (with hour): checks if more than [[old-dataset-hours-threshold]] hours old.
- For old-format names (date only): checks if more than 1 day old for backwards compatibility.
+ For new-format names (with hour): more than `hours-threshold` hours old, defaulting to
+ [[old-dataset-hours-threshold]]. Hour precision resolves conservatively, so a dataset goes between N and N+1 hours
+ after creation, never before N.
+
+ For old-format names (date only): more than 1 day old. Not parameterized -- those truncate to midnight, so an
+ hours-based threshold could delete one stamped 23:00 out from under a running test.
+
If the date/time is invalid, we return false (not old) to be safe - we only want to delete
datasets that match our known format."
- [dataset-name]
- ;; Try new format first: YYYY_MM_DD_HH_<uuid>_...
- (if-let [[_ year month day hour] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_(\d{2})_.*$" dataset-name)]
- (let [dataset-date-time (try
- (t/local-date-time (parse-long year) (parse-long month) (parse-long day) (parse-long hour) 0)
- (catch Throwable _ nil))]
- (if-not dataset-date-time
- false
- (t/before? dataset-date-time (u.date/add (utc-date-time) :hour (- old-dataset-hours-threshold)))))
- ;; TODO (bryan 12-23-25) Remove old-format checks when this has been running for a few days.
- ;; Fall back to old format: YYYY_MM_DD_<uuid>_...
- (when-let [[_ year month day] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_.*$" dataset-name)]
- (let [dataset-date (try
- (t/local-date (parse-long year) (parse-long month) (parse-long day))
- (catch Throwable _ nil))]
- (if-not dataset-date
- false
- (t/before? dataset-date (u.date/add (utc-date) :day -1)))))))
+ ([dataset-name]
+ (old-dataset-name? dataset-name nil))
+ ([dataset-name hours-threshold]
+ ;; Try new format first: YYYY_MM_DD_HH_<uuid>_...
+ (if-let [[_ year month day hour] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_(\d{2})_.*$" dataset-name)]
+ (let [dataset-date-time (try
+ (t/local-date-time (parse-long year) (parse-long month) (parse-long day) (parse-long hour) 0)
+ (catch Throwable _ nil))]
+ (if-not dataset-date-time
+ false
+ ;; a name stamped 10 was created in [10:00, 11:00), so age from the end of the hour -- otherwise an N hour
+ ;; threshold collects things only N-1 hours old
+ (t/before? (u.date/add dataset-date-time :hour 1)
+ (u.date/add (utc-date-time)
+ :hour (- (or hours-threshold old-dataset-hours-threshold))))))
+ ;; TODO (bryan 12-23-25) Remove old-format checks when this has been running for a few days.
+ ;; Fall back to old format: YYYY_MM_DD_<uuid>_...
+ (when-let [[_ year month day] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_.*$" dataset-name)]
+ (let [dataset-date (try
+ (t/local-date (parse-long year) (parse-long month) (parse-long day))
+ (catch Throwable _ nil))]
+ (if-not dataset-date
+ false
+ (t/before? dataset-date (u.date/add (utc-date) :day -1))))))))
(deftest ^:parallel old-dataset-name?-test
(testing "old-format names (date only) - more than 1 day old"
@@ -122,18 +132,31 @@
;; Ancient dates are old
"2023_02_01_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
"2023_02_01_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- ;; 4 hours ago is old
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -4)) "test-data")
- ;; 3 hours ago by hour is old (due to hour truncation, could be up to 3:59 hours old)
- (str (unique-prefix* (u.date/add (utc-date-time) :hour -3)) "test-data"))
+ ;; 5 hours ago is old
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -5)) "test-data"))
(are [s] (not (old-dataset-name? s))
;; Current time is not old
(str (unique-prefix*) "test-data")
- ;; 1 hour ago is not old
(str (unique-prefix* (u.date/add (utc-date-time) :hour -1)) "test-data")
- ;; 2 hours ago is not old
(str (unique-prefix* (u.date/add (utc-date-time) :hour -2)) "test-data")
+ ;; 3 hours ago by hour is NOT old: the name records only the hour, so it may have been stamped as late as
+ ;; :59 and be barely over 2 hours old. Collecting it here is what let a 2 hour threshold reach inside a
+ ;; running driver job.
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -3)) "test-data")
;; Future dates are not old
"2050_02_17_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
;; invalid hour is not old - only delete datasets matching our known format
- "2023_02_01_25_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")))
+ "2023_02_01_25_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"))
+ (testing "explicit hours-threshold, as passed by the nightly orphan sweep"
+ (are [s] (old-dataset-name? s 2)
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -4)) "test-data")
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -5)) "test-data"))
+ (are [s] (not (old-dataset-name? s 2))
+ (str (unique-prefix*) "test-data")
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -1)) "test-data")
+ ;; the guarantee the sweep relies on: nothing is collected before a full 2 hours have elapsed, no matter where
+ ;; in its hour the name was stamped
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -2)) "test-data"))
+ (testing "a smaller threshold does not make old-format (date-only) names collectable any sooner"
+ (let [today (str/replace (str (utc-date)) "-" "_")]
+ (is (not (old-dataset-name? (str today "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data") 2)))))))
diff --git a/test/metabase/test/data/gc.clj b/test/metabase/test/data/gc.clj
new file mode 100644
index 00000000000..a339aa69bcb
--- /dev/null
+++ b/test/metabase/test/data/gc.clj
@@ -0,0 +1,152 @@
+(ns metabase.test.data.gc
+ "Periodic sweep of orphaned test data in our shared cloud warehouses"
+ (:require
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabase.test.data.interface :as tx]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private min-temp-data-hours
+ "Floor on `:temp-data-hours`. Must stay above the longest driver job (90 min, in `drivers-stress-test.yml`) or we
+ delete a live run's data."
+ 2)
+
+(def ^:private min-fixture-hours
+ "Floor on `:fixture-hours`. Runs share these datasets; collecting them hourly would make every run rebuild its data."
+ 24)
+
+(def ^:private default-drivers
+ "Drivers implementing [[tx/gc-orphans!]]. Athena and Databricks are excluded: their datasets are preloaded, not
+ created by tests."
+ [:snowflake :bigquery-cloud-sdk :redshift])
+
+(def ^:private default-report-dir
+ "Where [[write-report!]] leaves its files for the workflow to upload as an artifact."
+ "target/gc-report")
+
+(defn- parse-drivers
+ "Driver keywords from a comma-separated string, defaulting to [[default-drivers]]. Blanks are dropped -- an empty
+ string is truthy, and would otherwise become the nameless keyword."
+ [drivers]
+ (or (seq (into [] (comp (map str/trim) (remove str/blank?) (map keyword))
+ (str/split (or drivers "") #",")))
+ default-drivers))
+
+(defn- sweep-driver!
+ "Sweep one driver and take a census of what is left, each best effort. A driver that cannot connect at all, or a
+ census that fails, must cost neither the other drivers nor the deletions this one did manage."
+ [driver options]
+ (let [results (try
+ (vec (tx/gc-orphans! driver options))
+ (catch Exception e
+ [{:name nil, :status :failed, :error (ex-message e)}]))
+ remaining (try
+ (tx/count-datasets driver)
+ (catch Exception e
+ (log/errorf "[%s] could not count remaining datasets: %s" (name driver) (ex-message e))
+ {}))
+ {deleted :deleted, failed :failed} (group-by :status results)]
+ (log/infof "[%s] %d deleted, %d failed" (name driver) (count deleted) (count failed))
+ (doseq [{dataset-name :name, :keys [error]} failed]
+ (log/errorf "[%s] %s: %s" (name driver) (or dataset-name "<server unreachable>") error))
+ {:driver (name driver)
+ :deleted (vec deleted)
+ :failed (vec failed)
+ :counts {:deleted (count deleted), :failed (count failed)}
+ :remaining remaining}))
+
+(defn- build-report
+ [options driver-reports]
+ {:generated-at (str (java.time.Instant/now))
+ :options options
+ :totals {:deleted (reduce + (map #(get-in % [:counts :deleted]) driver-reports))
+ :failed (reduce + (map #(get-in % [:counts :failed]) driver-reports))}
+ :drivers (vec driver-reports)})
+
+(defn- render-names
+ "A collapsed list of object names, or a note that there were none. Collapsed because a bad night can run to
+ hundreds of names and the summary still has to be skimmable."
+ [heading entries render-entry]
+ (if (empty? entries)
+ (format "%s: none\n\n" heading)
+ (format "<details><summary>%s (%d)</summary>\n\n%s\n</details>\n\n"
+ heading
+ (count entries)
+ (str/join "\n" (map render-entry entries)))))
+
+(defn- render-object
+ "One bullet for one object. Both `:server` and `:name` are absent when the driver itself could not be reached, and
+ the driver is already the heading, so an absent part is dropped rather than printed as `null`."
+ [{dataset-name :name, :keys [server error]}]
+ (str "- "
+ (str/join " — " (cond-> []
+ server (conj (format "`%s`" server))
+ dataset-name (conj (format "`%s`" dataset-name))))
+ (when (and error (or server dataset-name)) ": ")
+ error))
+
+(defn- render-driver [{:keys [driver counts remaining deleted failed]}]
+ (str (format "### %s — %d deleted, %d failed\n\n" driver (:deleted counts) (:failed counts))
+ (if (empty? remaining)
+ "Remaining datasets: unknown\n\n"
+ (str "Remaining datasets:\n\n"
+ (str/join "\n" (for [[server n] (sort-by key remaining)]
+ (format "- `%s`: %s" server (if (nil? n) "unknown" n))))
+ "\n\n"))
+ (render-names "Deleted" deleted render-object)
+ (render-names "Failed" failed render-object)))
+
+(defn- render-markdown [{:keys [generated-at options totals drivers]}]
+ (str (format "## DWH test data sweep\n\n**%d deleted, %d failed** — temp-data-hours %s, fixture-hours %s, %s\n\n"
+ (:deleted totals) (:failed totals)
+ (:temp-data-hours options) (:fixture-hours options) generated-at)
+ (str/join (map render-driver drivers))))
+
+(defn- write-report!
+ "Leave the report where the workflow can upload it, and append the human-readable half to the job summary so the
+ morning after is legible without downloading anything."
+ [report dir]
+ (let [markdown (render-markdown report)]
+ (.mkdirs (io/file dir))
+ (spit (io/file dir "report.json") (json/encode report))
+ (spit (io/file dir "report.md") markdown)
+ (when-let [summary-file (System/getenv "GITHUB_STEP_SUMMARY")]
+ (spit summary-file markdown :append true))
+ (log/infof "Wrote sweep report to %s" dir)))
+
+(defn gc-orphans!
+ "Sweep orphaned test data from each driver's shared cloud account.
+
+ `:drivers` is a comma-separated string, defaulting to [[default-drivers]].
+
+ `:temp-data-hours` (default 2) is the TTL for per-run garbage, `:fixture-hours` (default 72) for datasets runs
+ share. Floored at [[min-temp-data-hours]] and [[min-fixture-hours]].
+
+ `:report-dir` (default [[default-report-dir]]) receives `report.json` and `report.md`.
+
+ Failures don't stop the sweep. They are reported per object, written to the report, and fail the job at the end."
+ [{:keys [drivers temp-data-hours fixture-hours report-dir]}]
+ (let [options {:temp-data-hours (or temp-data-hours min-temp-data-hours)
+ :fixture-hours (or fixture-hours 72)}]
+ (doseq [[k floor] {:temp-data-hours min-temp-data-hours, :fixture-hours min-fixture-hours}
+ :let [v (get options k)]]
+ ;; whole hours, not merely numeric: the drivers interpolate these with `%d`, and a dispatch box lets someone
+ ;; hand us 2.5
+ (when-not (and (int? v) (>= v floor))
+ (throw (ex-info (format "%s must be a whole number of hours >= %d; refusing to sweep with %s"
+ k floor (pr-str v))
+ {:option k, :value v}))))
+ ;; we want the extensions, not their before-run hooks: Redshift's creates a session schema, which this job would
+ ;; then leak nightly
+ (binding [tx/*skip-before-run?* true]
+ (let [report (build-report options (mapv #(sweep-driver! % options) (parse-drivers drivers)))]
+ ;; write before throwing: the report is most wanted on the runs that fail
+ (write-report! report (or report-dir default-report-dir))
+ ;; fail loudly rather than going green having deleted nothing
+ (when (pos? (get-in report [:totals :failed]))
+ (throw (ex-info (format "%d object(s) could not be deleted"
+ (get-in report [:totals :failed]))
+ {:errors (into [] (comp (mapcat :failed) (map :error)) (:drivers report))})))))))
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index 24a5c0ad52b..0ecde4dd4bc 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -163,9 +163,15 @@
(defonce ^:private has-done-before-run (atom #{}))
+(def ^:dynamic *skip-before-run?*
+ "When true, loading a driver's test extensions does NOT run its [[before-run]] hook. Bound by the nightly sweep
+ ([[metabase.test.data.gc]]), which needs the extensions to dispatch [[gc-orphans!]] but not the hooks: Redshift's
+ creates a session schema, which the sweep would then leak nightly."
+ false)
+
;; this gets called below by [[load-test-extensions-namespace-if-needed]]
(defn- do-before-run-if-needed [driver]
- (when-not (@has-done-before-run driver)
+ (when-not (or *skip-before-run?* (@has-done-before-run driver))
(locking has-done-before-run
(when-not (@has-done-before-run driver)
(when (not= (get-method before-run driver) (get-method before-run ::test-extensions))
@@ -387,6 +393,52 @@
[driver]
(log/infof "%s has no after-run hooks." driver))
+(defmulti gc-orphans!
+ "Collect orphaned test data a previous run left behind in a shared cloud warehouse. Reports what was attempted, not
+ what was found. Runs nightly (`.github/workflows/test.cleanup-dwh-data.yml`) because [[after-run]] never fires when
+ a CI job is cancelled.
+
+ `options` are `:temp-data-hours` (per-run garbage) and `:fixture-hours` (datasets runs share).
+
+ Returns one map per object it tried to delete:
+
+ {:server \"cluster-a.example/testdb\" ; which account, project, or cluster+database it was on
+ :name \"sha__abc123_test_data\" ; nil only when the failure was reaching the server at all
+ :status :deleted or :failed
+ :error \"...\"} ; ex-message, present only when :failed
+
+ A failed object must still be reported by name: the nightly report exists so someone can tell which dataset is
+ stuck, and an exception alone does not carry that.
+
+ Implement only for drivers whose tests create objects in a shared cloud account -- not Athena or Databricks, whose
+ datasets are preloaded. Match only names the driver's own test extensions generate, never a bare wildcard."
+ {:arglists '([driver options])}
+ dispatch-on-driver-with-test-extensions
+ :hierarchy #'driver/hierarchy)
+
+(defmethod gc-orphans! ::test-extensions
+ [driver _options]
+ (log/infof "%s has no orphan GC; skipping." driver)
+ [])
+
+(defmulti count-datasets
+ "Census of every test dataset still on each server this driver's tests write to, taken after [[gc-orphans!]] has
+ run. Returns `{server-label count}`, keyed the same way as `:server` in [[gc-orphans!]]'s results.
+
+ Counts everything on the server, including datasets too young to be eligible for collection -- the number worth
+ watching is total pressure toward the account's dataset and table limits, not how much this run happened to be
+ allowed to touch.
+
+ A count that cannot be taken is `nil` rather than a missing key, so the report can say \"unknown\" instead of
+ silently omitting a server."
+ {:arglists '([driver])}
+ dispatch-on-driver-with-test-extensions
+ :hierarchy #'driver/hierarchy)
+
+(defmethod count-datasets ::test-extensions
+ [_driver]
+ {})
+
(defmulti drop-if-exists-and-create-db!
"Drop a database named `db-name` if it already exists, then create a new empty one with that name"
{:added "0.55.0" :arglists '([driver db-name & [just-drop]])}