The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Cleanup data warehouses nightly (#79849)
Ryan Laurie7e3ab7a682f2
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 312af2cf532..06a0b7df6d6 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -15,7 +15,7 @@:deprecated-var 86:discouraged-java-method 4:discouraged-namespace 81- :discouraged-var 132+ :discouraged-var 130:duplicate-require 1:equals-true 1:main-without-gen-class 1diff --git a/.github/workflows/reset-cloud-test-dataset.yml b/.github/workflows/reset-cloud-test-dataset.ymldeleted file mode 100644index 947a3c497f6..00000000000--- a/.github/workflows/reset-cloud-test-dataset.yml+++ /dev/null@@ -1,63 +0,0 @@-name: Reset Cloud Test Dataset--on:- workflow_dispatch:- inputs:- driver:- description: 'Driver to reset dataset for'- required: true- default: 'snowflake'- type: choice- options:- - snowflake- - bigquery-cloud-sdk- - redshift- - databricks- dataset-name:- description: 'Dataset name to drop (e.g., test-data)'- required: true- default: 'test-data'- type: string--jobs:- reset-dataset:- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}- timeout-minutes: 10- env:- CI: 'true'- DRIVERS: ${{ inputs.driver }}- # Snowflake- MB_SNOWFLAKE_TEST_USER: METABASE CI- MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}- MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}- MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}- MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}- # BigQuery- MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}- MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}- MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}- MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}- MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}- MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}- MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}- # Redshift- MB_REDSHIFT_TEST_USER: metabase_ci- MB_REDSHIFT_TEST_DB: testdb- MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }}- MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}- # Databricks- MB_DATABRICKS_TEST_HOST: ${{ secrets.MB_DATABRICKS_JDBC_TEST_HOST }}- MB_DATABRICKS_TEST_HTTP_PATH: ${{ secrets.MB_DATABRICKS_JDBC_TEST_HTTP_PATH }}- MB_DATABRICKS_TEST_TOKEN: ${{ secrets.MB_DATABRICKS_JDBC_TEST_TOKEN }}- MB_DATABRICKS_TEST_CATALOG: metabase_ci- MB_DATABRICKS_TEST_CLIENT_ID: ${{ secrets.MB_DATABRICKS_JDBC_TEST_CLIENT_ID }}- MB_DATABRICKS_TEST_OAUTH_SECRET: ${{ secrets.MB_DATABRICKS_JDBC_TEST_OAUTH_SECRET }}- steps:- - uses: actions/checkout@v6- - uses: ./.github/actions/prepare-backend- - name: Drop dataset- run: >-- clojure -X:dev:drivers:drivers-dev:test- metabase.test.data.impl/drop-dataset!- :driver '"${{ inputs.driver }}"'- :dataset-name '"${{ inputs.dataset-name }}"'diff --git a/.github/workflows/test.cleanup-dwh-data.yml b/.github/workflows/test.cleanup-dwh-data.ymlnew file mode 100644index 00000000000..66db86572de--- /dev/null+++ b/.github/workflows/test.cleanup-dwh-data.yml@@ -0,0 +1,81 @@+name: Cleanup DWH Test Data++on:+ schedule:+ - cron: '0 7 * * *'+ workflow_dispatch:+ inputs:+ drivers:+ description: 'Drivers to sweep (comma-separated)'+ required: false+ default: 'snowflake,bigquery-cloud-sdk,redshift'+ type: string+ temp-data-hours:+ description: 'TTL in hours for per-run garbage (minimum 2)'+ required: false+ default: '2'+ type: string+ fixture-hours:+ description: 'TTL in hours for datasets runs share (minimum 24)'+ required: false+ default: '96'+ type: string++# `inputs.*` is empty on the scheduled run, so each falls back to the nightly value.+env:+ GC_TEMP_HOURS: ${{ inputs.temp-data-hours || 2 }} # for temp data+ GC_FIXTURE_HOURS: ${{ inputs.fixture-hours || 96 }} # for reusable data+ GC_DRIVERS: ${{ inputs.drivers || 'snowflake,bigquery-cloud-sdk,redshift' }}++concurrency:+ group: test-cleanup-dwh-data+ cancel-in-progress: false++permissions:+ contents: read++jobs:+ gc:+ name: Sweep orphaned test data+ if: ${{ github.repository == 'metabase/metabase' }}+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 20+ env:+ CI: 'true'+ # Snowflake+ MB_SNOWFLAKE_TEST_USER: METABASE CI+ MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}+ MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}+ MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}+ MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}+ # BigQuery+ MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}+ # Redshift+ MB_REDSHIFT_TEST_USER: metabase_ci+ MB_REDSHIFT_TEST_DB: testdb+ MB_REDSHIFT_TEST_DB_ROUTING: dev+ MB_REDSHIFT_TEST_HOSTS: ${{ secrets.MB_REDSHIFT_TEST_HOSTS }}+ MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }}+ MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}+ steps:+ - uses: actions/checkout@v6+ - uses: ./.github/actions/prepare-backend+ - name: Sweep orphaned test data+ run: |+ clojure -X:dev:drivers:drivers-dev:test metabase.test.data.gc/gc-orphans! \+ :drivers "\"${{ env.GC_DRIVERS }}\"" \+ :temp-data-hours "${{ env.GC_TEMP_HOURS }}" \+ :fixture-hours "${{ env.GC_FIXTURE_HOURS }}" \+ :report-dir '"target/gc-report"'++ # The report is most wanted on the runs that fail, so upload it regardless of the sweep's outcome.+ - name: Upload sweep report+ if: ${{ !cancelled() }}+ continue-on-error: true+ uses: actions/upload-artifact@v7+ with:+ name: dwh-gc-report-${{ github.run_id }}-attempt-${{ github.run_attempt }}+ path: target/gc-report+ if-no-files-found: warndiff --git a/dev/src/dev/portal.clj b/dev/src/dev/portal.cljindex a1a2f4e2582..fba45b89c75 100644--- a/dev/src/dev/portal.clj+++ b/dev/src/dev/portal.clj@@ -69,7 +69,7 @@:dev.debug-qp/transformed-result, :dev.debug-qp/transformed-row}tag)(send-log (with-meta [before after]- {:portal.viewer/default :portal.viewer/diff})+ {:portal.viewer/default :portal.viewer/diff})(update (meta middleware-var) :ns #(.name %)))(send-log event)))@@ -91,10 +91,10 @@~@(when (seq args)(mapcat identity args))~after-sym ~(if (seq args)- ;; Has args: a list with their new symbols!- `(-> ~before-sym ~(list* (first form) (map first args)))- ;; No args: just inline the form.- `(-> ~before-sym ~form))]+ ;; Has args: a list with their new symbols!+ `(-> ~before-sym ~(list* (first form) (map first args)))+ ;; No args: just inline the form.+ `(-> ~before-sym ~form))](tap> [~(list `quote form)~@(when (seq args)`[(into {} [~@(for [[sym form] args]diff --git a/mage/src/mage/jar_download.clj b/mage/src/mage/jar_download.cljindex b725191dcc4..bec96f539b4 100644--- a/mage/src/mage/jar_download.clj+++ b/mage/src/mage/jar_download.clj@@ -190,7 +190,6 @@(when-not (fs/exists? (io/file dir))(println (c/blue "Creating directory " dir))(fs/create-dirs dir))-(if (fs/exists? jar-path)(println (str "Already downloaded " (c/green dl-url) " to " (c/green jar-path)", size: " (c/red (c/bold (jar-size-mb (fs/size jar-path))))))@@ -276,7 +275,6 @@(println "env:")(doseq [[k v] extra-env](println (str " " (c/yellow k) "=" (c/green v))))-(when (not (get extra-env "MB_PREMIUM_EMBEDDING_TOKEN"));; Check that the embedding token is set:(u/env "MB_PREMIUM_EMBEDDING_TOKEN"@@ -286,7 +284,6 @@:latest-version latest-version:jar-path jar-path:babashka/exit 1}))))-(println (str "Socket repl will open on " (c/green socket-port) ". ""See: https://lambdaisland.com/guides/clojure-repls/clojure-repls#org259d775"))(println (str "\n\n Open in browser: " (c/magenta "http://localhost:" port) "\n"))diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.cljindex 00deaad3804..eb4f6263389 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj@@ -385,20 +385,45 @@(project-id)(track-debounce-seconds))))))+(def ^:private old-dataset-name-pattern "sha_%")++(defn- old-dataset-names+ "Names of test datasets older than `hours`: tracked ones nothing has accessed in that long, plus untracked ones+ created that long ago. Excludes the current `test-data`, which [[destroy-dataset!]] refuses to delete anyway.++ Shared by [[delete-old-datasets!]] and the nightly sweep ([[tx/gc-orphans!]]), which differ only in `hours`."+ [hours]+ (let [current-test-data (test-dataset-id (tx/get-dataset-definition+ (data.impl/resolve-dataset-definition *ns* 'test-data)))]+ (into []+ (comp (map first)+ (remove #(= % current-test-data)))+ (execute! (str "(SELECT `name` FROM `%1$s.metabase_test_tracking.datasets`"+ " WHERE `name` LIKE '%2$s'"+ " AND `accessed_at` < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%3$d hour))"+ " UNION ALL "+ "(select schema_name from `%1$s`.INFORMATION_SCHEMA.SCHEMATA d+ where d.schema_name not in (select name from `%1$s.metabase_test_tracking.datasets`)+ and d.schema_name like '%2$s'+ and creation_time < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -%3$d hour))")+ (project-id)+ old-dataset-name-pattern+ hours))))++(defn- drop-datasets!+ "Delete each named dataset, reporting per dataset whether it went. See [[tx/gc-orphans!]] for the shape."+ [dataset-ids]+ (mapv (fn [dataset-id]+ (try+ (destroy-dataset! dataset-id)+ {:name dataset-id, :status :deleted}+ ;; usually just another job deleting the same dataset at the same time+ (catch Exception e+ {:name dataset-id, :status :failed, :error (ex-message e)})))+ dataset-ids))+(defn delete-old-datasets! []- (let [all-outdated (execute!- (str "(SELECT `name` FROM `%1$s.metabase_test_tracking.datasets`"- " WHERE `accessed_at` < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL %2$d SECOND))"- " UNION ALL "- "(select schema_name from `%1$s`.INFORMATION_SCHEMA.SCHEMATA d- where d.schema_name not in (select name from `%1$s.metabase_test_tracking.datasets`)- and d.schema_name like 'sha_%%'- and creation_time < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL %2$d SECOND))")- (project-id)- (.toSeconds reap-after))]- (doseq [outdated (map first all-outdated)]- (log/info (u/format-color 'blue "Deleting temporary dataset: %s`." outdated))- (destroy-dataset! outdated))))+ (drop-datasets! (old-dataset-names (.toHours reap-after))))(defonce ^:private deleted-old-datasets?(atom false))@@ -409,6 +434,17 @@(when (compare-and-set! deleted-old-datasets? false true)(delete-old-datasets!)))+;; Nightly garbage collection of orphaned test datasets.+(defmethod tx/gc-orphans! :bigquery-cloud-sdk+ [_driver {:keys [fixture-hours]}]+ (let [project (project-id)]+ (mapv #(assoc % :server project) (drop-datasets! (old-dataset-names fixture-hours)))))++(defmethod tx/count-datasets :bigquery-cloud-sdk+ [_driver]+ (let [project (project-id)]+ {project (ffirst (execute! "SELECT COUNT(*) FROM `%s`.INFORMATION_SCHEMA.SCHEMATA" project))}))+(defn- setup-tracking-dataset!"Idempotently create test tracking database"[]diff --git a/modules/drivers/redshift/test/metabase/test/data/redshift.clj b/modules/drivers/redshift/test/metabase/test/data/redshift.cljindex e28ac13b7f8..67d464717fc 100644--- a/modules/drivers/redshift/test/metabase/test/data/redshift.clj+++ b/modules/drivers/redshift/test/metabase/test/data/redshift.clj@@ -205,11 +205,14 @@:old-style-cache -- cache schemas without a `cache_info` table at allPure: makes 1-2 catalog queries but does NOT drop anything. Use the- `drop-orphan-*!` fns to act on the result."- [^java.sql.Connection conn]+ `drop-orphan-*!` fns to act on the result.++ `hours-threshold` is how old a test data schema must be to count as `:old`; nil for the usual default. Cache+ schemas are classified by their own TTL and are unaffected by it."+ [^java.sql.Connection conn hours-threshold](let [{old-convention :oldcaches-with-info :cache} (reduce (fn [acc s]- (cond (sql.tu.unique-prefix/old-dataset-name? s)+ (cond (sql.tu.unique-prefix/old-dataset-name? s hours-threshold)(update acc :old conj s)(str/starts-with? s "metabase_cache_")(update acc :cache conj s)@@ -227,23 +230,26 @@;;; --------------------------------- Destruction ----------------------------------(defn- drop-orphan-schemas!- "Drop every schema classified by [[orphan-schemas]] as expired/old. Per-entry- try/catch: never let one orphan block the rest.+ "Drop every schema classified by [[orphan-schemas]] as expired/old, reporting per schema whether it went. See+ [[tx/gc-orphans!]] for the shape; the caller adds `:server`, since a Statement does not know which cluster it is+ on. Never let one orphan block the rest.Takes the orphan-map directly so callers can preview-then-drop withoutre-querying. Caller owns the Statement."[^java.sql.Statement stmt orphans]- (let [drop-sql (fn [schema-name] (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema-name))]- (doseq [[k fmt-str] [[:old "Dropping old data schema: %s"]- [:expired-cache "Dropping expired cache schema: %s"]- [:lacking-created-at "Dropping cache without created-at info: %s"]- [:old-style-cache "Dropping old cache schema without `cache_info` table: %s"]]- schema (get orphans k)]- (log/infof fmt-str schema)- (try- (.execute stmt (drop-sql schema))- (catch Throwable e- (log/infof "Failed to drop %s, skipping: %s" schema (ex-message e)))))))+ (mapv (fn [[fmt-str schema]]+ (log/infof fmt-str schema)+ (try+ (.execute stmt (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema))+ {:name schema, :status :deleted}+ (catch Exception e+ {:name schema, :status :failed, :error (ex-message e)})))+ (for [[k fmt-str] [[:old "Dropping old data schema: %s"]+ [:expired-cache "Dropping expired cache schema: %s"]+ [:lacking-created-at "Dropping cache without created-at info: %s"]+ [:old-style-cache "Dropping old cache schema without `cache_info` table: %s"]]+ schema (get orphans k)]+ [fmt-str schema])))(defn- delete-old-schemas!"Remove unneeded schemas from redshift. Local databases are thrown away after@@ -253,10 +259,83 @@Glue: thin wrapper that calls the enumerator + dropper in order. To previewfrom a REPL, call [[orphan-schemas]] directly."[^java.sql.Connection conn]- (let [orphans (orphan-schemas conn)]+ (let [orphans (orphan-schemas conn nil)](with-open [stmt (.createStatement conn)](drop-orphan-schemas! stmt orphans))))+(defn- gc-hosts+ "Every cluster tests run against, not just the one a run would pick. `MB_REDSHIFT_TEST_HOSTS` is the fleet+ `drivers.yml` uses; `MB_REDSHIFT_TEST_HOST` is the cluster the stress-test workflows use, and nothing sets both.+ Taking either alone -- as [[random-host]] does, correctly, for a single run -- leaves the other to grow into the+ max-tables limit this sweep exists to prevent."+ []+ (or (not-empty (distinct (concat @hosts (some-> (tx/db-test-env-var :redshift :host) vector))))+ (throw (ex-info "no Redshift hosts configured: set MB_REDSHIFT_TEST_HOSTS or MB_REDSHIFT_TEST_HOST" {}))))++(defn- gc-connection-details+ "Every cluster and database a leaked schema could be in. Built from env vars rather than+ [[db-connection-details]], which pins one random host for the process (fine for tests, but `MB_REDSHIFT_TEST_HOSTS`+ is several clusters) and computes schema filters we don't need via [[unique-session-schema]]."+ []+ (for [host (gc-hosts)+ db [(tx/db-test-env-var :redshift :db "testdb")+ (tx/db-test-env-var :redshift :db-routing "dev")]]+ {:host host+ :port (parse-long (tx/db-test-env-var :redshift :port "5439"))+ :db db+ :user (tx/db-test-env-var :redshift :user "metabase_ci")+ :password (tx/db-test-env-var-or-throw :redshift :password)}))++(defn- server-label+ "`host/db`, the `:server` key identifying one cluster+database pair in the nightly report."+ [{:keys [host db]}]+ (str host "/" db))++(defn- with-gc-connection+ "Call `f` with a write-capable Connection to one cluster+database, or return `fallback` built from the exception: a+ cluster that is down, or a database that does not exist on it, must not cost us the others."+ [driver details f fallback]+ (try+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ (sql-jdbc.conn/connection-details->spec driver details)+ {:write? true}+ f)+ (catch Exception e+ (fallback e))))++(defmethod tx/gc-orphans! :redshift+ [driver {:keys [temp-data-hours]}]+ ;; Redshift schema names carry their own creation time, so we scan for age via the name+ (into []+ (mapcat (fn [details]+ (let [server (server-label details)]+ (with-gc-connection+ driver details+ (fn [^java.sql.Connection conn]+ (with-open [stmt (.createStatement conn)]+ (mapv #(assoc % :server server)+ (drop-orphan-schemas! stmt (orphan-schemas conn temp-data-hours)))))+ (fn [e]+ [{:server server, :name nil, :status :failed, :error (ex-message e)}])))))+ (gc-connection-details)))++(defmethod tx/count-datasets :redshift+ [driver]+ (into {}+ (map (fn [details]+ (let [server (server-label details)]+ [server (with-gc-connection+ driver details+ ;; every schema on the cluster, catalog schemas included: the number worth watching is+ ;; total pressure toward the max-tables limit, not our share of it+ (fn [^java.sql.Connection conn]+ (reduce (fn [n _] (inc n)) 0 (fetch-schemas conn)))+ (fn [e]+ (log/errorf "[redshift] could not count schemas on %s: %s" server (ex-message e))+ nil))])))+ (gc-connection-details)))+(defn- create-session-schema! [^java.sql.Connection conn](with-open [stmt (.createStatement conn)](doseq [^String sql [(format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" (unique-session-schema))diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.cljindex 123674275ac..91bd9519ebf 100644--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj@@ -115,22 +115,34 @@(jdbc/execute! spec [(format "DROP DATABASE \"%s\";" name)]))))(defn- old-dataset-names- "Return a collection of all dataset names that are old- -- tracked that haven't been touched in a while or are not tracked and too old"- []- (let [days-ago -5- ;; tracked UNION ALL untracked- ;; NB. currently appears that the second half never shows anything; all- ;; datasets currently appear to be tracked.- query "select name from metabase_test_tracking.PUBLIC.datasets- where accessed_at < dateadd(day, ?, current_timestamp())"]- (into [] (map :name) (jdbc/reducible-query (no-db-connection-spec)- [query days-ago]))))+ "Names of test databases old enough to delete, oldest first.++ [[qualified-db-name]] gives CI runs an `isolate_` name built from a random int -- per-run, never reused, so they+ age from `created`. Everything else is a `sha_` name from a local run, content-addressed and reused, so those age+ from `accessed_at` when the tracking table knows them.++ `accessed_at` is only trustworthy for `sha_`: the tracking table is keyed on dataset hash and its MERGE never+ updates `name`, so an `isolate_` row keeps the first run's name while later runs keep refreshing its `accessed_at`.++ Compares timestamps directly; `timestampdiff` counts boundary crossings, so an hour bucket calls a 10:59 database+ two hours old at 12:01."+ [{:keys [temp-data-hours fixture-hours]}]+ (into []+ (map :database_name)+ (jdbc/query (no-db-connection-spec)+ [(format+ "select d.database_name+ from metabase_test_tracking.information_schema.databases d+ left join metabase_test_tracking.PUBLIC.datasets t on t.name = d.database_name+ where (startswith(d.database_name, 'isolate_')+ and d.created < dateadd(hour, -%d, current_timestamp()))+ or (startswith(d.database_name, 'sha_')+ and coalesce(t.accessed_at, d.created) < dateadd(hour, -%d, current_timestamp()))+ order by d.created"+ temp-data-hours+ fixture-hours)])));;; --------------------------------- Destruction -----------------------------------;;;-;;; The whole-orchestration entry point ([[delete-old-test-data!]]) is the glue-;;; and gets called from `tx/create-db!`.(defn- with-write-stmt!"Open a write-capable Snowflake connection + Statement, call `f` with the stmt,@@ -145,55 +157,48 @@(with-open [stmt (.createStatement conn)](apply f stmt args)))))-(defn- drop-old-datasets!- "Drop test datasets (databases) prefixed by `sha_` that too old."- []- (when-let [old-datasets (not-empty (old-dataset-names))]+(defn- drop-datasets!+ "Drop each named test database and un-track it, reporting per database whether it went. See [[tx/gc-orphans!]] for+ the shape."+ [dataset-names]+ ;; nothing to drop is the common case on a healthy night; don't open a connection to discover that+ (if (empty? dataset-names)+ [](with-write-stmt!(fn [^java.sql.Statement stmt]- (doseq [dataset-name old-datasets]- ;; test-harness cleanup output goes to the CI console, not the app log- #_{:clj-kondo/ignore [:discouraged-var]}- (println "[Snowflake] Deleting old dataset:" dataset-name)- (try- (.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))- (.execute stmt (format "delete from metabase_test_tracking.PUBLIC.datasets where name = '%s';"- dataset-name))- ;; if this fails for some reason it's probably just because some other job tried to delete the dataset at the- ;; same time. No big deal. Just log this and carry on trying to delete the other datasets. If we don't end up- ;; deleting anything it's not the end of the world because it won't affect our ability to run our tests- (catch Throwable e- ;; test-harness cleanup output goes to the CI console, not the app log- #_{:clj-kondo/ignore [:discouraged-var]}- (println "[Snowflake] Error deleting old dataset:" (ex-message e)))))))))--(defn- delete-old-test-data!- "Delete old test data: datasets (databases) prefixed by sha_ that haven't been- accessed in a while."- []- ;; the printlns are on purpose because we want them to show up when running tests, even on CI, to make sure this- ;; stuff is working correctly. We can change it to `log` in the future when we're satisfied everything is working as- ;; intended -- Cam- #_{:clj-kondo/ignore [:discouraged-var]}- (println "[Snowflake] deleting old test data...")- ;; disabling this temporarily as it has caused very difficult-to-debug failures- ;; in CI. even tho the datasets *have* been accessed recently, they are still- ;; being deleted and reinserted, with race conditions that cause some data to be- ;; inserted three times. this does mean that if datasets change, old versions- ;; will not be cleaned up automatically and will need to be manually GCed.- ;; local testing shows that identifying old datasets works correctly, but- ;; sometimes randomly in CI it seems to decide that datasets are old and- ;; deletes them even tho they are not old.- #_(drop-old-datasets!))--(defonce ^:private deleted-old-test-data?- (atom false))+ (mapv (fn [dataset-name]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (println "[Snowflake] Deleting old dataset:" dataset-name)+ (try+ (.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))+ (.execute stmt (format "delete from metabase_test_tracking.PUBLIC.datasets where name = '%s';"+ dataset-name))+ {:name dataset-name, :status :deleted}+ ;; usually just another job deleting the same dataset at the same time+ (catch Exception e+ {:name dataset-name, :status :failed, :error (ex-message e)})))+ dataset-names)))))++;;; --------------------------------- Orphan GC ----------------------------------+;;;+;;; Nightly sweep (`.github/workflows/test.cleanup-dwh-data.yml`). This replaces the old in-process cleanup, which+;;; ran on every job and was disabled for causing hard-to-debug CI failures.-(defn- delete-old-test-data-if-needed!- "Call [[delete-old-test-data!]], only if we haven't done so already."+(defn- account+ "Label for the Snowflake account under sweep, used as the `:server` key in the nightly report."[]- (when (compare-and-set! deleted-old-test-data? false true)- (delete-old-test-data!)))+ (tx/db-test-env-var-or-throw :snowflake :account))++(defmethod tx/gc-orphans! :snowflake+ [_driver options]+ (let [server (account)]+ (mapv #(assoc % :server server) (drop-datasets! (old-dataset-names options)))))++(defmethod tx/count-datasets :snowflake+ [_driver]+ {(account) (:count (first (jdbc/query (no-db-connection-spec)+ ["select count(*) as count+ from metabase_test_tracking.information_schema.databases"])))})(defn- set-current-user-timezone![timezone]@@ -209,8 +214,6 @@[driver db-def & options];; qualify the DB name with the unique prefix(let [db-def (assoc db-def :database-name (qualified-db-name db-def))]- ;; clean up any old test data (datasets)- (delete-old-test-data-if-needed!);; Snowflake by default uses America/Los_Angeles timezone. See https://docs.snowflake.com/en/sql-reference/parameters#timezone.;; We expect UTC in tests. Hence fixing [[metabase.query-processor.timezone/database-timezone-id]] (PR #36413);; produced lot of failures. Following expression addresses that, setting timezone for the test user.@@ -432,8 +435,8 @@FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORYWHERE query_text LIKE 'DROP DATABASE %'ORDER BY end_time DESC limit 64"])- (old-dataset-names)- (drop-old-datasets!)+ ;; preview what the nightly sweep would collect, at its own thresholds+ (old-dataset-names {:temp-data-hours 2, :fixture-hours 72})(into [] (jdbc/reducible-query (no-db-connection-spec) ["select * from metabase_test_tracking.PUBLIC.datasets"]));; Tracked databases ordered by age(->> ["select d.name, d.accessed_at, i.created, timestampdiff('minute', i.created, d.accessed_at) as diff, timestampdiff('minute', i.created, current_timestamp()) as agediff --git a/test/metabase/driver/sql/test_util/unique_prefix.clj b/test/metabase/driver/sql/test_util/unique_prefix.cljindex bfebb8138a0..72100d03522 100644--- a/test/metabase/driver/sql/test_util/unique_prefix.clj+++ b/test/metabase/driver/sql/test_util/unique_prefix.clj@@ -64,36 +64,46 @@(memoize unique-prefix*))(def ^:private old-dataset-hours-threshold- "Number of hours after which a dataset is considered old and can be deleted. Because schema names only include the- hour (not minutes/seconds), actual deletion can occur anywhere from N to N+1 hours after creation. Set this high- enough that any running test won't have its schema deleted mid-run."+ "Number of hours after which a dataset is considered old and can be deleted. Set this high enough that any running+ test won't have its schema deleted mid-run."3)(defn old-dataset-name?"Is this dataset name old enough to be deleted?- For new-format names (with hour): checks if more than [[old-dataset-hours-threshold]] hours old.- For old-format names (date only): checks if more than 1 day old for backwards compatibility.+ For new-format names (with hour): more than `hours-threshold` hours old, defaulting to+ [[old-dataset-hours-threshold]]. Hour precision resolves conservatively, so a dataset goes between N and N+1 hours+ after creation, never before N.++ For old-format names (date only): more than 1 day old. Not parameterized -- those truncate to midnight, so an+ hours-based threshold could delete one stamped 23:00 out from under a running test.+If the date/time is invalid, we return false (not old) to be safe - we only want to deletedatasets that match our known format."- [dataset-name]- ;; Try new format first: YYYY_MM_DD_HH_<uuid>_...- (if-let [[_ year month day hour] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_(\d{2})_.*$" dataset-name)]- (let [dataset-date-time (try- (t/local-date-time (parse-long year) (parse-long month) (parse-long day) (parse-long hour) 0)- (catch Throwable _ nil))]- (if-not dataset-date-time- false- (t/before? dataset-date-time (u.date/add (utc-date-time) :hour (- old-dataset-hours-threshold)))))- ;; TODO (bryan 12-23-25) Remove old-format checks when this has been running for a few days.- ;; Fall back to old format: YYYY_MM_DD_<uuid>_...- (when-let [[_ year month day] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_.*$" dataset-name)]- (let [dataset-date (try- (t/local-date (parse-long year) (parse-long month) (parse-long day))- (catch Throwable _ nil))]- (if-not dataset-date- false- (t/before? dataset-date (u.date/add (utc-date) :day -1)))))))+ ([dataset-name]+ (old-dataset-name? dataset-name nil))+ ([dataset-name hours-threshold]+ ;; Try new format first: YYYY_MM_DD_HH_<uuid>_...+ (if-let [[_ year month day hour] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_(\d{2})_.*$" dataset-name)]+ (let [dataset-date-time (try+ (t/local-date-time (parse-long year) (parse-long month) (parse-long day) (parse-long hour) 0)+ (catch Throwable _ nil))]+ (if-not dataset-date-time+ false+ ;; a name stamped 10 was created in [10:00, 11:00), so age from the end of the hour -- otherwise an N hour+ ;; threshold collects things only N-1 hours old+ (t/before? (u.date/add dataset-date-time :hour 1)+ (u.date/add (utc-date-time)+ :hour (- (or hours-threshold old-dataset-hours-threshold))))))+ ;; TODO (bryan 12-23-25) Remove old-format checks when this has been running for a few days.+ ;; Fall back to old format: YYYY_MM_DD_<uuid>_...+ (when-let [[_ year month day] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_.*$" dataset-name)]+ (let [dataset-date (try+ (t/local-date (parse-long year) (parse-long month) (parse-long day))+ (catch Throwable _ nil))]+ (if-not dataset-date+ false+ (t/before? dataset-date (u.date/add (utc-date) :day -1))))))))(deftest ^:parallel old-dataset-name?-test(testing "old-format names (date only) - more than 1 day old"@@ -122,18 +132,31 @@;; Ancient dates are old"2023_02_01_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data""2023_02_01_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"- ;; 4 hours ago is old- (str (unique-prefix* (u.date/add (utc-date-time) :hour -4)) "test-data")- ;; 3 hours ago by hour is old (due to hour truncation, could be up to 3:59 hours old)- (str (unique-prefix* (u.date/add (utc-date-time) :hour -3)) "test-data"))+ ;; 5 hours ago is old+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -5)) "test-data"))(are [s] (not (old-dataset-name? s));; Current time is not old(str (unique-prefix*) "test-data")- ;; 1 hour ago is not old(str (unique-prefix* (u.date/add (utc-date-time) :hour -1)) "test-data")- ;; 2 hours ago is not old(str (unique-prefix* (u.date/add (utc-date-time) :hour -2)) "test-data")+ ;; 3 hours ago by hour is NOT old: the name records only the hour, so it may have been stamped as late as+ ;; :59 and be barely over 2 hours old. Collecting it here is what let a 2 hour threshold reach inside a+ ;; running driver job.+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -3)) "test-data");; Future dates are not old"2050_02_17_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data";; invalid hour is not old - only delete datasets matching our known format- "2023_02_01_25_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")))+ "2023_02_01_25_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"))+ (testing "explicit hours-threshold, as passed by the nightly orphan sweep"+ (are [s] (old-dataset-name? s 2)+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -4)) "test-data")+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -5)) "test-data"))+ (are [s] (not (old-dataset-name? s 2))+ (str (unique-prefix*) "test-data")+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -1)) "test-data")+ ;; the guarantee the sweep relies on: nothing is collected before a full 2 hours have elapsed, no matter where+ ;; in its hour the name was stamped+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -2)) "test-data"))+ (testing "a smaller threshold does not make old-format (date-only) names collectable any sooner"+ (let [today (str/replace (str (utc-date)) "-" "_")]+ (is (not (old-dataset-name? (str today "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data") 2)))))))diff --git a/test/metabase/test/data/gc.clj b/test/metabase/test/data/gc.cljnew file mode 100644index 00000000000..a339aa69bcb--- /dev/null+++ b/test/metabase/test/data/gc.clj@@ -0,0 +1,152 @@+(ns metabase.test.data.gc+ "Periodic sweep of orphaned test data in our shared cloud warehouses"+ (:require+ [clojure.java.io :as io]+ [clojure.string :as str]+ [metabase.test.data.interface :as tx]+ [metabase.util.json :as json]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(def ^:private min-temp-data-hours+ "Floor on `:temp-data-hours`. Must stay above the longest driver job (90 min, in `drivers-stress-test.yml`) or we+ delete a live run's data."+ 2)++(def ^:private min-fixture-hours+ "Floor on `:fixture-hours`. Runs share these datasets; collecting them hourly would make every run rebuild its data."+ 24)++(def ^:private default-drivers+ "Drivers implementing [[tx/gc-orphans!]]. Athena and Databricks are excluded: their datasets are preloaded, not+ created by tests."+ [:snowflake :bigquery-cloud-sdk :redshift])++(def ^:private default-report-dir+ "Where [[write-report!]] leaves its files for the workflow to upload as an artifact."+ "target/gc-report")++(defn- parse-drivers+ "Driver keywords from a comma-separated string, defaulting to [[default-drivers]]. Blanks are dropped -- an empty+ string is truthy, and would otherwise become the nameless keyword."+ [drivers]+ (or (seq (into [] (comp (map str/trim) (remove str/blank?) (map keyword))+ (str/split (or drivers "") #",")))+ default-drivers))++(defn- sweep-driver!+ "Sweep one driver and take a census of what is left, each best effort. A driver that cannot connect at all, or a+ census that fails, must cost neither the other drivers nor the deletions this one did manage."+ [driver options]+ (let [results (try+ (vec (tx/gc-orphans! driver options))+ (catch Exception e+ [{:name nil, :status :failed, :error (ex-message e)}]))+ remaining (try+ (tx/count-datasets driver)+ (catch Exception e+ (log/errorf "[%s] could not count remaining datasets: %s" (name driver) (ex-message e))+ {}))+ {deleted :deleted, failed :failed} (group-by :status results)]+ (log/infof "[%s] %d deleted, %d failed" (name driver) (count deleted) (count failed))+ (doseq [{dataset-name :name, :keys [error]} failed]+ (log/errorf "[%s] %s: %s" (name driver) (or dataset-name "<server unreachable>") error))+ {:driver (name driver)+ :deleted (vec deleted)+ :failed (vec failed)+ :counts {:deleted (count deleted), :failed (count failed)}+ :remaining remaining}))++(defn- build-report+ [options driver-reports]+ {:generated-at (str (java.time.Instant/now))+ :options options+ :totals {:deleted (reduce + (map #(get-in % [:counts :deleted]) driver-reports))+ :failed (reduce + (map #(get-in % [:counts :failed]) driver-reports))}+ :drivers (vec driver-reports)})++(defn- render-names+ "A collapsed list of object names, or a note that there were none. Collapsed because a bad night can run to+ hundreds of names and the summary still has to be skimmable."+ [heading entries render-entry]+ (if (empty? entries)+ (format "%s: none\n\n" heading)+ (format "<details><summary>%s (%d)</summary>\n\n%s\n</details>\n\n"+ heading+ (count entries)+ (str/join "\n" (map render-entry entries)))))++(defn- render-object+ "One bullet for one object. Both `:server` and `:name` are absent when the driver itself could not be reached, and+ the driver is already the heading, so an absent part is dropped rather than printed as `null`."+ [{dataset-name :name, :keys [server error]}]+ (str "- "+ (str/join " — " (cond-> []+ server (conj (format "`%s`" server))+ dataset-name (conj (format "`%s`" dataset-name))))+ (when (and error (or server dataset-name)) ": ")+ error))++(defn- render-driver [{:keys [driver counts remaining deleted failed]}]+ (str (format "### %s — %d deleted, %d failed\n\n" driver (:deleted counts) (:failed counts))+ (if (empty? remaining)+ "Remaining datasets: unknown\n\n"+ (str "Remaining datasets:\n\n"+ (str/join "\n" (for [[server n] (sort-by key remaining)]+ (format "- `%s`: %s" server (if (nil? n) "unknown" n))))+ "\n\n"))+ (render-names "Deleted" deleted render-object)+ (render-names "Failed" failed render-object)))++(defn- render-markdown [{:keys [generated-at options totals drivers]}]+ (str (format "## DWH test data sweep\n\n**%d deleted, %d failed** — temp-data-hours %s, fixture-hours %s, %s\n\n"+ (:deleted totals) (:failed totals)+ (:temp-data-hours options) (:fixture-hours options) generated-at)+ (str/join (map render-driver drivers))))++(defn- write-report!+ "Leave the report where the workflow can upload it, and append the human-readable half to the job summary so the+ morning after is legible without downloading anything."+ [report dir]+ (let [markdown (render-markdown report)]+ (.mkdirs (io/file dir))+ (spit (io/file dir "report.json") (json/encode report))+ (spit (io/file dir "report.md") markdown)+ (when-let [summary-file (System/getenv "GITHUB_STEP_SUMMARY")]+ (spit summary-file markdown :append true))+ (log/infof "Wrote sweep report to %s" dir)))++(defn gc-orphans!+ "Sweep orphaned test data from each driver's shared cloud account.++ `:drivers` is a comma-separated string, defaulting to [[default-drivers]].++ `:temp-data-hours` (default 2) is the TTL for per-run garbage, `:fixture-hours` (default 72) for datasets runs+ share. Floored at [[min-temp-data-hours]] and [[min-fixture-hours]].++ `:report-dir` (default [[default-report-dir]]) receives `report.json` and `report.md`.++ Failures don't stop the sweep. They are reported per object, written to the report, and fail the job at the end."+ [{:keys [drivers temp-data-hours fixture-hours report-dir]}]+ (let [options {:temp-data-hours (or temp-data-hours min-temp-data-hours)+ :fixture-hours (or fixture-hours 72)}]+ (doseq [[k floor] {:temp-data-hours min-temp-data-hours, :fixture-hours min-fixture-hours}+ :let [v (get options k)]]+ ;; whole hours, not merely numeric: the drivers interpolate these with `%d`, and a dispatch box lets someone+ ;; hand us 2.5+ (when-not (and (int? v) (>= v floor))+ (throw (ex-info (format "%s must be a whole number of hours >= %d; refusing to sweep with %s"+ k floor (pr-str v))+ {:option k, :value v}))))+ ;; we want the extensions, not their before-run hooks: Redshift's creates a session schema, which this job would+ ;; then leak nightly+ (binding [tx/*skip-before-run?* true]+ (let [report (build-report options (mapv #(sweep-driver! % options) (parse-drivers drivers)))]+ ;; write before throwing: the report is most wanted on the runs that fail+ (write-report! report (or report-dir default-report-dir))+ ;; fail loudly rather than going green having deleted nothing+ (when (pos? (get-in report [:totals :failed]))+ (throw (ex-info (format "%d object(s) could not be deleted"+ (get-in report [:totals :failed]))+ {:errors (into [] (comp (mapcat :failed) (map :error)) (:drivers report))})))))))diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.cljindex 24a5c0ad52b..0ecde4dd4bc 100644--- a/test/metabase/test/data/interface.clj+++ b/test/metabase/test/data/interface.clj@@ -163,9 +163,15 @@(defonce ^:private has-done-before-run (atom #{}))+(def ^:dynamic *skip-before-run?*+ "When true, loading a driver's test extensions does NOT run its [[before-run]] hook. Bound by the nightly sweep+ ([[metabase.test.data.gc]]), which needs the extensions to dispatch [[gc-orphans!]] but not the hooks: Redshift's+ creates a session schema, which the sweep would then leak nightly."+ false)+;; this gets called below by [[load-test-extensions-namespace-if-needed]](defn- do-before-run-if-needed [driver]- (when-not (@has-done-before-run driver)+ (when-not (or *skip-before-run?* (@has-done-before-run driver))(locking has-done-before-run(when-not (@has-done-before-run driver)(when (not= (get-method before-run driver) (get-method before-run ::test-extensions))@@ -387,6 +393,52 @@[driver](log/infof "%s has no after-run hooks." driver))+(defmulti gc-orphans!+ "Collect orphaned test data a previous run left behind in a shared cloud warehouse. Reports what was attempted, not+ what was found. Runs nightly (`.github/workflows/test.cleanup-dwh-data.yml`) because [[after-run]] never fires when+ a CI job is cancelled.++ `options` are `:temp-data-hours` (per-run garbage) and `:fixture-hours` (datasets runs share).++ Returns one map per object it tried to delete:++ {:server \"cluster-a.example/testdb\" ; which account, project, or cluster+database it was on+ :name \"sha__abc123_test_data\" ; nil only when the failure was reaching the server at all+ :status :deleted or :failed+ :error \"...\"} ; ex-message, present only when :failed++ A failed object must still be reported by name: the nightly report exists so someone can tell which dataset is+ stuck, and an exception alone does not carry that.++ Implement only for drivers whose tests create objects in a shared cloud account -- not Athena or Databricks, whose+ datasets are preloaded. Match only names the driver's own test extensions generate, never a bare wildcard."+ {:arglists '([driver options])}+ dispatch-on-driver-with-test-extensions+ :hierarchy #'driver/hierarchy)++(defmethod gc-orphans! ::test-extensions+ [driver _options]+ (log/infof "%s has no orphan GC; skipping." driver)+ [])++(defmulti count-datasets+ "Census of every test dataset still on each server this driver's tests write to, taken after [[gc-orphans!]] has+ run. Returns `{server-label count}`, keyed the same way as `:server` in [[gc-orphans!]]'s results.++ Counts everything on the server, including datasets too young to be eligible for collection -- the number worth+ watching is total pressure toward the account's dataset and table limits, not how much this run happened to be+ allowed to touch.++ A count that cannot be taken is `nil` rather than a missing key, so the report can say \"unknown\" instead of+ silently omitting a server."+ {:arglists '([driver])}+ dispatch-on-driver-with-test-extensions+ :hierarchy #'driver/hierarchy)++(defmethod count-datasets ::test-extensions+ [_driver]+ {})+(defmulti drop-if-exists-and-create-db!"Drop a database named `db-name` if it already exists, then create a new empty one with that name"{:added "0.55.0" :arglists '([driver db-name & [just-drop]])}