Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Stop dropping old datasets from snowflake. (#74663)

technomancy9a0bb9110944
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 1d6962972db..d9161a802e7 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -1626,7 +1626,9 @@ jobs:
# if you add a driver job, you must add it to this list for it to be a required check
- be-tests-h2
- be-tests-athena-ee
- - be-tests-bigquery-cloud-sdk-ee
+ # these are not reliable, but we want to run them anyway to gather data about failures
+ # - be-tests-snowflake-ee
+ # - be-tests-bigquery-cloud-sdk-ee
- be-tests-druid-ee
- be-tests-databricks-ee
- be-tests-druid-jdbc-ee
@@ -1638,7 +1640,6 @@ jobs:
- be-tests-postgres
- be-tests-presto-jdbc-ee
- be-tests-redshift-ee
- - be-tests-snowflake-ee
- be-tests-sparksql-ee
- be-tests-sqlite-ee
- be-tests-sqlserver
@@ -1666,8 +1667,7 @@ jobs:
}));
// are all jobs skipped or successful?
- if (jobs.every(job => (job.result === 'skipped' || job.result === 'success' ||
- job.name == 'be-tests-starburst-ee' || job.name == 'be-tests-bigquery-cloud-sdk-ee'))) {
+ if (jobs.every(job => (job.result === 'skipped' || job.result === 'success'))) {
console.log("");
console.log(" _------. ");
console.log(" / , \_ ");
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 54674dc6a23..74001a3a12c 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -102,6 +102,8 @@
[]
(let [days-ago -5
;; tracked UNION ALL untracked
+ ;; NB. currently appears that the second half never shows anything; all
+ ;; datasets currently appear to be tracked.
query "(select name from metabase_test_tracking.PUBLIC.datasets
where accessed_at < dateadd(day, ?, current_timestamp()))
UNION All
@@ -212,9 +214,9 @@
(.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))
(.execute stmt (format "delete from metabase_test_tracking.PUBLIC.datasets where name = '%s';"
dataset-name))
- ;; if this fails for some reason it's probably just because some other job tried to delete the dataset at the
- ;; same time. No big deal. Just log this and carry on trying to delete the other datasets. If we don't end up
- ;; deleting anything it's not the end of the world because it won't affect our ability to run our tests
+ ;; if this fails for some reason it's probably just because some other job tried to delete the dataset at the
+ ;; same time. No big deal. Just log this and carry on trying to delete the other datasets. If we don't end up
+ ;; deleting anything it's not the end of the world because it won't affect our ability to run our tests
(catch Throwable e
#_{:clj-kondo/ignore [:discouraged-var]}
(println "[Snowflake] Error deleting old dataset:" (ex-message e)))))))))
@@ -283,7 +285,15 @@
;; intended -- Cam
#_{:clj-kondo/ignore [:discouraged-var]}
(println "[Snowflake] deleting old test data...")
- (drop-old-datasets!)
+ ;; disabling this temporarily as it has caused very difficult-to-debug failures
+ ;; in CI. even tho the datasets *have* been accessed recently, they are still
+ ;; being deleted and reinserted, with race conditions that cause some data to be
+ ;; inserted three times. this does mean that if datasets change, old versions
+ ;; will not be cleaned up automatically and will need to be manually GCed.
+ ;; local testing shows that identifying old datasets works correctly, but
+ ;; sometimes randomly in CI it seems to decide that datasets are old and
+ ;; deletes them even tho they are not old.
+ ;; (drop-old-datasets!)
(drop-orphan-isolation-schemas!)
(drop-orphan-isolation-users!)
(drop-orphan-isolation-roles!))
@@ -497,6 +507,7 @@
(comment
(old-dataset-names)
+ (drop-old-datasets!)
(into [] (jdbc/reducible-query (no-db-connection-spec) ["select * from metabase_test_tracking.PUBLIC.datasets"]))
;; Tracked databases ordered by age
(->> ["select d.name, d.accessed_at, i.created, timestampdiff('minute', i.created, d.accessed_at) as diff, timestampdiff('minute', i.created, current_timestamp()) as age