The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Stop dropping old datasets from snowflake. (#74663)
technomancy9a0bb9110944
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 1d6962972db..d9161a802e7 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -1626,7 +1626,9 @@ jobs:# if you add a driver job, you must add it to this list for it to be a required check- be-tests-h2- be-tests-athena-ee- - be-tests-bigquery-cloud-sdk-ee+ # these are not reliable, but we want to run them anyway to gather data about failures+ # - be-tests-snowflake-ee+ # - be-tests-bigquery-cloud-sdk-ee- be-tests-druid-ee- be-tests-databricks-ee- be-tests-druid-jdbc-ee@@ -1638,7 +1640,6 @@ jobs:- be-tests-postgres- be-tests-presto-jdbc-ee- be-tests-redshift-ee- - be-tests-snowflake-ee- be-tests-sparksql-ee- be-tests-sqlite-ee- be-tests-sqlserver@@ -1666,8 +1667,7 @@ jobs:}));// are all jobs skipped or successful?- if (jobs.every(job => (job.result === 'skipped' || job.result === 'success' ||- job.name == 'be-tests-starburst-ee' || job.name == 'be-tests-bigquery-cloud-sdk-ee'))) {+ if (jobs.every(job => (job.result === 'skipped' || job.result === 'success'))) {console.log("");console.log(" _------. ");console.log(" / , \_ ");diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.cljindex 54674dc6a23..74001a3a12c 100644--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj@@ -102,6 +102,8 @@[](let [days-ago -5;; tracked UNION ALL untracked+ ;; NB. currently appears that the second half never shows anything; all+ ;; datasets currently appear to be tracked.query "(select name from metabase_test_tracking.PUBLIC.datasetswhere accessed_at < dateadd(day, ?, current_timestamp()))UNION All@@ -212,9 +214,9 @@(.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))(.execute stmt (format "delete from metabase_test_tracking.PUBLIC.datasets where name = '%s';"dataset-name))- ;; if this fails for some reason it's probably just because some other job tried to delete the dataset at the- ;; same time. No big deal. Just log this and carry on trying to delete the other datasets. If we don't end up- ;; deleting anything it's not the end of the world because it won't affect our ability to run our tests+ ;; if this fails for some reason it's probably just because some other job tried to delete the dataset at the+ ;; same time. No big deal. Just log this and carry on trying to delete the other datasets. If we don't end up+ ;; deleting anything it's not the end of the world because it won't affect our ability to run our tests(catch Throwable e#_{:clj-kondo/ignore [:discouraged-var]}(println "[Snowflake] Error deleting old dataset:" (ex-message e)))))))))@@ -283,7 +285,15 @@;; intended -- Cam#_{:clj-kondo/ignore [:discouraged-var]}(println "[Snowflake] deleting old test data...")- (drop-old-datasets!)+ ;; disabling this temporarily as it has caused very difficult-to-debug failures+ ;; in CI. even tho the datasets *have* been accessed recently, they are still+ ;; being deleted and reinserted, with race conditions that cause some data to be+ ;; inserted three times. this does mean that if datasets change, old versions+ ;; will not be cleaned up automatically and will need to be manually GCed.+ ;; local testing shows that identifying old datasets works correctly, but+ ;; sometimes randomly in CI it seems to decide that datasets are old and+ ;; deletes them even tho they are not old.+ ;; (drop-old-datasets!)(drop-orphan-isolation-schemas!)(drop-orphan-isolation-users!)(drop-orphan-isolation-roles!))@@ -497,6 +507,7 @@(comment(old-dataset-names)+ (drop-old-datasets!)(into [] (jdbc/reducible-query (no-db-connection-spec) ["select * from metabase_test_tracking.PUBLIC.datasets"]));; Tracked databases ordered by age(->> ["select d.name, d.accessed_at, i.created, timestampdiff('minute', i.created, d.accessed_at) as diff, timestampdiff('minute', i.created, current_timestamp()) as age