Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Isolate snowflake DB names in CI aggressively (#77669)

technomancy4e5bb2daa81f
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index fe3bdf795e4..f5110e282b9 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -7,7 +7,6 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.test :as mt]
[metabase.test.data.impl :as data.impl]
- [metabase.test.data.impl.get-or-create :as test.data.impl.get-or-create]
[metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
[metabase.test.data.sql-jdbc :as sql-jdbc.tx]
@@ -19,9 +18,7 @@
(:import
(java.sql PreparedStatement ResultSet)
(java.time Instant)
- (java.time.temporal ChronoUnit)
- (java.util.concurrent.locks ReadWriteLock Lock)
- (net.snowflake.client.api.exception SnowflakeSQLException)))
+ (java.time.temporal ChronoUnit)))
(set! *warn-on-reflection* true)
@@ -47,17 +44,18 @@
:type/Time "TIME(3)"}]
(defmethod sql.tx/field-base-type->sql-type [:snowflake base-type] [_ _] sql-type))
+;; in CI use a completely different set of databases for each run and tear down
+;; all of them when the job completes; see after-run below.
+(defonce dataset-prefix (str (rand-int 9999999)))
+
(defn qualified-db-name
- "Prepend `database-name` with the hash of the db-def so we don't stomp on any other jobs running at the same
- time."
+ "Isolate db name so we don't stomp on any other jobs running at the same time."
[{:keys [database-name] :as db-def}]
- (cond (str/starts-with? database-name "sha_")
- database-name
- ;; releases get their own isolated datasets
- (tx/on-master-or-release-branch?)
- (str "sha_rel_" (tx/hash-dataset db-def) "_" database-name)
- :else
- (str "sha__" (tx/hash-dataset db-def) "_" database-name)))
+ (cond (or (str/starts-with? database-name "isolate_")
+ (str/starts-with? database-name "sha_")) database-name
+ ;; isolate if we are in a CI job
+ (System/getenv "GITHUB_REF_NAME") (str "isolate_" dataset-prefix database-name)
+ :else (str "sha_" (tx/hash-dataset db-def) "_" database-name)))
(defmethod tx/dbdef->connection-details :snowflake
[_driver context dbdef]
@@ -105,6 +103,15 @@
;;; --------------------------------- Cleanup ----------------------------------
+(defmethod tx/after-run :snowflake [_driver]
+ (let [spec (no-db-connection-spec)
+ query "select name from metabase_test_tracking.PUBLIC.datasets
+ where name like 'isolate_%s%%'"]
+ (doseq [{:keys [name]} (jdbc/query spec [(format query dataset-prefix)])]
+ (jdbc/query spec
+ ["DELETE FROM metabase_test_tracking.PUBLIC.datasets where name = ?" name])
+ (jdbc/execute! spec [(format "DROP DATABASE \"%s\";" name)]))))
+
(defn- old-dataset-names
"Return a collection of all dataset names that are old
-- tracked that haven't been touched in a while or are not tracked and too old"
@@ -342,6 +349,8 @@
(let [database-name (qualified-db-name dbdef)
sql (format "DROP DATABASE \"%s\";" database-name)]
(log/infof "[Snowflake] %s" sql)
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (println "[Snowflake] destroy database " database-name (:database-name dbdef))
(jdbc/query (no-db-connection-spec)
["DELETE FROM metabase_test_tracking.PUBLIC.datasets where name = ?" database-name])
(jdbc/execute! (no-db-connection-spec) [sql])))
@@ -518,6 +527,13 @@
(jdbc/execute! spec (format "GRANT ROLE %s TO USER %s" "ACCOUNTADMIN" db-user))))
(comment
+ (let [test-data (tx/get-dataset-definition (data.impl/resolve-dataset-definition
+ *ns* 'test-data))]
+ (tx/dataset-already-loaded? :snowflake test-data))
+ (jdbc/query (no-db-connection-spec) ["SELECT query_text, end_time
+ FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
+ WHERE query_text LIKE 'DROP DATABASE %'
+ ORDER BY end_time DESC limit 64"])
(old-dataset-names)
(drop-old-datasets!)
(into [] (jdbc/reducible-query (no-db-connection-spec) ["select * from metabase_test_tracking.PUBLIC.datasets"]))
@@ -641,66 +657,3 @@
"TIME" :type/Time
;; Default: unknown types get :type/*
:type/*))
-
-;; Sadly Snowflake does not implement locks outside very limited scope of
-;; automatic locking around DDL; there are no advisory locks, so we are stuck
-;; building them ourselves out of table rows.
-(defn- setup-locks! []
- ;; Reuse the existing tracking database, but make a new table.
- (with-write-stmt! (fn [^java.sql.Statement stmt]
- (.executeQuery stmt "CREATE DATABASE IF NOT EXISTS metabase_test_tracking;")))
- ;; normal tables literally cannot have primary keys enforced! must be hybrid.
- (with-write-stmt! (fn [^java.sql.Statement stmt]
- (.executeQuery stmt "CREATE HYBRID TABLE IF NOT EXISTS metabase_test_tracking.PUBLIC.locks
- (dataset TEXT PRIMARY KEY, at TIMESTAMPTZ DEFAULT current_timestamp())")))
- ;; unfortuantely with-redefs in the test suite can mean that we end up trying
- ;; to create locks as other users which will need access to the locks table
- (with-write-stmt! (fn [^java.sql.Statement stmt]
- (.executeQuery stmt "GRANT ALL ON metabase_test_tracking.PUBLIC.locks TO SYSADMIN"))))
-
-(alter-var-root #'setup-locks! memoize)
-
-(defn- try-lock! [^java.sql.Statement stmt dataset-name]
- (try
- (.executeQuery stmt (format "INSERT INTO metabase_test_tracking.PUBLIC.locks (dataset) VALUES ('%s')"
- dataset-name))
- true
- (catch SnowflakeSQLException e
- (when-not (= "A primary key already exists." (.getMessage e))
- (throw e))
- (with-write-stmt! (fn [^java.sql.Statement stmt]
- (.executeQuery stmt "DELETE FROM metabase_test_tracking.PUBLIC.locks
- WHERE TIMEDIFF('seconds', at, current_timestamp()::TIMESTAMPTZ) > 60")))
- false)))
-
-(defn- lock! [dataset-name]
- (setup-locks!)
- (loop [tries 0]
- #_{:clj-kondo/ignore [:discouraged-var]}
- (println "[Snowflake] locking attempt" tries "on" dataset-name)
- (let [locked? (with-write-stmt! try-lock! dataset-name)]
- (when (< 1000 tries)
- (throw (Exception. "could not acquire snowflake lock")))
- (when (not locked?)
- (Thread/sleep 100)
- (recur (inc tries))))))
-
-(defn- unlock! [dataset-name ^java.sql.Statement stmt]
- #_{:clj-kondo/ignore [:discouraged-var]}
- (println "[Snowflake] unlocking" dataset-name)
- (.executeQuery stmt (format "DELETE FROM metabase_test_tracking.PUBLIC.locks WHERE dataset = '%s'"
- dataset-name)))
-
-(defmethod test.data.impl.get-or-create/dataset-lock :snowflake
- [_driver dataset]
- (reify ReadWriteLock
- (readLock [_]
- (reify Lock
- (lock [_])
- (unlock [_])))
- (writeLock [_]
- (reify Lock
- (lock [_]
- (lock! (qualified-db-name dataset)))
- (unlock [_]
- (with-write-stmt! (partial unlock! (qualified-db-name dataset))))))))