The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Isolate snowflake DB names in CI aggressively (#77669)
technomancy4e5bb2daa81f
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.cljindex fe3bdf795e4..f5110e282b9 100644--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj@@ -7,7 +7,6 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.test :as mt][metabase.test.data.impl :as data.impl]- [metabase.test.data.impl.get-or-create :as test.data.impl.get-or-create][metabase.test.data.interface :as tx][metabase.test.data.sql :as sql.tx][metabase.test.data.sql-jdbc :as sql-jdbc.tx]@@ -19,9 +18,7 @@(:import(java.sql PreparedStatement ResultSet)(java.time Instant)- (java.time.temporal ChronoUnit)- (java.util.concurrent.locks ReadWriteLock Lock)- (net.snowflake.client.api.exception SnowflakeSQLException)))+ (java.time.temporal ChronoUnit)))(set! *warn-on-reflection* true)@@ -47,17 +44,18 @@:type/Time "TIME(3)"}](defmethod sql.tx/field-base-type->sql-type [:snowflake base-type] [_ _] sql-type))+;; in CI use a completely different set of databases for each run and tear down+;; all of them when the job completes; see after-run below.+(defonce dataset-prefix (str (rand-int 9999999)))+(defn qualified-db-name- "Prepend `database-name` with the hash of the db-def so we don't stomp on any other jobs running at the same- time."+ "Isolate db name so we don't stomp on any other jobs running at the same time."[{:keys [database-name] :as db-def}]- (cond (str/starts-with? database-name "sha_")- database-name- ;; releases get their own isolated datasets- (tx/on-master-or-release-branch?)- (str "sha_rel_" (tx/hash-dataset db-def) "_" database-name)- :else- (str "sha__" (tx/hash-dataset db-def) "_" database-name)))+ (cond (or (str/starts-with? database-name "isolate_")+ (str/starts-with? database-name "sha_")) database-name+ ;; isolate if we are in a CI job+ (System/getenv "GITHUB_REF_NAME") (str "isolate_" dataset-prefix database-name)+ :else (str "sha_" (tx/hash-dataset db-def) "_" database-name)))(defmethod tx/dbdef->connection-details :snowflake[_driver context dbdef]@@ -105,6 +103,15 @@;;; --------------------------------- Cleanup ----------------------------------+(defmethod tx/after-run :snowflake [_driver]+ (let [spec (no-db-connection-spec)+ query "select name from metabase_test_tracking.PUBLIC.datasets+ where name like 'isolate_%s%%'"]+ (doseq [{:keys [name]} (jdbc/query spec [(format query dataset-prefix)])]+ (jdbc/query spec+ ["DELETE FROM metabase_test_tracking.PUBLIC.datasets where name = ?" name])+ (jdbc/execute! spec [(format "DROP DATABASE \"%s\";" name)]))))+(defn- old-dataset-names"Return a collection of all dataset names that are old-- tracked that haven't been touched in a while or are not tracked and too old"@@ -342,6 +349,8 @@(let [database-name (qualified-db-name dbdef)sql (format "DROP DATABASE \"%s\";" database-name)](log/infof "[Snowflake] %s" sql)+ #_{:clj-kondo/ignore [:discouraged-var]}+ (println "[Snowflake] destroy database " database-name (:database-name dbdef))(jdbc/query (no-db-connection-spec)["DELETE FROM metabase_test_tracking.PUBLIC.datasets where name = ?" database-name])(jdbc/execute! (no-db-connection-spec) [sql])))@@ -518,6 +527,13 @@(jdbc/execute! spec (format "GRANT ROLE %s TO USER %s" "ACCOUNTADMIN" db-user))))(comment+ (let [test-data (tx/get-dataset-definition (data.impl/resolve-dataset-definition+ *ns* 'test-data))]+ (tx/dataset-already-loaded? :snowflake test-data))+ (jdbc/query (no-db-connection-spec) ["SELECT query_text, end_time+ FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY+ WHERE query_text LIKE 'DROP DATABASE %'+ ORDER BY end_time DESC limit 64"])(old-dataset-names)(drop-old-datasets!)(into [] (jdbc/reducible-query (no-db-connection-spec) ["select * from metabase_test_tracking.PUBLIC.datasets"]))@@ -641,66 +657,3 @@"TIME" :type/Time;; Default: unknown types get :type/*:type/*))--;; Sadly Snowflake does not implement locks outside very limited scope of-;; automatic locking around DDL; there are no advisory locks, so we are stuck-;; building them ourselves out of table rows.-(defn- setup-locks! []- ;; Reuse the existing tracking database, but make a new table.- (with-write-stmt! (fn [^java.sql.Statement stmt]- (.executeQuery stmt "CREATE DATABASE IF NOT EXISTS metabase_test_tracking;")))- ;; normal tables literally cannot have primary keys enforced! must be hybrid.- (with-write-stmt! (fn [^java.sql.Statement stmt]- (.executeQuery stmt "CREATE HYBRID TABLE IF NOT EXISTS metabase_test_tracking.PUBLIC.locks- (dataset TEXT PRIMARY KEY, at TIMESTAMPTZ DEFAULT current_timestamp())")))- ;; unfortuantely with-redefs in the test suite can mean that we end up trying- ;; to create locks as other users which will need access to the locks table- (with-write-stmt! (fn [^java.sql.Statement stmt]- (.executeQuery stmt "GRANT ALL ON metabase_test_tracking.PUBLIC.locks TO SYSADMIN"))))--(alter-var-root #'setup-locks! memoize)--(defn- try-lock! [^java.sql.Statement stmt dataset-name]- (try- (.executeQuery stmt (format "INSERT INTO metabase_test_tracking.PUBLIC.locks (dataset) VALUES ('%s')"- dataset-name))- true- (catch SnowflakeSQLException e- (when-not (= "A primary key already exists." (.getMessage e))- (throw e))- (with-write-stmt! (fn [^java.sql.Statement stmt]- (.executeQuery stmt "DELETE FROM metabase_test_tracking.PUBLIC.locks- WHERE TIMEDIFF('seconds', at, current_timestamp()::TIMESTAMPTZ) > 60")))- false)))--(defn- lock! [dataset-name]- (setup-locks!)- (loop [tries 0]- #_{:clj-kondo/ignore [:discouraged-var]}- (println "[Snowflake] locking attempt" tries "on" dataset-name)- (let [locked? (with-write-stmt! try-lock! dataset-name)]- (when (< 1000 tries)- (throw (Exception. "could not acquire snowflake lock")))- (when (not locked?)- (Thread/sleep 100)- (recur (inc tries))))))--(defn- unlock! [dataset-name ^java.sql.Statement stmt]- #_{:clj-kondo/ignore [:discouraged-var]}- (println "[Snowflake] unlocking" dataset-name)- (.executeQuery stmt (format "DELETE FROM metabase_test_tracking.PUBLIC.locks WHERE dataset = '%s'"- dataset-name)))--(defmethod test.data.impl.get-or-create/dataset-lock :snowflake- [_driver dataset]- (reify ReadWriteLock- (readLock [_]- (reify Lock- (lock [_])- (unlock [_])))- (writeLock [_]- (reify Lock- (lock [_]- (lock! (qualified-db-name dataset)))- (unlock [_]- (with-write-stmt! (partial unlock! (qualified-db-name dataset))))))))