Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add locks around Snowflake dataset creation (#74876)

technomancyaa55ae49d342
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index 69262eff91d..50b8bf87dfd 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -1360,6 +1360,10 @@
:snowflake
(let [original-set-current-user-timezone! @#'test.data.snowflake/set-current-user-timezone!
original-dbdef->connection-details (get-method tx/dbdef->connection-details :snowflake)]
+ ;; load dataset as the default user before we switch to the belize-based
+ ;; user who may not have permission to create the dataset
+ (mt/dataset good-datetimes-in-belize
+ (mt/id :GOOD_DATETIMES))
(with-redefs [test.data.snowflake/set-current-user-timezone!
(fn [_timezone]
(original-set-current-user-timezone! "America/Belize"))
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 74001a3a12c..def3142b39d 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -7,6 +7,7 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.test :as mt]
[metabase.test.data.impl :as data.impl]
+ [metabase.test.data.impl.get-or-create :as test.data.impl.get-or-create]
[metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
[metabase.test.data.sql-jdbc :as sql-jdbc.tx]
@@ -18,7 +19,9 @@
(:import
(java.sql PreparedStatement ResultSet)
(java.time Instant)
- (java.time.temporal ChronoUnit)))
+ (java.time.temporal ChronoUnit)
+ (java.util.concurrent.locks ReadWriteLock Lock)
+ (net.snowflake.client.api.exception SnowflakeSQLException)))
(set! *warn-on-reflection* true)
@@ -192,14 +195,14 @@
"Open a write-capable Snowflake connection + Statement, call `f` with the stmt,
close everything. Centralizes the boilerplate so the per-resource drop fns
don't repeat it."
- [f]
+ [f & args]
(sql-jdbc.execute/do-with-connection-with-options
:snowflake
(no-db-connection-spec)
{:write? true}
(fn [^java.sql.Connection conn]
(with-open [stmt (.createStatement conn)]
- (f stmt)))))
+ (apply f stmt args)))))
(defn- drop-old-datasets!
"Drop test datasets (databases) prefixed by `sha_` that are >2 days old."
@@ -293,7 +296,7 @@
;; local testing shows that identifying old datasets works correctly, but
;; sometimes randomly in CI it seems to decide that datasets are old and
;; deletes them even tho they are not old.
- ;; (drop-old-datasets!)
+ (drop-old-datasets!)
(drop-orphan-isolation-schemas!)
(drop-orphan-isolation-users!)
(drop-orphan-isolation-roles!))
@@ -625,3 +628,66 @@
"TIME" :type/Time
;; Default: unknown types get :type/*
:type/*))
+
+;; Sadly Snowflake does not implement locks outside very limited scope of
+;; automatic locking around DDL; there are no advisory locks, so we are stuck
+;; building them ourselves out of table rows.
+(defn- setup-locks! []
+ ;; Reuse the existing tracking database, but make a new table.
+ (with-write-stmt! (fn [^java.sql.Statement stmt]
+ (.executeQuery stmt "CREATE DATABASE IF NOT EXISTS metabase_test_tracking;")))
+ ;; normal tables literally cannot have primary keys enforced! must be hybrid.
+ (with-write-stmt! (fn [^java.sql.Statement stmt]
+ (.executeQuery stmt "CREATE HYBRID TABLE IF NOT EXISTS metabase_test_tracking.PUBLIC.locks
+ (dataset TEXT PRIMARY KEY, at TIMESTAMPTZ DEFAULT current_timestamp())")))
+ ;; unfortuantely with-redefs in the test suite can mean that we end up trying
+ ;; to create locks as other users which will need access to the locks table
+ (with-write-stmt! (fn [^java.sql.Statement stmt]
+ (.executeQuery stmt "GRANT ALL ON metabase_test_tracking.PUBLIC.locks TO SYSADMIN"))))
+
+(alter-var-root #'setup-locks! memoize)
+
+(defn- try-lock! [^java.sql.Statement stmt dataset-name]
+ (try
+ (.executeQuery stmt (format "INSERT INTO metabase_test_tracking.PUBLIC.locks (dataset) VALUES ('%s')"
+ dataset-name))
+ true
+ (catch SnowflakeSQLException e
+ (when-not (= "A primary key already exists." (.getMessage e))
+ (throw e))
+ (with-write-stmt! (fn [^java.sql.Statement stmt]
+ (.executeQuery stmt "DELETE FROM metabase_test_tracking.PUBLIC.locks
+ WHERE TIMEDIFF('seconds', at, current_timestamp()::TIMESTAMPTZ) > 60")))
+ false)))
+
+(defn- lock! [dataset-name]
+ (setup-locks!)
+ (loop [tries 0]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (println "[Snowflake] locking attempt" tries "on" dataset-name)
+ (let [locked? (with-write-stmt! try-lock! dataset-name)]
+ (when (< 1000 tries)
+ (throw (Exception. "could not acquire snowflake lock")))
+ (when (not locked?)
+ (Thread/sleep 100)
+ (recur (inc tries))))))
+
+(defn- unlock! [dataset-name ^java.sql.Statement stmt]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (println "[Snowflake] unlocking" dataset-name)
+ (.executeQuery stmt (format "DELETE FROM metabase_test_tracking.PUBLIC.locks WHERE dataset = '%s'"
+ dataset-name)))
+
+(defmethod test.data.impl.get-or-create/dataset-lock :snowflake
+ [_driver dataset-name]
+ (reify ReadWriteLock
+ (readLock [_]
+ (reify Lock
+ (lock [_])
+ (unlock [_])))
+ (writeLock [_]
+ (reify Lock
+ (lock [_]
+ (lock! dataset-name))
+ (unlock [_]
+ (with-write-stmt! (partial unlock! dataset-name)))))))
diff --git a/test/metabase/actions/test_util.clj b/test/metabase/actions/test_util.clj
index f142f8a6b05..96d38376166 100644
--- a/test/metabase/actions/test_util.clj
+++ b/test/metabase/actions/test_util.clj
@@ -125,7 +125,9 @@
...)"
{:style/indent :defn}
[table-definitions & body]
- `(do-with-dataset-definition (tx/dataset-definition "temp-test-data" ~table-definitions) (fn [] ~@body)))
+ `(do-with-dataset-definition (tx/dataset-definition (str "temp-test-data" (u.random/random-name))
+ ~table-definitions)
+ (fn [] ~@body)))
(defmacro with-empty-db
"Sets the current dataset to a freshly created db that gets destroyed at the conclusion of `body`.
diff --git a/test/metabase/sync/sync_metadata/indexes_test.clj b/test/metabase/sync/sync_metadata/indexes_test.clj
index dc666a84b83..86c5c9f918e 100644
--- a/test/metabase/sync/sync_metadata/indexes_test.clj
+++ b/test/metabase/sync/sync_metadata/indexes_test.clj
@@ -8,9 +8,7 @@
[metabase.sync.core :as sync]
[metabase.sync.sync-metadata.indexes :as sync.indexes]
[metabase.test :as mt]
- [metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
- [metabase.util :as u]
[toucan2.core :as t2]))
(deftest sync-single-indexed-columns-test
@@ -30,19 +28,12 @@
[{:field-name "first" :indexed? false :base-type :type/Integer}
{:field-name "second" :indexed? false :base-type :type/Integer}]
[[1 2]]]]]
- (mt/with-temp-test-data
- ds
- (try
- (jdbc/execute! (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
- (sql.tx/create-index-sql driver/*driver* "table" ["first" "second"]))
- (sync/sync-database! (mt/db))
- (is (true? (t2/select-one-fn :database_indexed :model/Field (mt/id :table :first))))
- (is (not= true (t2/select-one-fn :database_indexed :model/Field (mt/id :table :second))))
- (finally
- ;; clean the db so this test is repeatable
- (t2/delete! :model/Database (mt/id))
- (u/ignore-exceptions
- (tx/destroy-db! driver/*driver* ds))))))))
+ (mt/with-temp-test-data ds
+ (jdbc/execute! (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+ (sql.tx/create-index-sql driver/*driver* "table" ["first" "second"]))
+ (sync/sync-database! (mt/db))
+ (is (true? (t2/select-one-fn :database_indexed :model/Field (mt/id :table :first))))
+ (is (not= true (t2/select-one-fn :database_indexed :model/Field (mt/id :table :second))))))))
(driver/register! ::not-support-index-test :abstract? true)
diff --git a/test/metabase/test/data/impl/get_or_create.clj b/test/metabase/test/data/impl/get_or_create.clj
index b69845fcd69..00e594ef701 100644
--- a/test/metabase/test/data/impl/get_or_create.clj
+++ b/test/metabase/test/data/impl/get_or_create.clj
@@ -36,7 +36,7 @@
multiple times in parallel -- for example my Oracle test that runs 30 sync calls at the same time to make sure
nothing explodes and cursors aren't leaked. To make sure this doesn't happen we'll keep a map of
- [driver dataset-name] -> ReentrantReadWriteLock
+ [driver dataset-name] -> ReadWriteLock
and make sure data can be loaded and synced for a given driver + dataset in a synchronized fashion. Code path looks
like this:
@@ -57,7 +57,7 @@
Because each driver and dataset has its own lock, various datasets can be loaded in parallel, but this will prevent
the same dataset from being loaded multiple times."
- {:arglists '(^java.util.concurrent.locks.ReentrantReadWriteLock [driver dataset-name])}
+ {:arglists '(^java.util.concurrent.locks.ReadWriteLock [driver dataset-name])}
tx/dispatch-on-driver-with-test-extensions
:hierarchy #'driver/hierarchy)