Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Avoid recreating datasets in snowflake (#75682)

technomancy38fba4025f08
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index def3142b39d..e557811b607 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -99,6 +99,8 @@
[]
(sql-jdbc.conn/connection-details->spec :snowflake (tx/dbdef->connection-details :snowflake :server nil)))
+;;; --------------------------------- Cleanup ----------------------------------
+
(defn- old-dataset-names
"Return a collection of all dataset names that are old
-- tracked that haven't been touched in a while or are not tracked and too old"
@@ -107,14 +109,10 @@
;; tracked UNION ALL untracked
;; NB. currently appears that the second half never shows anything; all
;; datasets currently appear to be tracked.
- query "(select name from metabase_test_tracking.PUBLIC.datasets
- where accessed_at < dateadd(day, ?, current_timestamp()))
- UNION All
- (select database_name from metabase_test_tracking.information_schema.databases d
- where d.database_name not in (select name from metabase_test_tracking.PUBLIC.datasets)
- and d.database_name like 'sha_%'
- and created < dateadd(day, ?, current_timestamp()))"]
- (into [] (map :name) (jdbc/reducible-query (no-db-connection-spec) [query days-ago days-ago]))))
+ query "select name from metabase_test_tracking.PUBLIC.datasets
+ where accessed_at < dateadd(day, ?, current_timestamp())"]
+ (into [] (map :name) (jdbc/reducible-query (no-db-connection-spec)
+ [query days-ago]))))
(defn- orphan-isolation-schemas
"Return a collection of schema names with mb__isolation_ prefix that are more than 3 hours old,
@@ -296,7 +294,7 @@
;; local testing shows that identifying old datasets works correctly, but
;; sometimes randomly in CI it seems to decide that datasets are old and
;; deletes them even tho they are not old.
- (drop-old-datasets!)
+ ;; (drop-old-datasets!)
(drop-orphan-isolation-schemas!)
(drop-orphan-isolation-users!)
(drop-orphan-isolation-roles!))
@@ -547,6 +545,10 @@
[_driver _feature _database]
(not (tx/on-master-or-release-branch?)))
+;; too much contention here causing unreliable tests
+(defmethod driver/database-supports? [:snowflake :test/dynamic-dataset-loading]
+ [_driver _feature _database] false)
+
(defmethod tx/fake-sync-schema :snowflake
[_driver]
"PUBLIC")
diff --git a/test/metabase/driver/common_test.clj b/test/metabase/driver/common_test.clj
index 0a5345a3f30..276011eb5a6 100644
--- a/test/metabase/driver/common_test.clj
+++ b/test/metabase/driver/common_test.clj
@@ -141,7 +141,8 @@
(driver/drop-table! driver db-id qualified-table-name))))))))
(deftest insert-from-jsonl-file-test
- (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :mongo :clickhouse}
+ ;; TODO: give this driver set a name, or use features here. what are we selecting for?
+ (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :sqlserver :mongo :clickhouse}
(mt/with-empty-db
(let [driver driver/*driver*
db-id (mt/id)
diff --git a/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj
index 30ce39fc657..51b18a57f24 100644
--- a/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj
+++ b/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj
@@ -64,39 +64,44 @@
:redshift))
(deftest fast-active-tables-test
- (is (= ["CATEGORIES" "CHECKINS" "ORDERS" "PEOPLE" "PRODUCTS" "REVIEWS" "USERS" "VENUES"]
- (sql-jdbc.execute/do-with-connection-with-options
- (or driver/*driver* :h2)
- (mt/db)
- nil
- (fn [^Connection conn]
- ;; We have to mock this to make it work with all DBs
- (mt/with-dynamic-fn-redefs [sql-jdbc.describe-database/all-schemas (constantly #{"PUBLIC"})]
- (->> (into [] (sql-jdbc.describe-database/fast-active-tables (or driver/*driver* :h2) conn nil nil))
- (map :name)
- sort)))))))
+ (is (set/subset? #{"CATEGORIES" "CHECKINS" "ORDERS" "PEOPLE" "PRODUCTS"
+ "REVIEWS" "USERS" "VENUES"}
+ (sql-jdbc.execute/do-with-connection-with-options
+ (or driver/*driver* :h2)
+ (mt/db)
+ nil
+ (fn [^Connection conn]
+ ;; We have to mock this to make it work with all DBs
+ (mt/with-dynamic-fn-redefs [sql-jdbc.describe-database/all-schemas
+ (constantly #{"PUBLIC"})]
+ (->> (into [] (sql-jdbc.describe-database/fast-active-tables
+ (or driver/*driver* :h2) conn nil nil))
+ (map :name)
+ set)))))))
(deftest post-filtered-active-tables-test
- (is (= ["CATEGORIES" "CHECKINS" "ORDERS" "PEOPLE" "PRODUCTS" "REVIEWS" "USERS" "VENUES"]
- (sql-jdbc.execute/do-with-connection-with-options
- :h2
- (mt/db)
- nil
- (fn [^Connection conn]
- (->> (into [] (sql-jdbc.describe-database/post-filtered-active-tables :h2 conn nil nil))
- (map :name)
- sort))))))
+ (is (set/subset? #{"CATEGORIES" "CHECKINS" "ORDERS" "PEOPLE" "PRODUCTS"
+ "REVIEWS" "USERS" "VENUES"}
+ (sql-jdbc.execute/do-with-connection-with-options
+ :h2
+ (mt/db)
+ nil
+ (fn [^Connection conn]
+ (->> (into [] (sql-jdbc.describe-database/post-filtered-active-tables
+ :h2 conn nil nil))
+ (map :name)
+ set))))))
(deftest describe-database-test
- (is (= {:tables #{{:name "USERS", :schema "PUBLIC", :description nil, :is_writable true}
- {:name "VENUES", :schema "PUBLIC", :description nil, :is_writable true}
- {:name "CATEGORIES", :schema "PUBLIC", :description nil, :is_writable true}
- {:name "CHECKINS", :schema "PUBLIC", :description nil, :is_writable true}
- {:name "ORDERS", :schema "PUBLIC", :description nil, :is_writable true}
- {:name "PEOPLE", :schema "PUBLIC", :description nil, :is_writable true}
- {:name "PRODUCTS", :schema "PUBLIC", :description nil, :is_writable true}
- {:name "REVIEWS", :schema "PUBLIC", :description nil, :is_writable true}}}
- (sql-jdbc.describe-database/describe-database :h2 (mt/id)))))
+ (is (set/subset? #{{:name "USERS", :schema "PUBLIC", :description nil, :is_writable true}
+ {:name "VENUES", :schema "PUBLIC", :description nil, :is_writable true}
+ {:name "CATEGORIES", :schema "PUBLIC", :description nil, :is_writable true}
+ {:name "CHECKINS", :schema "PUBLIC", :description nil, :is_writable true}
+ {:name "ORDERS", :schema "PUBLIC", :description nil, :is_writable true}
+ {:name "PEOPLE", :schema "PUBLIC", :description nil, :is_writable true}
+ {:name "PRODUCTS", :schema "PUBLIC", :description nil, :is_writable true}
+ {:name "REVIEWS", :schema "PUBLIC", :description nil, :is_writable true}}
+ (:tables (sql-jdbc.describe-database/describe-database :h2 (mt/id))))))
(defn- describe-database-with-open-resultset-count!
"Just like `describe-database`, but instead of returning the database description returns the number of ResultSet
diff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.clj
index 801c91fa4db..f75ca91cc64 100644
--- a/test/metabase/driver/sql_jdbc_test.clj
+++ b/test/metabase/driver/sql_jdbc_test.clj
@@ -1,6 +1,7 @@
(ns ^:mb/driver-tests metabase.driver.sql-jdbc-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.sql-jdbc-test]}}}}}}
(:require
+ [clojure.set :as set]
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
@@ -24,9 +25,11 @@
(set! *warn-on-reflection* true)
(deftest ^:parallel describe-database-test
- (is (= {:tables (set (for [table ["CATEGORIES" "VENUES" "CHECKINS" "USERS" "ORDERS" "PEOPLE" "PRODUCTS" "REVIEWS"]]
- {:name table, :schema "PUBLIC", :description nil, :is_writable true}))}
- (driver/describe-database :h2 (mt/db)))))
+ (is (set/subset? (set (for [table ["CATEGORIES" "VENUES" "CHECKINS" "USERS"
+ "ORDERS" "PEOPLE" "PRODUCTS" "REVIEWS"]]
+ {:name table, :schema "PUBLIC",
+ :description nil, :is_writable true}))
+ (:tables (driver/describe-database :h2 (mt/db))))))
(deftest describe-fields-sync-with-composite-pks-test
(testing "Make sure syncing a table that has a composite pks works"