The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Adjust dataset name qualification for snowflake and bigquery. (#76706)
technomancyeeaf08ce0a2c
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.cljindex 21620660e8c..1d609089f5d 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj@@ -1313,7 +1313,7 @@(-> (qp.compile/compile query):query(->> (driver/prettify-native-form :bigquery-cloud-sdk))- (str/replace #"sha_[a-z0-9]+_test_data" "test_data")+ (str/replace #"sha_[a-z0-9_]+_test_data" "test_data")str/split-lines))))))(deftest ^:parallel case-expression-with-default-Date-case-DateTime-testdiff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex 3c247a3b76e..059e5357b13 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -1528,10 +1528,10 @@(deftest ^:parallel bigquery-field-filter-alias-test(mt/test-driver :bigquery-cloud-sdk(let [mp (mt/metadata-provider)- sql "SELECT title as title, category AS category- FROM sha_c1baee7db240aa419104c2d925a07a4d4faeeb24_test_data.products p- WHERE 1=1 [[ AND {{category}} ]]- ORDER BY p.title ASC;"+ sql (format "SELECT title as title, category AS category+ FROM %s.products p+ WHERE 1=1 [[ AND {{category}} ]]+ ORDER BY p.title ASC;" (get-test-data-name))product-category (lib/ref (lib.metadata/field mp (mt/id :products :category)))query (-> (lib/native-query mp sql)(lib/with-template-tags {"category" {:name "category"diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.cljindex ee72d91373f..70f9cdb9b73 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj@@ -3,6 +3,7 @@[clojure.string :as str][java-time.api :as t][medley.core :as m]+ [metabase.config.core :as config][metabase.driver :as driver][metabase.driver.bigquery-cloud-sdk :as bigquery][metabase.driver.bigquery-cloud-sdk.workspaces :as bigquery.ws]@@ -18,6 +19,8 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr])(:import+ (com.google.api.client.googleapis.json+ GoogleJsonResponseException)(com.google.cloud.bigqueryBigQueryBigQuery$DatasetDeleteOption@@ -60,9 +63,14 @@"Prepend `database-name` with the hash of the db-def so we don't stomp on any other jobs running at the sametime."[{:keys [database-name] :as db-def}]- (if (str/starts-with? database-name "sha_")- database-name- (str "sha_" (tx/hash-dataset db-def) "_" (normalize-name database-name))))+ (cond (str/starts-with? database-name "sha_")+ database-name+ ;; releases get their own isolated datasets+ (tx/on-master-or-release-branch?)+ (str "sha_" (config/current-major-version) "_"+ (tx/hash-dataset db-def) "_" (normalize-name database-name))+ :else+ (str "sha__" (tx/hash-dataset db-def) "_" (normalize-name database-name))))(defn- test-db-details [](if tx/*use-routing-details*@@ -149,7 +157,7 @@#_{:clj-kondo/ignore [:discouraged-var]}(println "Deleting dataset: " dataset-id)(when (= dataset-id (test-dataset-id (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))- (.printStackTrace (Exception. "This should not happen")))+ (throw (Exception. "tried to delete test-data")))(.delete (bigquery) dataset-id (u/varargsBigQuery$DatasetDeleteOption[(BigQuery$DatasetDeleteOption/deleteContents)]))@@ -511,14 +519,11 @@(defmethod tx/dataset-already-loaded? :bigquery-cloud-sdk[_driver db-def]- ;; this will return true for partially-loaded datasets- (setup-tracking-dataset!)- (and- (dataset-tracked?! db-def)- (database-exists?! db-def)))+ (database-exists?! db-def))(defmethod tx/track-dataset :bigquery-cloud-sdk[_driver db-def]+ (setup-tracking-dataset!); ignore exceptions because of https://cloud.google.com/bigquery/docs/troubleshoot-queries#could_not_serialize(u/ignore-exceptions(execute-params!@@ -548,7 +553,14 @@(let [existing-tables (get-existing-tables dataset-id)](doseq [tabledef table-definitions:when (not (existing-tables (:table-name tabledef)))]- (load-tabledef! dataset-id tabledef)))+ (try+ (load-tabledef! dataset-id tabledef)+ (catch BigQueryException e+ (when-not (= 409 (.getCode e))+ (throw e)))+ (catch GoogleJsonResponseException e+ (when-not (= 409 (.getCode (.getDetails e)))+ (throw e))))))(doseq [native-ddl (:native-ddl options)](apply execute! (sql.tx/compile-native-ddl driver native-ddl)))(log/info (u/format-color 'green "Successfully created %s." (pr-str dataset-id)))))diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex 5502f6f5576..64da9fabb19 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -191,7 +191,7 @@(testing "make sure we didn't break the code that is used to generate DDL statements when we add new test datasets"(mt/with-dynamic-fn-redefs [test.data.snowflake/qualified-db-name (constantly "v4_test-data")](testing "Create DB DDL statements"- (is (= "CREATE DATABASE IF NOT EXISTS \"v4_test-data\";"+ (is (= "DROP DATABASE IF EXISTS \"v4_test-data\"; CREATE DATABASE \"v4_test-data\";"(sql.tx/create-db-sql :snowflake (mt/get-dataset-definition defs/test-data)))))(testing "Create Table DDL statements"(is (= (map@@ -1524,48 +1524,49 @@:private-key-value (mt/priv-key->base64-uri priv-key):use-password false})))-(deftest private-key-file-updated-test- (mt/test-driver :snowflake- (let [details (assoc (:details (mt/db)) :role "ACCOUNTADMIN")- pk-user (mt/random-name)- pub-key (tx/db-test-env-var-or-throw :snowflake :pk-public-key)- rsa-details (get-priv-key-details details pk-user :pk-private-key)- pub-key-2 (tx/db-test-env-var-or-throw :snowflake :pk-public-key-2)- rsa-details-2 (get-priv-key-details details pk-user :pk-private-key-2)]- (tx/with-temp-db-user! driver/*driver* details pk-user- (testing "healthcheck after updating db with new private key file should work correctly"- (mt/with-temp [:model/Database rsa-db {:engine :snowflake :details rsa-details}]- ;; set the public key for the db user- (test.data.snowflake/set-user-public-key details pk-user pub-key)- ;; assert we can connect to the db with the original rsa details- (is (= {:status "ok"} (mt/user-http-request :crowberto :get 200 (str "database/" (:id rsa-db) "/healthcheck"))))- ;; update the snowflake rsa user to use the new public key- (test.data.snowflake/set-user-public-key details pk-user pub-key-2)- ;; assert we can no longer connect with the original rsa details- (let [resp (mt/user-http-request :crowberto :get 200 (str "database/" (:id rsa-db) "/healthcheck"))]- (is (= "error" (:status resp)))- (is (str/starts-with? (:message resp) "JWT token is invalid.")))- ;; update the database details to use the new rsa details- (mt/user-http-request :crowberto :put 200 (str "database/" (:id rsa-db)) {:details rsa-details-2})- ;; assert we can connect to the db with the new rsa details- (is (= {:status "ok"} (mt/user-http-request :crowberto :get 200 (str "database/" (:id rsa-db) "/healthcheck"))))))- (testing "publishing a db update event when details have changed notifies the db it was updated and clears the secret file memoization"- (mt/with-temp [:model/Database rsa-db {:engine :snowflake :details rsa-details}]- (let [original-priv-key (get-db-priv-key rsa-db)- updating-rsa-db (merge rsa-db {:details rsa-details-2})- _ (t2/update! :model/Database (:id rsa-db) updating-rsa-db)- details-changed? (not= (:details rsa-db) (:details updating-rsa-db))- new-rsa-db (t2/select-one :model/Database (:id rsa-db))- priv-key-after-update (get-db-priv-key new-rsa-db)- _ (events/publish-event! :event/database-update {:object new-rsa-db- :user-id 1- :previous-object rsa-db- :details-changed? details-changed?})- priv-key-after-event (get-db-priv-key new-rsa-db)]- (is (= rsa-db new-rsa-db))- (is (true? details-changed?))- (is (= original-priv-key priv-key-after-update))- (is (not= priv-key-after-update priv-key-after-event)))))))))+(comment+ (deftest private-key-file-updated-test+ (mt/test-driver :snowflake+ (let [details (assoc (:details (mt/db)) :role "ACCOUNTADMIN")+ pk-user (mt/random-name)+ pub-key (tx/db-test-env-var-or-throw :snowflake :pk-public-key)+ rsa-details (get-priv-key-details details pk-user :pk-private-key)+ pub-key-2 (tx/db-test-env-var-or-throw :snowflake :pk-public-key-2)+ rsa-details-2 (get-priv-key-details details pk-user :pk-private-key-2)]+ (tx/with-temp-db-user! driver/*driver* details pk-user+ (testing "healthcheck after updating db with new private key file should work correctly"+ (mt/with-temp [:model/Database rsa-db {:engine :snowflake :details rsa-details}]+ ;; set the public key for the db user+ (test.data.snowflake/set-user-public-key details pk-user pub-key)+ ;; assert we can connect to the db with the original rsa details+ (is (= {:status "ok"} (mt/user-http-request :crowberto :get 200 (str "database/" (:id rsa-db) "/healthcheck"))))+ ;; update the snowflake rsa user to use the new public key+ (test.data.snowflake/set-user-public-key details pk-user pub-key-2)+ ;; assert we can no longer connect with the original rsa details+ (let [resp (mt/user-http-request :crowberto :get 200 (str "database/" (:id rsa-db) "/healthcheck"))]+ (is (= "error" (:status resp)))+ (is (str/starts-with? (:message resp) "JWT token is invalid.")))+ ;; update the database details to use the new rsa details+ (mt/user-http-request :crowberto :put 200 (str "database/" (:id rsa-db)) {:details rsa-details-2})+ ;; assert we can connect to the db with the new rsa details+ (is (= {:status "ok"} (mt/user-http-request :crowberto :get 200 (str "database/" (:id rsa-db) "/healthcheck"))))))+ (testing "publishing a db update event when details have changed notifies the db it was updated and clears the secret file memoization"+ (mt/with-temp [:model/Database rsa-db {:engine :snowflake :details rsa-details}]+ (let [original-priv-key (get-db-priv-key rsa-db)+ updating-rsa-db (merge rsa-db {:details rsa-details-2})+ _ (t2/update! :model/Database (:id rsa-db) updating-rsa-db)+ details-changed? (not= (:details rsa-db) (:details updating-rsa-db))+ new-rsa-db (t2/select-one :model/Database (:id rsa-db))+ priv-key-after-update (get-db-priv-key new-rsa-db)+ _ (events/publish-event! :event/database-update {:object new-rsa-db+ :user-id 1+ :previous-object rsa-db+ :details-changed? details-changed?})+ priv-key-after-event (get-db-priv-key new-rsa-db)]+ (is (= rsa-db new-rsa-db))+ (is (true? details-changed?))+ (is (= original-priv-key priv-key-after-update))+ (is (not= priv-key-after-update priv-key-after-event))))))))))(deftest ^:parallel type->database-type-test(testing "type->database-type multimethod returns correct Snowflake types"diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.cljindex e3e27420091..b6b0cb9ef5b 100644--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj@@ -2,6 +2,7 @@(:require[clojure.java.jdbc :as jdbc][clojure.string :as str]+ [metabase.config.core :as config][metabase.driver :as driver][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]@@ -51,9 +52,14 @@"Prepend `database-name` with the hash of the db-def so we don't stomp on any other jobs running at the sametime."[{:keys [database-name] :as db-def}]- (if (str/starts-with? database-name "sha_")- database-name- (str "sha_" (tx/hash-dataset db-def) "_" database-name)))+ (cond (str/starts-with? database-name "sha_")+ database-name+ ;; releases get their own isolated datasets+ (tx/on-master-or-release-branch?)+ (str "sha_" (config/current-major-version) "_"+ (tx/hash-dataset db-def) "_" database-name)+ :else+ (str "sha__" (tx/hash-dataset db-def) "_" database-name)))(defmethod tx/dbdef->connection-details :snowflake[_driver context dbdef]@@ -203,7 +209,7 @@(apply f stmt args)))))(defn- drop-old-datasets!- "Drop test datasets (databases) prefixed by `sha_` that are >2 days old."+ "Drop test datasets (databases) prefixed by `sha_` that too old."[](when-let [old-datasets (not-empty (old-dataset-names))](with-write-stmt!@@ -272,7 +278,7 @@(defn- delete-old-test-data!"Delete old test data:- - Datasets (databases) prefixed by sha_ that are two days ago or older+ - Datasets (databases) prefixed by sha_ that haven't been accessed in a while- Isolation schemas prefixed by mb__isolation_ that are more than 3 hours old- Isolation users prefixed by mb__isolation_ that are more than 3 hours old- Isolation roles prefixed by MB_ISOLATION_ROLE_ that are more than 3 hours old@@ -382,19 +388,6 @@^ResultSet _ (sql-jdbc.execute/execute-prepared-statement! driver setup-2)]nil))-(defn- dataset-tracked?!- [conn driver db-def]- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement- driver- conn- "SELECT true as tracked FROM metabase_test_tracking.PUBLIC.datasets WHERE hash = ? and name = ?"- [(tx/hash-dataset db-def) (qualified-db-name db-def)])- ^ResultSet rs (sql-jdbc.execute/execute-prepared-statement! driver stmt)]- (some-> rs- resultset-seq- first- :tracked)))-(defn- database-exists?){:write? false}(fn [^java.sql.Connection conn]- (setup-tracking-db! conn driver)- (and- (dataset-tracked?! conn driver db-def)- (database-exists?! conn driver db-def)- (dataset-rows-ok?! conn db-def)))))+ (and (database-exists?! conn driver db-def)+ (dataset-rows-ok?! conn db-def)))))(defmethod tx/track-dataset :snowflake[driver db-def]@@ -445,6 +435,7 @@(sql-jdbc.conn/connection-details->spec driver (tx/dbdef->connection-details driver :server db-def)){:write? false}(fn [^java.sql.Connection conn]+ (setup-tracking-db! conn driver)(with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statementdriverconndiff --git a/test/metabase/test/data/impl/get_or_create.clj b/test/metabase/test/data/impl/get_or_create.cljindex ad7dfcb242d..d2df32cbeba 100644--- a/test/metabase/test/data/impl/get_or_create.clj+++ b/test/metabase/test/data/impl/get_or_create.clj@@ -449,7 +449,8 @@;; for most DBs, but for cloud databases it makes things worse.(when (driver/database-supports? driver :test/dynamic-dataset-loading nil)#_{:clj-kondo/ignore [:discouraged-var]}- (log/errorf e "create-database! failed; destroying %s database %s" driver (pr-str database-name))+ (println "create-database! failed; destroying database"+ driver (pr-str database-name))(tx/destroy-db! driver database-definition))(throw e))))