The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Enable fake-sync for Snowflake driver tests (#68322)
bryan0982712225e4
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex db5b888fb53..f770123d384 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -36,6 +36,7 @@[metabase.test :as mt][metabase.test.data.dataset-definitions :as defs][metabase.test.data.impl :as data.impl]+ [metabase.test.data.impl.get-or-create :as test.get-or-create][metabase.test.data.interface :as tx][metabase.test.data.snowflake :as test.data.snowflake][metabase.test.data.sql :as sql.tx]@@ -294,29 +295,31 @@:additional-options "foo=bar"))))))))(deftest describe-database-test+ ;; This test calls driver/describe-database which queries Snowflake directly.+ ;; Requires real sync (not fake-sync) so tables actually exist in Snowflake.(mt/test-driver :snowflake- (testing "describe-database"- (let [expected {:tables- #{{:name "users", :schema "PUBLIC", :description nil}- {:name "venues", :schema "PUBLIC", :description nil}- {:name "checkins", :schema "PUBLIC", :description nil}- {:name "categories", :schema "PUBLIC", :description nil}- {:name "orders", :schema "PUBLIC", :description nil}- {:name "people", :schema "PUBLIC", :description nil}- {:name "products", :schema "PUBLIC", :description nil}- {:name "reviews", :schema "PUBLIC", :description nil}}}]- (testing "should work with normal details"- (is (= expected- (driver/describe-database :snowflake (mt/db)))))- (testing "should accept either `:db` or `:dbname` in the details, working around a bug with the original impl"- (is (= expected- (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :dbname})))))- (testing "should throw an Exception if details have neither `:db` nor `:dbname`"- (is (thrown? Exception- (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :xyz})))))- (testing "should use the NAME FROM DETAILS instead of the DB DISPLAY NAME to fetch metadata (#8864)"- (is (= expected- (driver/describe-database :snowflake (assoc (mt/db) :name "ABC")))))))))+ (tx/with-driver-supports-feature! [:snowflake :test/use-fake-sync false]+ (testing "describe-database"+ (let [expected-tables #{{:name "users", :schema "PUBLIC", :description nil}+ {:name "venues", :schema "PUBLIC", :description nil}+ {:name "checkins", :schema "PUBLIC", :description nil}+ {:name "categories", :schema "PUBLIC", :description nil}+ {:name "orders", :schema "PUBLIC", :description nil}+ {:name "people", :schema "PUBLIC", :description nil}+ {:name "products", :schema "PUBLIC", :description nil}+ {:name "reviews", :schema "PUBLIC", :description nil}}]+ (testing "should work with normal details"+ (is (= expected-tables+ (:tables (driver/describe-database :snowflake (mt/db))))))+ (testing "should accept either `:db` or `:dbname` in the details, working around a bug with the original impl"+ (is (= expected-tables+ (:tables (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :dbname}))))))+ (testing "should throw an Exception if details have neither `:db` nor `:dbname`"+ (is (thrown? Exception+ (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :xyz})))))+ (testing "should use the NAME FROM DETAILS instead of the DB DISPLAY NAME to fetch metadata (#8864)"+ (is (= expected-tables+ (:tables (driver/describe-database :snowflake (assoc (mt/db) :name "ABC")))))))))))(deftest describe-database-default-schema-test(testing "describe-database should include Tables from all schemas even if the DB has a default schema (#38135)"@@ -1423,23 +1426,44 @@(is (= exp-rows (mt/rows result))))))))(deftest snowflake-with-dbname-in-details-gets-synced-test+ ;; This test calls driver/describe-database which queries Snowflake directly.+ ;; Requires real sync (not fake-sync) so tables actually exist in Snowflake.(testing "db with a valid db and an invalid dbname in details should be synced with db correctly"(mt/test-driver :snowflake- (let [priv-key-val (mt/priv-key->base64-uri (tx/db-test-env-var-or-throw :snowflake :private-key))]- (mt/with-temp [:model/Database db {:engine :snowflake- :details (-> (:details (mt/db))- (dissoc :private-key-id)- (assoc :private-key-options "uploaded")- (assoc :private-key-value priv-key-val)- (assoc :use-password false)- (assoc :dbname nil))}]- (is (= {:tables- #{{:name "users", :schema "PUBLIC", :description nil}- {:name "venues", :schema "PUBLIC", :description nil}- {:name "checkins", :schema "PUBLIC", :description nil}- {:name "categories", :schema "PUBLIC", :description nil}- {:name "orders", :schema "PUBLIC", :description nil}- {:name "people", :schema "PUBLIC", :description nil}- {:name "products", :schema "PUBLIC", :description nil}- {:name "reviews", :schema "PUBLIC", :description nil}}}- (driver/describe-database :snowflake db))))))))+ (tx/with-driver-supports-feature! [:snowflake :test/use-fake-sync false]+ (let [priv-key-val (mt/priv-key->base64-uri (tx/db-test-env-var-or-throw :snowflake :private-key))+ expected-tables #{{:name "users", :schema "PUBLIC", :description nil}+ {:name "venues", :schema "PUBLIC", :description nil}+ {:name "checkins", :schema "PUBLIC", :description nil}+ {:name "categories", :schema "PUBLIC", :description nil}+ {:name "orders", :schema "PUBLIC", :description nil}+ {:name "people", :schema "PUBLIC", :description nil}+ {:name "products", :schema "PUBLIC", :description nil}+ {:name "reviews", :schema "PUBLIC", :description nil}}]+ (mt/with-temp [:model/Database db {:engine :snowflake+ :details (-> (:details (mt/db))+ (dissoc :private-key-id)+ (assoc :private-key-options "uploaded")+ (assoc :private-key-value priv-key-val)+ (assoc :use-password false)+ (assoc :dbname nil))}]+ (is (= expected-tables+ (:tables (driver/describe-database :snowflake db))))))))))++;;; ------------------------------------------------ Fake Sync Tests ------------------------------------------------+;; Tests to validate that fake sync produces correct metadata for Snowflake.+;; The key difference from Redshift: Snowflake uses plain table names ("venues") because each+;; dataset has its own database, while Redshift uses qualified names ("test_data_venues").++(deftest ^:parallel fake-sync-transformation-test+ (mt/test-driver :snowflake+ (testing "Snowflake fake-sync uses plain table names (not qualified)"+ ;; Snowflake has separate databases per dataset, so tables are just "users"+ ;; not "transform_test_users" like Redshift which shares a single database.+ (let [dbdef {:database-name "transform-test"+ :table-definitions [{:table-name "users"+ :field-definitions [{:field-name "name" :base-type :type/Text}]}]}+ rows (@#'test.get-or-create/dbdef->fake-sync-rows :snowflake 123 dbdef)+ table (:table-row (first rows))]+ (is (= "users" (:name table)) "Should be plain name, not 'transform_test_users'")+ (is (= "PUBLIC" (:schema table)))))))diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.cljindex 5c482b59305..232b0f488dc 100644--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj@@ -32,6 +32,8 @@:type/Decimal "DECIMAL":type/Float "FLOAT":type/Integer "INTEGER"+ ;; :type/Number is used by tx/id-field-type for Snowflake PKs+ :type/Number "NUMBER":type/Text "TEXT";; 3 = millisecond precision. Default is allegedly 9 (nanosecond precision) according to;; https://docs.snowflake.com/en/sql-reference/data-types-datetime#time, but it seems like@@ -387,3 +389,91 @@(into [] (map (juxt :database_name :created)))))(defmethod sql.tx/session-schema :snowflake [_driver] "PUBLIC")++;;; ------------------------------------------------ Fake Sync Support ------------------------------------------------++;; Enable fake sync for Snowflake on feature branches.+;; Fake sync skips network calls to the database for metadata sync, which saves significant CI time.+;; On master/release branches, use real sync to catch any sync regressions.+(defmethod driver/database-supports? [:snowflake :test/use-fake-sync]+ [_driver _feature _database]+ (not (tx/on-master-or-release-branch?)))++(defmethod tx/fake-sync-schema :snowflake+ [_driver]+ "PUBLIC")++(defmethod tx/fake-sync-table-name :snowflake+ [_driver _database-name table-name]+ ;; Snowflake uses separate databases per dataset, so table names are NOT prefixed+ ;; with the database name. Unlike Redshift (which uses test_data_venues), Snowflake+ ;; tables are just "venues" within the sha_xxx_test_data database.+ table-name)++(defmethod tx/fake-sync-database-type :snowflake+ [_driver base-type]+ ;; Return the database_type as Snowflake's query processor expects it.+ ;; The QP uses lowercase types without precision (e.g., "time" not "TIME(3)").+ ;; Snowflake normalizes types: TEXT->VARCHAR, FLOAT->DOUBLE, INTEGER->NUMBER+ ;;+ ;; For timezone columns: DDL uses TIMESTAMP_TZ, but DB reports as TIMESTAMPTZ+ (case base-type+ :type/Text "VARCHAR"+ :type/Float "DOUBLE"+ :type/Integer "NUMBER"+ :type/BigInteger "NUMBER"+ :type/Number "NUMBER"+ :type/Boolean "BOOLEAN"+ :type/Date "date"+ :type/DateTime "timestampntz"+ :type/DateTimeWithTZ "timestamptz" ; DDL: TIMESTAMP_TZ, reported as: TIMESTAMPTZ+ :type/DateTimeWithLocalTZ "timestamptz"+ :type/DateTimeWithZoneID "timestamptz"+ :type/DateTimeWithZoneOffset "timestamptz"+ :type/Time "time"+ :type/TimeWithLocalTZ "time"+ :type/TimeWithZoneOffset "time"+ ;; For other types, use the creation type+ (sql.tx/field-base-type->sql-type :snowflake base-type)))++(defmethod tx/fake-sync-base-type :snowflake+ [_driver base-type]+ ;; Snowflake normalizes some types. Real sync maps them to specific base_types,+ ;; so fake-sync must match what sync would produce:+ ;; - INTEGER/BIGINT -> NUMBER -> :type/Number+ ;; - TimeWithLocalTZ/TimeWithZoneOffset -> TIME -> :type/Time (Snowflake only has one TIME type)+ ;; - DateTimeWithTZ/DateTimeWithZoneID/DateTimeWithZoneOffset -> TIMESTAMP_TZ -> :type/DateTimeWithLocalTZ+ ;; (Note: :type/DateTimeWithTZ -> TIMESTAMP_TZ -> sync as TIMESTAMPTZ -> :type/DateTimeWithLocalTZ)+ (case base-type+ :type/Integer :type/Number+ :type/BigInteger :type/Number+ :type/TimeWithLocalTZ :type/Time+ :type/TimeWithZoneOffset :type/Time+ :type/DateTimeWithTZ :type/DateTimeWithLocalTZ+ :type/DateTimeWithZoneID :type/DateTimeWithLocalTZ+ :type/DateTimeWithZoneOffset :type/DateTimeWithLocalTZ+ ;; Other types are unchanged+ base-type))++(defmethod tx/fake-sync-native-base-type :snowflake+ [_driver native-type]+ ;; Map native Snowflake type strings to their base_type.+ ;; These must match what sql-jdbc.sync/database-type->base-type returns for Snowflake.+ ;; See metabase.driver.snowflake for the full mapping.+ (case (some-> native-type u/upper-case-en)+ ;; Timestamp types+ "TIMESTAMPTZ" :type/DateTimeWithLocalTZ+ "TIMESTAMPLTZ" :type/DateTimeWithTZ+ "TIMESTAMPNTZ" :type/DateTime+ "TIMESTAMP" :type/DateTime+ ;; Other common types+ "VARCHAR" :type/Text+ "TEXT" :type/Text+ "NUMBER" :type/Number+ "FLOAT" :type/Float+ "DOUBLE" :type/Float+ "BOOLEAN" :type/Boolean+ "DATE" :type/Date+ "TIME" :type/Time+ ;; Default: unknown types get :type/*+ :type/*))diff --git a/test/metabase/test/data/impl/get_or_create.clj b/test/metabase/test/data/impl/get_or_create.cljindex 636bd8b566a..a97d40cf717 100644--- a/test/metabase/test/data/impl/get_or_create.clj+++ b/test/metabase/test/data/impl/get_or_create.clj@@ -132,7 +132,8 @@;; ---------------------- Pure Transformation Functions ----------------------(defn- field-def->row- "Convert a FieldDefinition to a Field row map (without table_id, added during insertion).+ "Convert a FieldDefinition to a Field row map for fake-sync insertion.+ Creates the metadata row that would normally come from syncing the database.Handles three forms of base-type:1. {:native \"BINARY(8)\"} - driver-specific native type string2. {:natives {:postgres \"BYTEA\" :redshift \"VARBYTE\"}} - per-driver native types@@ -147,13 +148,18 @@(get-in base-type [:natives driver]):else- ;; Use the SQL type from sql.tx/field-base-type->sql-type to get- ;; the actual database type (e.g., "TIMESTAMPTZ" instead of "DateTimeWithTZ").- ;; Use requiring-resolve to avoid cyclic dependency.- ((requiring-resolve 'metabase.test.data.sql/field-base-type->sql-type) driver base-type))+ ;; Use fake-sync-database-type to get the type the database reports+ ;; (which may differ from the DDL type used to create columns).+ ;; E.g., Snowflake: TEXT->VARCHAR, FLOAT->DOUBLE, INTEGER->NUMBER+ (tx/fake-sync-database-type driver base-type))actual-base-type (if (map? base-type)- (or effective-type :type/*)- base-type)+ ;; For native types, use effective-type if provided,+ ;; otherwise ask the driver what base_type sync would produce+ (or effective-type+ (tx/fake-sync-native-base-type driver database-type))+ ;; Use fake-sync-base-type to get what the database actually reports+ ;; E.g., Snowflake: :type/Integer -> :type/Number (since INTEGER->NUMBER)+ (tx/fake-sync-base-type driver base-type))semantic-type (condpk? :type/PK(some? fk) :type/FK@@ -175,11 +181,11 @@"Convert a TableDefinition to a map of {:table-name, :table-row, :field-rows}.This is a pure function - no database insertion."[driver db-id database-name {:keys [table-name table-comment field-definitions] :as _table-def}]- (let [schema (tx/fake-sync-schema driver)- qualified-name (tx/db-qualified-table-name database-name table-name)+ (let [schema (tx/fake-sync-schema driver)+ sync-table-name (tx/fake-sync-table-name driver database-name table-name)has-custom-pk? (some :pk? field-definitions)table-row {:db_id db-id- :name qualified-name+ :name sync-table-name:schema schema:display_name (humanization/name->human-readable-name :simple table-name):description table-commentdiff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.cljindex cd945cf3278..bb79d45d055 100644--- a/test/metabase/test/data/interface.clj+++ b/test/metabase/test/data/interface.clj@@ -528,6 +528,64 @@[_driver]nil)+(defmulti fake-sync-table-name+ "Return the table name to use for fake sync Table rows.+ Default uses `db-qualified-table-name` (e.g., 'test_data_venues') for drivers+ that share a single database across datasets (like Redshift, Oracle).+ Drivers with separate databases per dataset (like Snowflake) should override+ to return just the table name."+ {:arglists '([driver database-name table-name]), :added "0.57.0"}+ dispatch-on-driver-with-test-extensions+ :hierarchy #'driver/hierarchy)++(defmethod fake-sync-table-name ::test-extensions+ [_driver database-name table-name]+ (db-qualified-table-name database-name table-name))++(defmulti fake-sync-database-type+ "Return the database_type string for fake sync Field rows.+ By default uses `sql.tx/field-base-type->sql-type` (the DDL type used to create columns).+ Drivers where the reported type differs from the creation type (like Snowflake where+ TEXT becomes VARCHAR, FLOAT becomes DOUBLE) should override to return what the+ database actually reports in INFORMATION_SCHEMA."+ {:arglists '([driver base-type]), :added "0.57.0"}+ dispatch-on-driver-with-test-extensions+ :hierarchy #'driver/hierarchy)++(defmethod fake-sync-database-type ::test-extensions+ [driver base-type]+ ;; Default: use the DDL type (what we create columns with)+ ((requiring-resolve 'metabase.test.data.sql/field-base-type->sql-type) driver base-type))++(defmulti fake-sync-base-type+ "Return the base_type for fake sync Field rows.+ By default returns the base-type from the test definition unchanged.+ Drivers where the reported type differs (like Snowflake where INTEGER columns+ are reported as NUMBER with base_type :type/Number) should override."+ {:arglists '([driver base-type]), :added "0.57.0"}+ dispatch-on-driver-with-test-extensions+ :hierarchy #'driver/hierarchy)++(defmethod fake-sync-base-type ::test-extensions+ [_driver base-type]+ ;; Default: use the base-type from the test definition unchanged+ base-type)++(defmulti fake-sync-native-base-type+ "Return the base_type for fake sync Field rows when using native database types.+ Called when base-type is a map like {:native \"timestamptz\"} and no effective-type+ is specified. By default returns :type/*, but drivers should override this to+ return the base_type that sync would actually produce for that native type.+ For example, Snowflake should map \"timestamptz\" -> :type/DateTimeWithLocalTZ."+ {:arglists '([driver native-type-string]), :added "0.57.0"}+ dispatch-on-driver-with-test-extensions+ :hierarchy #'driver/hierarchy)++(defmethod fake-sync-native-base-type ::test-extensions+ [_driver _native-type]+ ;; Default: return :type/* for unknown native types+ :type/*)+(defn on-master-or-release-branch?"Returns true if running on master or a release-* branch.Detection methods (in priority order):