Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Enable fake-sync for Snowflake driver tests (#68322)

bryan0982712225e4
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index db5b888fb53..f770123d384 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -36,6 +36,7 @@
[metabase.test :as mt]
[metabase.test.data.dataset-definitions :as defs]
[metabase.test.data.impl :as data.impl]
+ [metabase.test.data.impl.get-or-create :as test.get-or-create]
[metabase.test.data.interface :as tx]
[metabase.test.data.snowflake :as test.data.snowflake]
[metabase.test.data.sql :as sql.tx]
@@ -294,29 +295,31 @@
:additional-options "foo=bar"))))))))
(deftest describe-database-test
+ ;; This test calls driver/describe-database which queries Snowflake directly.
+ ;; Requires real sync (not fake-sync) so tables actually exist in Snowflake.
(mt/test-driver :snowflake
- (testing "describe-database"
- (let [expected {:tables
- #{{:name "users", :schema "PUBLIC", :description nil}
- {:name "venues", :schema "PUBLIC", :description nil}
- {:name "checkins", :schema "PUBLIC", :description nil}
- {:name "categories", :schema "PUBLIC", :description nil}
- {:name "orders", :schema "PUBLIC", :description nil}
- {:name "people", :schema "PUBLIC", :description nil}
- {:name "products", :schema "PUBLIC", :description nil}
- {:name "reviews", :schema "PUBLIC", :description nil}}}]
- (testing "should work with normal details"
- (is (= expected
- (driver/describe-database :snowflake (mt/db)))))
- (testing "should accept either `:db` or `:dbname` in the details, working around a bug with the original impl"
- (is (= expected
- (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :dbname})))))
- (testing "should throw an Exception if details have neither `:db` nor `:dbname`"
- (is (thrown? Exception
- (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :xyz})))))
- (testing "should use the NAME FROM DETAILS instead of the DB DISPLAY NAME to fetch metadata (#8864)"
- (is (= expected
- (driver/describe-database :snowflake (assoc (mt/db) :name "ABC")))))))))
+ (tx/with-driver-supports-feature! [:snowflake :test/use-fake-sync false]
+ (testing "describe-database"
+ (let [expected-tables #{{:name "users", :schema "PUBLIC", :description nil}
+ {:name "venues", :schema "PUBLIC", :description nil}
+ {:name "checkins", :schema "PUBLIC", :description nil}
+ {:name "categories", :schema "PUBLIC", :description nil}
+ {:name "orders", :schema "PUBLIC", :description nil}
+ {:name "people", :schema "PUBLIC", :description nil}
+ {:name "products", :schema "PUBLIC", :description nil}
+ {:name "reviews", :schema "PUBLIC", :description nil}}]
+ (testing "should work with normal details"
+ (is (= expected-tables
+ (:tables (driver/describe-database :snowflake (mt/db))))))
+ (testing "should accept either `:db` or `:dbname` in the details, working around a bug with the original impl"
+ (is (= expected-tables
+ (:tables (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :dbname}))))))
+ (testing "should throw an Exception if details have neither `:db` nor `:dbname`"
+ (is (thrown? Exception
+ (driver/describe-database :snowflake (update (mt/db) :details set/rename-keys {:db :xyz})))))
+ (testing "should use the NAME FROM DETAILS instead of the DB DISPLAY NAME to fetch metadata (#8864)"
+ (is (= expected-tables
+ (:tables (driver/describe-database :snowflake (assoc (mt/db) :name "ABC")))))))))))
(deftest describe-database-default-schema-test
(testing "describe-database should include Tables from all schemas even if the DB has a default schema (#38135)"
@@ -1423,23 +1426,44 @@
(is (= exp-rows (mt/rows result))))))))
(deftest snowflake-with-dbname-in-details-gets-synced-test
+ ;; This test calls driver/describe-database which queries Snowflake directly.
+ ;; Requires real sync (not fake-sync) so tables actually exist in Snowflake.
(testing "db with a valid db and an invalid dbname in details should be synced with db correctly"
(mt/test-driver :snowflake
- (let [priv-key-val (mt/priv-key->base64-uri (tx/db-test-env-var-or-throw :snowflake :private-key))]
- (mt/with-temp [:model/Database db {:engine :snowflake
- :details (-> (:details (mt/db))
- (dissoc :private-key-id)
- (assoc :private-key-options "uploaded")
- (assoc :private-key-value priv-key-val)
- (assoc :use-password false)
- (assoc :dbname nil))}]
- (is (= {:tables
- #{{:name "users", :schema "PUBLIC", :description nil}
- {:name "venues", :schema "PUBLIC", :description nil}
- {:name "checkins", :schema "PUBLIC", :description nil}
- {:name "categories", :schema "PUBLIC", :description nil}
- {:name "orders", :schema "PUBLIC", :description nil}
- {:name "people", :schema "PUBLIC", :description nil}
- {:name "products", :schema "PUBLIC", :description nil}
- {:name "reviews", :schema "PUBLIC", :description nil}}}
- (driver/describe-database :snowflake db))))))))
+ (tx/with-driver-supports-feature! [:snowflake :test/use-fake-sync false]
+ (let [priv-key-val (mt/priv-key->base64-uri (tx/db-test-env-var-or-throw :snowflake :private-key))
+ expected-tables #{{:name "users", :schema "PUBLIC", :description nil}
+ {:name "venues", :schema "PUBLIC", :description nil}
+ {:name "checkins", :schema "PUBLIC", :description nil}
+ {:name "categories", :schema "PUBLIC", :description nil}
+ {:name "orders", :schema "PUBLIC", :description nil}
+ {:name "people", :schema "PUBLIC", :description nil}
+ {:name "products", :schema "PUBLIC", :description nil}
+ {:name "reviews", :schema "PUBLIC", :description nil}}]
+ (mt/with-temp [:model/Database db {:engine :snowflake
+ :details (-> (:details (mt/db))
+ (dissoc :private-key-id)
+ (assoc :private-key-options "uploaded")
+ (assoc :private-key-value priv-key-val)
+ (assoc :use-password false)
+ (assoc :dbname nil))}]
+ (is (= expected-tables
+ (:tables (driver/describe-database :snowflake db))))))))))
+
+;;; ------------------------------------------------ Fake Sync Tests ------------------------------------------------
+;; Tests to validate that fake sync produces correct metadata for Snowflake.
+;; The key difference from Redshift: Snowflake uses plain table names ("venues") because each
+;; dataset has its own database, while Redshift uses qualified names ("test_data_venues").
+
+(deftest ^:parallel fake-sync-transformation-test
+ (mt/test-driver :snowflake
+ (testing "Snowflake fake-sync uses plain table names (not qualified)"
+ ;; Snowflake has separate databases per dataset, so tables are just "users"
+ ;; not "transform_test_users" like Redshift which shares a single database.
+ (let [dbdef {:database-name "transform-test"
+ :table-definitions [{:table-name "users"
+ :field-definitions [{:field-name "name" :base-type :type/Text}]}]}
+ rows (@#'test.get-or-create/dbdef->fake-sync-rows :snowflake 123 dbdef)
+ table (:table-row (first rows))]
+ (is (= "users" (:name table)) "Should be plain name, not 'transform_test_users'")
+ (is (= "PUBLIC" (:schema table)))))))
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 5c482b59305..232b0f488dc 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -32,6 +32,8 @@
:type/Decimal "DECIMAL"
:type/Float "FLOAT"
:type/Integer "INTEGER"
+ ;; :type/Number is used by tx/id-field-type for Snowflake PKs
+ :type/Number "NUMBER"
:type/Text "TEXT"
;; 3 = millisecond precision. Default is allegedly 9 (nanosecond precision) according to
;; https://docs.snowflake.com/en/sql-reference/data-types-datetime#time, but it seems like
@@ -387,3 +389,91 @@
(into [] (map (juxt :database_name :created)))))
(defmethod sql.tx/session-schema :snowflake [_driver] "PUBLIC")
+
+;;; ------------------------------------------------ Fake Sync Support ------------------------------------------------
+
+;; Enable fake sync for Snowflake on feature branches.
+;; Fake sync skips network calls to the database for metadata sync, which saves significant CI time.
+;; On master/release branches, use real sync to catch any sync regressions.
+(defmethod driver/database-supports? [:snowflake :test/use-fake-sync]
+ [_driver _feature _database]
+ (not (tx/on-master-or-release-branch?)))
+
+(defmethod tx/fake-sync-schema :snowflake
+ [_driver]
+ "PUBLIC")
+
+(defmethod tx/fake-sync-table-name :snowflake
+ [_driver _database-name table-name]
+ ;; Snowflake uses separate databases per dataset, so table names are NOT prefixed
+ ;; with the database name. Unlike Redshift (which uses test_data_venues), Snowflake
+ ;; tables are just "venues" within the sha_xxx_test_data database.
+ table-name)
+
+(defmethod tx/fake-sync-database-type :snowflake
+ [_driver base-type]
+ ;; Return the database_type as Snowflake's query processor expects it.
+ ;; The QP uses lowercase types without precision (e.g., "time" not "TIME(3)").
+ ;; Snowflake normalizes types: TEXT->VARCHAR, FLOAT->DOUBLE, INTEGER->NUMBER
+ ;;
+ ;; For timezone columns: DDL uses TIMESTAMP_TZ, but DB reports as TIMESTAMPTZ
+ (case base-type
+ :type/Text "VARCHAR"
+ :type/Float "DOUBLE"
+ :type/Integer "NUMBER"
+ :type/BigInteger "NUMBER"
+ :type/Number "NUMBER"
+ :type/Boolean "BOOLEAN"
+ :type/Date "date"
+ :type/DateTime "timestampntz"
+ :type/DateTimeWithTZ "timestamptz" ; DDL: TIMESTAMP_TZ, reported as: TIMESTAMPTZ
+ :type/DateTimeWithLocalTZ "timestamptz"
+ :type/DateTimeWithZoneID "timestamptz"
+ :type/DateTimeWithZoneOffset "timestamptz"
+ :type/Time "time"
+ :type/TimeWithLocalTZ "time"
+ :type/TimeWithZoneOffset "time"
+ ;; For other types, use the creation type
+ (sql.tx/field-base-type->sql-type :snowflake base-type)))
+
+(defmethod tx/fake-sync-base-type :snowflake
+ [_driver base-type]
+ ;; Snowflake normalizes some types. Real sync maps them to specific base_types,
+ ;; so fake-sync must match what sync would produce:
+ ;; - INTEGER/BIGINT -> NUMBER -> :type/Number
+ ;; - TimeWithLocalTZ/TimeWithZoneOffset -> TIME -> :type/Time (Snowflake only has one TIME type)
+ ;; - DateTimeWithTZ/DateTimeWithZoneID/DateTimeWithZoneOffset -> TIMESTAMP_TZ -> :type/DateTimeWithLocalTZ
+ ;; (Note: :type/DateTimeWithTZ -> TIMESTAMP_TZ -> sync as TIMESTAMPTZ -> :type/DateTimeWithLocalTZ)
+ (case base-type
+ :type/Integer :type/Number
+ :type/BigInteger :type/Number
+ :type/TimeWithLocalTZ :type/Time
+ :type/TimeWithZoneOffset :type/Time
+ :type/DateTimeWithTZ :type/DateTimeWithLocalTZ
+ :type/DateTimeWithZoneID :type/DateTimeWithLocalTZ
+ :type/DateTimeWithZoneOffset :type/DateTimeWithLocalTZ
+ ;; Other types are unchanged
+ base-type))
+
+(defmethod tx/fake-sync-native-base-type :snowflake
+ [_driver native-type]
+ ;; Map native Snowflake type strings to their base_type.
+ ;; These must match what sql-jdbc.sync/database-type->base-type returns for Snowflake.
+ ;; See metabase.driver.snowflake for the full mapping.
+ (case (some-> native-type u/upper-case-en)
+ ;; Timestamp types
+ "TIMESTAMPTZ" :type/DateTimeWithLocalTZ
+ "TIMESTAMPLTZ" :type/DateTimeWithTZ
+ "TIMESTAMPNTZ" :type/DateTime
+ "TIMESTAMP" :type/DateTime
+ ;; Other common types
+ "VARCHAR" :type/Text
+ "TEXT" :type/Text
+ "NUMBER" :type/Number
+ "FLOAT" :type/Float
+ "DOUBLE" :type/Float
+ "BOOLEAN" :type/Boolean
+ "DATE" :type/Date
+ "TIME" :type/Time
+ ;; Default: unknown types get :type/*
+ :type/*))
diff --git a/test/metabase/test/data/impl/get_or_create.clj b/test/metabase/test/data/impl/get_or_create.clj
index 636bd8b566a..a97d40cf717 100644
--- a/test/metabase/test/data/impl/get_or_create.clj
+++ b/test/metabase/test/data/impl/get_or_create.clj
@@ -132,7 +132,8 @@
;; ---------------------- Pure Transformation Functions ----------------------
(defn- field-def->row
- "Convert a FieldDefinition to a Field row map (without table_id, added during insertion).
+ "Convert a FieldDefinition to a Field row map for fake-sync insertion.
+ Creates the metadata row that would normally come from syncing the database.
Handles three forms of base-type:
1. {:native \"BINARY(8)\"} - driver-specific native type string
2. {:natives {:postgres \"BYTEA\" :redshift \"VARBYTE\"}} - per-driver native types
@@ -147,13 +148,18 @@
(get-in base-type [:natives driver])
:else
- ;; Use the SQL type from sql.tx/field-base-type->sql-type to get
- ;; the actual database type (e.g., "TIMESTAMPTZ" instead of "DateTimeWithTZ").
- ;; Use requiring-resolve to avoid cyclic dependency.
- ((requiring-resolve 'metabase.test.data.sql/field-base-type->sql-type) driver base-type))
+ ;; Use fake-sync-database-type to get the type the database reports
+ ;; (which may differ from the DDL type used to create columns).
+ ;; E.g., Snowflake: TEXT->VARCHAR, FLOAT->DOUBLE, INTEGER->NUMBER
+ (tx/fake-sync-database-type driver base-type))
actual-base-type (if (map? base-type)
- (or effective-type :type/*)
- base-type)
+ ;; For native types, use effective-type if provided,
+ ;; otherwise ask the driver what base_type sync would produce
+ (or effective-type
+ (tx/fake-sync-native-base-type driver database-type))
+ ;; Use fake-sync-base-type to get what the database actually reports
+ ;; E.g., Snowflake: :type/Integer -> :type/Number (since INTEGER->NUMBER)
+ (tx/fake-sync-base-type driver base-type))
semantic-type (cond
pk? :type/PK
(some? fk) :type/FK
@@ -175,11 +181,11 @@
"Convert a TableDefinition to a map of {:table-name, :table-row, :field-rows}.
This is a pure function - no database insertion."
[driver db-id database-name {:keys [table-name table-comment field-definitions] :as _table-def}]
- (let [schema (tx/fake-sync-schema driver)
- qualified-name (tx/db-qualified-table-name database-name table-name)
+ (let [schema (tx/fake-sync-schema driver)
+ sync-table-name (tx/fake-sync-table-name driver database-name table-name)
has-custom-pk? (some :pk? field-definitions)
table-row {:db_id db-id
- :name qualified-name
+ :name sync-table-name
:schema schema
:display_name (humanization/name->human-readable-name :simple table-name)
:description table-comment
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index cd945cf3278..bb79d45d055 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -528,6 +528,64 @@
[_driver]
nil)
+(defmulti fake-sync-table-name
+ "Return the table name to use for fake sync Table rows.
+ Default uses `db-qualified-table-name` (e.g., 'test_data_venues') for drivers
+ that share a single database across datasets (like Redshift, Oracle).
+ Drivers with separate databases per dataset (like Snowflake) should override
+ to return just the table name."
+ {:arglists '([driver database-name table-name]), :added "0.57.0"}
+ dispatch-on-driver-with-test-extensions
+ :hierarchy #'driver/hierarchy)
+
+(defmethod fake-sync-table-name ::test-extensions
+ [_driver database-name table-name]
+ (db-qualified-table-name database-name table-name))
+
+(defmulti fake-sync-database-type
+ "Return the database_type string for fake sync Field rows.
+ By default uses `sql.tx/field-base-type->sql-type` (the DDL type used to create columns).
+ Drivers where the reported type differs from the creation type (like Snowflake where
+ TEXT becomes VARCHAR, FLOAT becomes DOUBLE) should override to return what the
+ database actually reports in INFORMATION_SCHEMA."
+ {:arglists '([driver base-type]), :added "0.57.0"}
+ dispatch-on-driver-with-test-extensions
+ :hierarchy #'driver/hierarchy)
+
+(defmethod fake-sync-database-type ::test-extensions
+ [driver base-type]
+ ;; Default: use the DDL type (what we create columns with)
+ ((requiring-resolve 'metabase.test.data.sql/field-base-type->sql-type) driver base-type))
+
+(defmulti fake-sync-base-type
+ "Return the base_type for fake sync Field rows.
+ By default returns the base-type from the test definition unchanged.
+ Drivers where the reported type differs (like Snowflake where INTEGER columns
+ are reported as NUMBER with base_type :type/Number) should override."
+ {:arglists '([driver base-type]), :added "0.57.0"}
+ dispatch-on-driver-with-test-extensions
+ :hierarchy #'driver/hierarchy)
+
+(defmethod fake-sync-base-type ::test-extensions
+ [_driver base-type]
+ ;; Default: use the base-type from the test definition unchanged
+ base-type)
+
+(defmulti fake-sync-native-base-type
+ "Return the base_type for fake sync Field rows when using native database types.
+ Called when base-type is a map like {:native \"timestamptz\"} and no effective-type
+ is specified. By default returns :type/*, but drivers should override this to
+ return the base_type that sync would actually produce for that native type.
+ For example, Snowflake should map \"timestamptz\" -> :type/DateTimeWithLocalTZ."
+ {:arglists '([driver native-type-string]), :added "0.57.0"}
+ dispatch-on-driver-with-test-extensions
+ :hierarchy #'driver/hierarchy)
+
+(defmethod fake-sync-native-base-type ::test-extensions
+ [_driver _native-type]
+ ;; Default: return :type/* for unknown native types
+ :type/*)
+
(defn on-master-or-release-branch?
"Returns true if running on master or a release-* branch.
Detection methods (in priority order):