The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add CSV uploads to Snowflake (#77239)
da60e1b995d38e
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 296180c0dd8..0aceaca33cd 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -1329,8 +1329,11 @@ jobs:- name: Driver Teststest-args: >-:only-tags [:mb/driver-tests]- :exclude-tags [:mb/transforms-python-test]+ :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]':fail-fast? true+ - name: Upload Tests+ test-args: >-+ :only-tags [:mb/upload-tests]- name: Transforms Python Teststest-args: >-:only-tags [:mb/transforms-python-test]diff --git a/docs/databases/uploads.md b/docs/databases/uploads.mdindex 3a60077d873..e9e4188ed60 100644--- a/docs/databases/uploads.md+++ b/docs/databases/uploads.md@@ -19,6 +19,7 @@ To manage upload settings, admins can hit cmd/ctrl + K and search for "Settings- [PostgreSQL](../databases/connections/postgresql.md)- [MySQL](../databases/connections/mysql.md)+- [Snowflake](../databases/connections/snowflake.md)- [Redshift](../databases/connections/redshift.md)- [ClickHouse](../databases/connections/clickhouse.md) (only supported on ClickHouse Cloud)diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex 0360e478739..c8b34fb9890 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -71,6 +71,7 @@:collate true:now true:database-routing true+ :uploads true:metadata/table-existence-check true:regex/lookaheads-and-lookbehinds false:transforms/accurate-rows-affected false@@ -382,6 +383,63 @@[_driver base-type](type->database-type base-type))+(defmethod driver/upload-type->database-type :snowflake+ [_driver upload-type]+ (case upload-type+ :metabase.upload/varchar-255 [[:varchar 255]]+ :metabase.upload/text [:text]+ :metabase.upload/int [:bigint]+ ;; `_mb_row_id` must follow insertion order, but the default `NOORDER` allocates ids from per-cluster+ ;; ranges, leaving gaps or even out-of-order values between inserts. `ORDER` fixes that by serializing+ ;; id generation -- an acceptable cost, since uploads insert from a single connection.+ :metabase.upload/auto-incrementing-int-pk [:number [:identity 1 1] :order]+ :metabase.upload/float [:double]+ :metabase.upload/boolean [:boolean]+ :metabase.upload/date [:date]+ :metabase.upload/datetime [:timestamp_ntz]+ :metabase.upload/offset-datetime [:timestamp_tz]))++;; Snowflake's JDBC driver cannot bind `java.time` values: `setObject` handles temporal binds only via the+;; legacy `java.sql.Date`/`Time`/`Timestamp` classes, and throws on anything else.+;;+;; Converting through those legacy classes is no better: `setTimestamp` binds nanos-since-epoch as+;; `TIMESTAMP_LTZ` computed in the JVM default zone, while the session runs with `TIMEZONE=UTC`+;; (see [[connection-details->spec]]), so wall-clock times would shift whenever the JVM zone isn't UTC.+;;+;; String binds sidestep both problems: Snowflake parses them server-side into the column's type, independent+;; of any timezone. Nine fractional digits keep full nanosecond precision -- the CSV parser accepts arbitrary+;; sub-second precision and Snowflake timestamps store up to 9 digits.+(defn- temporal-bind->string+ "Convert a temporal upload value to a string bind that Snowflake will coerce to the column type.+ Non-temporal values pass through unchanged."+ [v]+ (condp instance? v+ LocalDate (u.date/format v)+ LocalDateTime (u.date/format "yyyy-MM-dd HH:mm:ss.SSSSSSSSS" v)+ OffsetDateTime (u.date/format "yyyy-MM-dd HH:mm:ss.SSSSSSSSS xx" v)+ v))++(defmethod driver/insert-into! :snowflake+ [driver db-id table-name column-names values]+ ;; Snowflake's fast bulk path (`PUT` + `COPY INTO` from a stage) is unavailable because [[set-put-get]]+ ;; enforces `enablePutGet=false` on every connection (#73578), so use the generic multi-row `INSERT`.+ ((get-method driver/insert-into! :sql-jdbc)+ driver db-id table-name column-names+ (map #(mapv temporal-bind->string %) values)))++(defmethod driver/add-columns! :snowflake+ [driver db-id table-name column-definitions & args]+ ;; Snowflake doesn't support adding multiple columns in one statement, so add them one at a time+ (let [add-column! (get-method driver/add-columns! :sql-jdbc)]+ (doseq [[column definition] column-definitions]+ (apply add-column! driver db-id table-name {column definition} args))))++(defmethod driver/allowed-promotions :snowflake+ [_driver]+ ;; Snowflake's `ALTER COLUMN` can widen a type (e.g. VARCHAR length, NUMBER precision) but never convert+ ;; to a different one, so no promotions are possible+ {})+(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :seconds] [_ _ expr] [:to_timestamp_tz expr])(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :milliseconds] [_ _ expr] [:to_timestamp_tz expr 3])(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :microseconds] [_ _ expr] [:to_timestamp_tz expr 6])diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex 64da9fabb19..691690e8bff 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -1734,3 +1734,46 @@(mt/with-native-query-testing-context query(is (some? (mt/rows (qp/process-query query)))"Hour bucketing on a time field from a source query should not error")))))))++(deftest upload-type->database-type-test+ (testing "upload types render to the expected Snowflake DDL via the generic create-table! path"+ ;; End-to-end behavior is covered by [[metabase.upload.impl-test]].+ (let [column-type (fn [upload-type]+ (driver/upload-type->database-type :snowflake upload-type))]+ (is (= (str "CREATE TABLE \"PUBLIC\".\"test_uploads\" ("+ "\"_mb_row_id\" NUMBER IDENTITY(1, 1) ORDER, "+ "\"name\" VARCHAR(255), "+ "\"bio\" TEXT, "+ "\"count\" BIGINT, "+ "\"ratio\" DOUBLE, "+ "\"active\" BOOLEAN, "+ "\"birthday\" DATE, "+ "\"created\" TIMESTAMP_NTZ, "+ "\"updated\" TIMESTAMP_TZ, "+ "PRIMARY KEY(\"_mb_row_id\"))")+ (@#'driver.sql-jdbc/create-table!-sql+ :snowflake+ "PUBLIC.test_uploads"+ (array-map+ :_mb_row_id (column-type :metabase.upload/auto-incrementing-int-pk)+ :name (column-type :metabase.upload/varchar-255)+ :bio (column-type :metabase.upload/text)+ :count (column-type :metabase.upload/int)+ :ratio (column-type :metabase.upload/float)+ :active (column-type :metabase.upload/boolean)+ :birthday (column-type :metabase.upload/date)+ :created (column-type :metabase.upload/datetime)+ :updated (column-type :metabase.upload/offset-datetime))+ :primary-key [:_mb_row_id]))))))++(deftest temporal-bind->string-test+ (testing "temporal upload binds keep full nanosecond precision"+ ;; The CSV parser accepts arbitrary sub-second precision, so formatting with fewer than 9 fractional+ ;; digits would silently truncate values on upload.+ (are [v expected] (= expected (#'driver.snowflake/temporal-bind->string v))+ #t "2026-07-08" "2026-07-08"+ #t "2026-07-08T01:02:03.123456789" "2026-07-08 01:02:03.123456789"+ (t/offset-date-time 2026 7 8 1 2 3 123456789+ (t/zone-offset "+02:00")) "2026-07-08 01:02:03.123456789 +0200"+ "not temporal" "not temporal"+ 42 42)))diff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.cljindex 3fb605d95be..7be77cfb861 100644--- a/test/metabase/transforms/execute_test.clj+++ b/test/metabase/transforms/execute_test.clj@@ -142,7 +142,7 @@(mt/as-admin(testing "create-table-from-schema! should create the table successfully"(transforms-base.u/create-table-from-schema! driver db-id table-schema)- (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]+ (let [table-exists? (driver/table-exists? driver (mt/db) {:schema schema-name :name table-name})](is (some? table-exists?) "Table should exist in the database schema")(driver/drop-table! driver db-id (:name table-schema))))))))