Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Map Snowflake big integers to :type/BigInteger (#67609)

Cam Saulbd1e91191c04
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
index ad9a9544326..88c93bff4ad 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
@@ -442,29 +442,30 @@
(deftest clickhouse-filtered-aggregate-functions-test-table-metadata
(mt/test-driver :clickhouse
(mt/dataset introspection_db
- (is (= {:name "aggregate_functions_filter_test"
- :fields #{(merge base-field
- {:name "id",
- :database-type "Int32",
- :base-type :type/Integer,
+ (is (=? {:name "aggregate_functions_filter_test"
+ :fields [(merge base-field
+ {:name "id"
+ :database-type "Int32"
+ :base-type :type/Integer
:database-required true
:database-position 0})
(merge base-field
- {:name "idx"
- :database-type "UInt8"
- :base-type :type/Integer
+ {:name "idx"
+ :database-type "UInt8"
+ :base-type :type/Integer
:database-position 1})
(merge base-field
- {:name "lowest_value"
- :database-type "SimpleAggregateFunction(min, UInt8)"
- :base-type :type/Integer
+ {:name "lowest_value"
+ :database-type "SimpleAggregateFunction(min, UInt8)"
+ :base-type :type/Integer
:database-position 3})
(merge base-field
- {:name "count"
- :database-type "SimpleAggregateFunction(sum, Int64)"
- :base-type :type/BigInteger
- :database-position 4})}}
- (driver/describe-table :clickhouse (mt/db) {:name "aggregate_functions_filter_test"}))))))
+ {:name "count"
+ :database-type "SimpleAggregateFunction(sum, Int64)"
+ :base-type :type/BigInteger
+ :database-position 4})]}
+ (-> (driver/describe-table :clickhouse (mt/db) {:name "aggregate_functions_filter_test"})
+ (update :fields (partial sort-by :database-position))))))))
(deftest clickhouse-filtered-aggregate-functions-test-result-set
(mt/test-driver :clickhouse
@@ -583,5 +584,3 @@
[(format "DROP VIEW IF EXISTS %s.parameterized_view" db-name)
(format "DROP TABLE IF EXISTS %s.table_after_view" db-name)]
details))))))))
-
-
diff --git a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
index 322564b97ae..5c19261a622 100644
--- a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
+++ b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
@@ -77,7 +77,7 @@
(defmethod tx/dbdef->connection-details :clickhouse [driver context {:keys [database-name]}]
(merge
{:host (mt/db-test-env-var :clickhouse :host)
- :port (Integer/parseInt (mt/db-test-env-var :clickhouse :port))
+ :port (parse-long (mt/db-test-env-var-or-throw :clickhouse :port))
:enable-multiple-db true}
(when-let [user (mt/db-test-env-var :clickhouse :user)]
{:user user})
diff --git a/modules/drivers/druid-jdbc/test/metabase/driver/druid_jdbc_test.clj b/modules/drivers/druid-jdbc/test/metabase/driver/druid_jdbc_test.clj
index adc831624dd..973007e9547 100644
--- a/modules/drivers/druid-jdbc/test/metabase/driver/druid_jdbc_test.clj
+++ b/modules/drivers/druid-jdbc/test/metabase/driver/druid_jdbc_test.clj
@@ -36,103 +36,104 @@
(testing "describe-table"
(is (=? {:schema "druid"
:name "checkins"
- :fields #{{:name "__time",
- :database-type "TIMESTAMP",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable false
- :base-type :type/DateTime,
- :database-position 0,
- :json-unfolding false}
- {:name "count",
- :database-type "BIGINT",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/BigInteger,
- :database-position 10,
- :json-unfolding false}
- {:name "id",
- :database-type "BIGINT",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/BigInteger,
- :database-position 1,
- :json-unfolding false}
- {:name "unique_users",
- :database-type "COMPLEX<hyperUnique>",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/DruidHyperUnique,
- :database-position 11,
- :json-unfolding false}
- {:name "user_last_login",
- :database-type "VARCHAR",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/Text,
- :database-position 2,
- :json-unfolding false}
- {:name "user_name",
- :database-type "VARCHAR",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/Text,
- :database-position 3,
- :json-unfolding false}
- {:name "user_password",
- :database-type "VARCHAR",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/Text,
- :database-position 4,
- :json-unfolding false}
- {:name "venue_category_name",
- :database-type "VARCHAR",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/Text,
- :database-position 5,
- :json-unfolding false}
- {:name "venue_latitude",
- :database-type "DOUBLE",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/Float,
- :database-position 6,
- :json-unfolding false}
- {:name "venue_longitude",
- :database-type "DOUBLE",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/Float,
- :database-position 7,
- :json-unfolding false}
- {:name "venue_name",
- :database-type "VARCHAR",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/Text,
- :database-position 8,
- :json-unfolding false}
- {:name "venue_price",
- :database-type "BIGINT",
- :database-required false,
- :database-is-auto-increment false,
- :database-is-nullable true
- :base-type :type/BigInteger,
- :database-position 9,
- :json-unfolding false}}}
- (driver/describe-table :druid-jdbc (mt/db) {:schema "druid" :name "checkins"}))))
+ :fields [{:name "__time",
+ :database-type "TIMESTAMP",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable false
+ :base-type :type/DateTime,
+ :database-position 0,
+ :json-unfolding false}
+ {:name "count",
+ :database-type "BIGINT",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/BigInteger,
+ :database-position 10,
+ :json-unfolding false}
+ {:name "id",
+ :database-type "BIGINT",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/BigInteger,
+ :database-position 1,
+ :json-unfolding false}
+ {:name "unique_users",
+ :database-type "COMPLEX<hyperUnique>",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/DruidHyperUnique,
+ :database-position 11,
+ :json-unfolding false}
+ {:name "user_last_login",
+ :database-type "VARCHAR",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/Text,
+ :database-position 2,
+ :json-unfolding false}
+ {:name "user_name",
+ :database-type "VARCHAR",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/Text,
+ :database-position 3,
+ :json-unfolding false}
+ {:name "user_password",
+ :database-type "VARCHAR",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/Text,
+ :database-position 4,
+ :json-unfolding false}
+ {:name "venue_category_name",
+ :database-type "VARCHAR",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/Text,
+ :database-position 5,
+ :json-unfolding false}
+ {:name "venue_latitude",
+ :database-type "DOUBLE",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/Float,
+ :database-position 6,
+ :json-unfolding false}
+ {:name "venue_longitude",
+ :database-type "DOUBLE",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/Float,
+ :database-position 7,
+ :json-unfolding false}
+ {:name "venue_name",
+ :database-type "VARCHAR",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/Text,
+ :database-position 8,
+ :json-unfolding false}
+ {:name "venue_price",
+ :database-type "BIGINT",
+ :database-required false,
+ :database-is-auto-increment false,
+ :database-is-nullable true
+ :base-type :type/BigInteger,
+ :database-position 9,
+ :json-unfolding false}]}
+ (-> (driver/describe-table :druid-jdbc (mt/db) {:schema "druid" :name "checkins"})
+ (update :fields (partial sort-by :name))))))
(testing "Full sync does not throw an exception (field values are calculated only for eligible fields)"
(is (=? [::success some?]
(try (let [result (sync/sync-database! (mt/db))]
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index 7de56fb0ff2..344a126a022 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -1,6 +1,6 @@
(ns metabase.driver.snowflake
"Snowflake Driver."
- (:refer-clojure :exclude [select-keys not-empty get-in])
+ (:refer-clojure :exclude [select-keys not-empty get-in mapv])
(:require
[buddy.core.codecs :as codecs]
[clojure.java.jdbc :as jdbc]
@@ -33,22 +33,13 @@
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
[metabase.util.log :as log]
- [metabase.util.performance :refer [select-keys not-empty get-in]]
+ [metabase.util.malli :as mu]
+ [metabase.util.performance :refer [get-in mapv not-empty select-keys]]
[ring.util.codec :as codec])
(:import
(java.io File)
- (java.sql
- Connection
- DatabaseMetaData
- ResultSet
- Types)
- (java.time
- LocalDate
- LocalDateTime
- LocalTime
- OffsetDateTime
- OffsetTime
- ZonedDateTime)
+ (java.sql Connection DatabaseMetaData ResultSet ResultSetMetaData Types)
+ (java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
(java.util Properties)
(net.snowflake.client.jdbc SnowflakeConnectString SnowflakeSQLException)))
@@ -269,48 +260,71 @@
;; file. Hence it is moved to connection url. https://github.com/metabase/metabase/issues/43600
(maybe-add-role-to-spec-url details))))
-(defmethod sql-jdbc.sync/database-type->base-type :snowflake
- [_driver base-type]
- ({:NUMBER :type/Number
- :DECIMAL :type/Decimal
- :NUMERIC :type/Number
- :INT :type/Integer
- :INTEGER :type/Integer
- :BIGINT :type/BigInteger
- :SMALLINT :type/Integer
- :TINYINT :type/Integer
- :BYTEINT :type/Integer
- :FLOAT :type/Float
- :FLOAT4 :type/Float
- :FLOAT8 :type/Float
- :DOUBLE :type/Float
- (keyword "DOUBLE PRECISION") :type/Float
- :REAL :type/Float
- :VARCHAR :type/Text
- :CHAR :type/Text
- :CHARACTER :type/Text
- :STRING :type/Text
- :TEXT :type/Text
- :GEOGRAPHY :type/SerializedJSON
- :BINARY :type/*
- :VARBINARY :type/*
- :BOOLEAN :type/Boolean
- :DATE :type/Date
- :DATETIME :type/DateTime
- :TIME :type/Time
- :TIMESTAMP :type/DateTime
+(mu/defn- database-type->base-type
+ [database-type :- string?
+ jdbc-type :- [:maybe int?]]
+ (case database-type
+ "BIGINT" :type/BigInteger
+ "BINARY" :type/*
+ "BOOLEAN" :type/Boolean
+ "BYTEINT" :type/Integer
+ "CHAR" :type/Text
+ "CHARACTER" :type/Text
+ "DECIMAL" :type/Decimal
+ "DOUBLE PRECISION" :type/Float
+ "DOUBLE" :type/Float
+ "FLOAT" :type/Float
+ "FLOAT4" :type/Float
+ "FLOAT8" :type/Float
+ "GEOGRAPHY" :type/SerializedJSON
+ "INT" :type/Integer
+ "INTEGER" :type/Integer
+ "NUMBER" (if (= jdbc-type Types/BIGINT)
+ :type/BigInteger
+ :type/Number)
+ "NUMERIC" :type/Number
+ "REAL" :type/Float
+ "SMALLINT" :type/Integer
+ "STRING" :type/Text
+ "TEXT" :type/Text
+ "TINYINT" :type/Integer
+ "VARBINARY" :type/*
+ "VARCHAR" :type/Text
+ "DATE" :type/Date
+ "DATETIME" :type/DateTime
+ "TIME" :type/Time
+ "TIMESTAMP" :type/DateTime
;; This is a weird one. A timestamp with local time zone, stored without time zone but treated as being in the
;; Session time zone for filtering purposes etc.
- :TIMESTAMPLTZ :type/DateTimeWithTZ
+ "TIMESTAMPLTZ" :type/DateTimeWithTZ
;; timestamp with no time zone
- :TIMESTAMPNTZ :type/DateTime
+ "TIMESTAMPNTZ" :type/DateTime
;; timestamp with time zone normalized to UTC, similar to Postgres
- :TIMESTAMPTZ :type/DateTimeWithLocalTZ
+ "TIMESTAMPTZ" :type/DateTimeWithLocalTZ
;; `VARIANT` is allowed to be any type. See https://docs.snowflake.com/en/sql-reference/data-types-semistructured
- :VARIANT :type/SnowflakeVariant
+ "VARIANT" :type/SnowflakeVariant
;; Maybe also type *
- :OBJECT :type/Dictionary
- :ARRAY :type/*} base-type))
+ "OBJECT" :type/Dictionary
+ "ARRAY" :type/*
+ #_else :type/*))
+
+(defmethod sql-jdbc.sync/database-type->base-type :snowflake
+ [_driver database-type]
+ (database-type->base-type (name database-type) nil))
+
+(defmethod sql-jdbc.execute/column-metadata :snowflake
+ [driver ^ResultSetMetaData rsmeta]
+ (mapv (fn [^Long i]
+ (let [col-name (.getColumnLabel rsmeta i)
+ db-type-name (sql-jdbc.execute/db-type-name driver rsmeta i)
+ jdbc-type (.getColumnType rsmeta i)
+ base-type (database-type->base-type db-type-name jdbc-type)]
+ (log/tracef "Column %d '%s' is a %s (JDBC Type = %d) which is mapped to base type %s for driver %s\n"
+ i col-name db-type-name jdbc-type base-type driver)
+ {:name col-name
+ :base_type (or base-type :type/*)
+ :database_type db-type-name}))
+ (sql-jdbc.execute/column-range rsmeta)))
(defmulti ^:private type->database-type
"Internal type->database-type multimethod for Snowflake that dispatches on type."
@@ -683,6 +697,16 @@
;; for more info.
(vec (sql-jdbc.describe-database/db-tables driver (.getMetaData conn) "%" db-name)))}))))))
+(defmethod sql-jdbc.sync/describe-table-fields :snowflake
+ [driver conn table database]
+ ;; The default implementation of [[sql-jdbc.sync/describe-table-fields]] doesn't use both Database Type (`NUMBER`)
+ ;; and JDBC Type (`Types/BIGINT`) to determine base type, so update the ones from the default implementation using
+ ;; our own logic.
+ (letfn [(fix-base-type [col]
+ (assoc col :base-type (database-type->base-type (:database-type col) (:jdbc-type col))))]
+ (mapv fix-base-type
+ ((get-method sql-jdbc.sync/describe-table-fields :sql-jdbc) driver conn table database))))
+
(defmethod driver/describe-table :snowflake
[driver database table]
(sql-jdbc.execute/do-with-connection-with-options
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index 46a8f4c8890..6fb3ba3f8b2 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -89,7 +89,7 @@
:database-type "NUMBER"
:database-required false
:database-is-auto-increment true
- :base-type :type/Number
+ :base-type :type/BigInteger
:json-unfolding false
:database-position 0
:pk? true}
@@ -104,7 +104,7 @@
:database-type "NUMBER"
:database-required false
:database-is-auto-increment false
- :base-type :type/Number
+ :base-type :type/BigInteger
:json-unfolding false
:database-position 2}
{:name "latitude"
@@ -125,7 +125,7 @@
:database-type "NUMBER"
:database-required false
:database-is-auto-increment false
- :base-type :type/Number
+ :base-type :type/BigInteger
:json-unfolding false
:database-position 5}]
(sort-by :database-position
@@ -375,7 +375,7 @@
(map (partial into {})
(t2/select [:model/Table :name] :db_id (u/the-id database)))))))))))
-(defn- do-with-dynamic-table
+(defn- do-with-dynamic-table!
[thunk]
(mt/dataset (mt/dataset-definition
"dynamic-db"
@@ -389,29 +389,29 @@
(:db details) (:db details) (:db details))])
(thunk))))
-(defmacro with-dynamic-table
+(defmacro with-dynamic-table!
"Create a db with 2 tables: metabase_users and metabase_fan, in which metabase_fan is a dynamic table."
[& body]
- `(do-with-dynamic-table (fn [] ~@body)))
+ `(do-with-dynamic-table! (fn [] ~@body)))
(deftest sync-dynamic-tables-test
(testing "Should be able to sync dynamic tables"
(mt/test-driver :snowflake
- (with-dynamic-table
+ (with-dynamic-table!
(sync/sync-database! (t2/select-one :model/Database (mt/id)))
(testing "both base tables and dynamic tables should be synced"
(is (= #{"metabase_fan" "metabase_users"}
(t2/select-fn-set :name :model/Table :db_id (mt/id))))
(testing "the fields for dynamic tables are synced correctly"
(is (= #{{:name "name" :base_type :type/Text}
- {:name "id" :base_type :type/Number}}
- (set (t2/select [:model/Field :name :base_type]
- :table_id (t2/select-one-pk :model/Table :name "metabase_fan" :db_id (mt/id))))))))))))
+ {:name "id" :base_type :type/BigInteger}}
+ (set (t2/select-fn-set identity [:model/Field :name :base_type]
+ :table_id (t2/select-one-pk :model/Table :name "metabase_fan" :db_id (mt/id))))))))))))
(deftest dynamic-table-helpers-test
(testing "test to make sure various methods called on dynamic tables work"
(mt/test-driver :snowflake
- (with-dynamic-table
+ (with-dynamic-table!
(sql-jdbc.execute/do-with-connection-with-options
:snowflake
(mt/db)
@@ -444,11 +444,11 @@
(deftest ^:sequential describe-table-test
(mt/test-driver :snowflake
(testing "make sure describe-table uses the NAME FROM DETAILS too"
- (is (= {:name "categories"
- :schema "PUBLIC"
- :fields #{{:name "id"
+ (is (=? {:name "categories"
+ :schema "PUBLIC"
+ :fields [{:name "id"
:database-type "NUMBER"
- :base-type :type/Number
+ :base-type :type/BigInteger
:pk? true
:database-position 0
:database-is-auto-increment true
@@ -462,8 +462,9 @@
:database-is-auto-increment false
:database-is-nullable false
:database-required true
- :json-unfolding false}}}
- (driver/describe-table :snowflake (assoc (mt/db) :name "ABC") (t2/select-one :model/Table :id (mt/id :categories))))))))
+ :json-unfolding false}]}
+ (-> (driver/describe-table :snowflake (assoc (mt/db) :name "ABC") (t2/select-one :model/Table :id (mt/id :categories)))
+ (update :fields (partial sort-by :name))))))))
(deftest ^:sequential describe-table-fks-test
(mt/test-driver :snowflake
@@ -1108,7 +1109,7 @@
:snowflake
(mt/dataset
good-datetimes-in-belize
- (is (= [["id" "NUMBER" :type/Number 0]
+ (is (= [["id" "NUMBER" :type/BigInteger 0]
["IN_Z_OFFSET" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 1]
["IN_VARIOUS_OFFSETS" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 2]
["JUST_NTZ" "TIMESTAMPNTZ" :type/DateTime 3]
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 4ef1f996bcb..6c9a7027788 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -234,25 +234,12 @@
(defmethod sql.tx/pk-sql-type :snowflake [_] "INTEGER AUTOINCREMENT")
-(defmethod tx/id-field-type :snowflake [_] :type/Number)
+(defmethod tx/id-field-type :snowflake [_] :type/BigInteger)
(defmethod load-data/row-xform :snowflake
[_driver _dbdef tabledef]
(load-data/maybe-add-ids-xform tabledef))
-(defmethod tx/aggregate-column-info :snowflake
- ([driver ag-type]
- (merge
- ((get-method tx/aggregate-column-info ::tx/test-extensions) driver ag-type)
- (when (#{:count :cum-count} ag-type)
- {:base_type :type/Number})))
-
- ([driver ag-type field]
- (merge
- ((get-method tx/aggregate-column-info ::tx/test-extensions) driver ag-type field)
- (when (#{:count :cum-count} ag-type)
- {:base_type :type/Number}))))
-
(defmethod sql.tx/generated-column-sql :snowflake [_ expr]
(format "AS (%s)" expr))
@@ -478,13 +465,16 @@
[_driver base-type]
;; Snowflake normalizes some types. Real sync maps them to specific base_types,
;; so fake-sync must match what sync would produce:
- ;; - INTEGER/BIGINT -> NUMBER -> :type/Number
+ ;;
+ ;; - `(BIG)INTEGER` -> `NUMBER` + JDBC Type `java.sql.Types/BIGINT` -> `:type/BigInteger` (as of #67609)
+ ;;
;; - TimeWithLocalTZ/TimeWithZoneOffset -> TIME -> :type/Time (Snowflake only has one TIME type)
+ ;;
;; - DateTimeWithTZ/DateTimeWithZoneID/DateTimeWithZoneOffset -> TIMESTAMP_TZ -> :type/DateTimeWithLocalTZ
;; (Note: :type/DateTimeWithTZ -> TIMESTAMP_TZ -> sync as TIMESTAMPTZ -> :type/DateTimeWithLocalTZ)
(case base-type
- :type/Integer :type/Number
- :type/BigInteger :type/Number
+ :type/Integer :type/BigInteger
+ :type/BigInteger :type/BigInteger
:type/TimeWithLocalTZ :type/Time
:type/TimeWithZoneOffset :type/Time
:type/DateTimeWithTZ :type/DateTimeWithLocalTZ
diff --git a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
index 8a8d524a959..2687fd33ef0 100644
--- a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
+++ b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
@@ -137,9 +137,9 @@
(driver/describe-database driver db))))
(testing "timestamp column should exist"
- (is (= {:name "timestamp_table"
- :schema nil
- :fields #{{:name "created_at"
+ (is (=? {:name "timestamp_table"
+ :schema nil
+ :fields [{:name "created_at"
:database-type "TIMESTAMP"
:base-type :type/DateTime
:database-position 0
@@ -147,8 +147,9 @@
:json-unfolding false
:database-is-auto-increment false
:database-is-nullable true
- :database-is-generated false}}}
- (driver/describe-table driver db (t2/select-one :model/Table :id (mt/id :timestamp_table)))))))))))))
+ :database-is-generated false}]}
+ (-> (driver/describe-table driver db (t2/select-one :model/Table :id (mt/id :timestamp_table)))
+ (update :fields (partial sort-by :name)))))))))))))
(deftest select-query-datetime
(mt/test-driver :sqlite
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index 6b272934562..c39a0e003b2 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -652,7 +652,9 @@
[_ rs _ i]
(get-object-of-class-thunk rs i java.time.OffsetTime))
-(defn- column-range [^ResultSetMetaData rsmeta]
+(defn column-range
+ "Return a sequence of all the column indexes in a JDBC result set. Column indexes in JDBC start at `1`."
+ [^ResultSetMetaData rsmeta]
(range 1 (inc (.getColumnCount rsmeta))))
(defn- log-readers [driver ^ResultSetMetaData rsmeta fns]
@@ -687,6 +689,7 @@
(thunk))))))
(defn- resolve-missing-base-types
+ "Resolve missing base types in initial column `metadatas` using [[driver/dynamic-database-types-lookup]]."
[driver metadatas]
(if (driver-api/initialized?)
(let [missing (keep (fn [{:keys [database_type base_type]}]
@@ -727,10 +730,7 @@
:database_type db-type-name}))
(column-range rsmeta))
(resolve-missing-base-types driver)
- (mapv (fn [{:keys [base_type] :as metadata}]
- (if (nil? base_type)
- (assoc metadata :base_type :type/*)
- metadata)))))
+ (mapv #(u/assoc-default % :base_type :type/*))))
(defn reducible-rows
"Returns an object that can be reduced to fetch the rows and columns in a `ResultSet` in a driver-specific way (e.g.
diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
index f2359df89cd..9225400f05c 100644
--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj
+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
@@ -150,23 +150,22 @@
(merge
{:name column-name
:database-type (.getString rs "TYPE_NAME")
+ :jdbc-type (.getInt rs "DATA_TYPE")
:database-is-auto-increment auto-increment?
:database-required required?}
-
;; in the same way drivers are free to not return these attributes, and leave them undefined
;; we should treat unknown values accordingly
(u/remove-nils
{;; COLUMN_DEF = "" observed with clickhouse, druid, "" is never a valid SQL expression, so treat as undefined
;; we should probably not-empty for the purposes of required, but trying to retain existing behaviour for now
- :database-default (not-empty default)
- :database-is-generated generated
- :database-is-nullable nullable})
-
+ :database-default (not-empty default)
+ :database-is-generated generated
+ :database-is-nullable nullable})
(when-let [remarks (.getString rs "REMARKS")]
(when-not (str/blank? remarks)
{:field-comment remarks})))))))
-(defn ^:private fields-metadata
+(defn- fields-metadata
[driver ^Connection conn {schema :schema, table-name :name} ^String db-name-or-nil]
{:pre [(instance? Connection conn) (string? table-name)]}
(reify clojure.lang.IReduceInit
@@ -229,7 +228,8 @@
:database-required
:database-is-auto-increment
:database-is-generated
- :database-is-nullable])
+ :database-is-nullable
+ :jdbc-type])
{:table-schema (:table-schema col) ;; can be nil
:base-type base-type
;; json-unfolding is true by default for JSON fields, but this can be overridden at the DB level
diff --git a/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj
index 7880a075626..d96fbab1945 100644
--- a/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj
+++ b/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj
@@ -318,7 +318,7 @@
:+fns [#(identical? (get-method sql-jdbc.sync/have-select-privilege? :sql-jdbc)
(get-method sql-jdbc.sync/have-select-privilege? %))]
:-features [:table-privileges]})
- (letfn [(probe-error-fn [conn sql]
+ (letfn [(probe-error-fn [^Connection conn sql]
(.close conn)
(.prepareStatement conn sql))]
(testing "we will retry syncing a table once if the connection is closed"
diff --git a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
index a9d533174aa..c5841d7ed3b 100644
--- a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
+++ b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
@@ -60,6 +60,7 @@
:database-is-auto-increment true
:database-is-generated false
:database-is-nullable false
+ :jdbc-type java.sql.Types/BIGINT
:json-unfolding false}
{:name "NAME"
:database-type "CHARACTER VARYING"
@@ -69,6 +70,7 @@
:database-is-auto-increment false
:database-is-generated false
:database-is-nullable true
+ :jdbc-type java.sql.Types/VARCHAR
:json-unfolding false}
{:name "CATEGORY_ID"
:database-type "INTEGER"
@@ -78,6 +80,7 @@
:database-is-auto-increment false
:database-is-generated false
:database-is-nullable true
+ :jdbc-type java.sql.Types/INTEGER
:json-unfolding false}
{:name "LATITUDE"
:database-type "DOUBLE PRECISION"
@@ -87,6 +90,7 @@
:database-is-auto-increment false
:database-is-generated false
:database-is-nullable true
+ :jdbc-type java.sql.Types/FLOAT
:json-unfolding false}
{:name "LONGITUDE"
:database-type "DOUBLE PRECISION"
@@ -96,6 +100,7 @@
:database-is-auto-increment false
:database-is-generated false
:database-is-nullable true
+ :jdbc-type java.sql.Types/FLOAT
:json-unfolding false}
{:name "PRICE"
:database-type "INTEGER"
@@ -105,6 +110,7 @@
:database-is-auto-increment false
:database-is-generated false
:database-is-nullable true
+ :jdbc-type java.sql.Types/INTEGER
:json-unfolding false}}}
(driver/describe-table :h2 (mt/db) {:name "VENUES"})))))
@@ -128,6 +134,7 @@
:database-position 0
:database-required false
:database-type "INTEGER"
+ :jdbc-type java.sql.Types/INTEGER
:name "id"
:pk? true
:json-unfolding false}
@@ -138,6 +145,7 @@
:database-position 1
:database-required false
:database-type "INTEGER"
+ :jdbc-type java.sql.Types/INTEGER
:name "count"
:json-unfolding false}
{:base-type :type/Integer
@@ -147,6 +155,7 @@
:database-position 2
:database-required true
:database-type "INTEGER"
+ :jdbc-type java.sql.Types/INTEGER
:name "rank"
:json-unfolding false}}
:name "employee_counter"}