Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix duplicate coercion on custom columns from joins (#63941)

Cam Saul87923e1e1d9c
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.clj
index feabede3b8f..854ade18f5a 100644
--- a/modules/drivers/athena/src/metabase/driver/athena.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena.clj
@@ -21,16 +21,9 @@
[metabase.util.log :as log]
[metabase.util.performance :refer [some select-keys mapv]])
(:import
- (java.sql
- Connection
- DatabaseMetaData
- Date
- ResultSet
- Time
- Timestamp
- Types)
+ (java.sql Connection DatabaseMetaData Date ResultSet Time Types)
(java.time OffsetDateTime ZonedDateTime)
- [java.util UUID]))
+ (java.util UUID)))
(set! *warn-on-reflection* true)
@@ -155,29 +148,24 @@
(defmethod sql-jdbc.execute/read-column-thunk [:athena Types/TIMESTAMP_WITH_TIMEZONE]
[_driver ^ResultSet rs _rs-meta ^Long i]
- (fn []
- ;; Using ZonedDateTime if available to conform tests first. OffsetDateTime if former is not available.
- (when-some [^Timestamp timestamp (.getObject rs i Timestamp)]
- (let [timestamp-instant (.toInstant timestamp)
- results-timezone (driver-api/results-timezone-id)]
+ (let [results-timezone (driver-api/results-timezone-id)]
+ (fn []
+ ;; always get the timestamp as a string because the underlying JDBC implementation parses strings to
+ ;; `java.sql.Timestamp` anyway and it barfs if it has a nanosecond component.
+ (when-some [t (some-> (.getString rs i) u.date/parse)]
+ ;; Using ZonedDateTime if available to conform tests first. OffsetDateTime if former is not available.
(try
- (t/zoned-date-time timestamp-instant (t/zone-id results-timezone))
+ (u.date/with-time-zone-same-instant t results-timezone)
(catch Throwable _
(log/warnf "Failed to construct ZonedDateTime from `%s` using `%s` timezone."
- (pr-str timestamp-instant)
+ (pr-str t)
(pr-str results-timezone))
- (try
- (t/offset-date-time timestamp-instant results-timezone)
- (catch Throwable _
- (log/warnf "Failed to construct OffsetDateTime from `%s` using `%s` offset. Using `Z` fallback."
- (pr-str timestamp-instant)
- (pr-str results-timezone))
- (t/offset-date-time timestamp-instant "Z")))))))))
+ t))))))
(defmethod sql-jdbc.execute/read-column-thunk [:athena Types/TIMESTAMP]
[_driver ^ResultSet rs _rs-meta ^Long i]
- (fn [] (some-> ^Timestamp (.getObject rs i Timestamp)
- (t/local-date-time))))
+ (fn []
+ (some-> (.getString rs i) u.date/parse)))
(defmethod sql-jdbc.execute/read-column-thunk [:athena Types/DATE]
[_driver ^ResultSet rs _rs-meta ^Long i]
@@ -292,8 +280,25 @@
(sql.qp/adjust-day-of-week driver [:day_of_week expr]))
(defmethod sql.qp/unix-timestamp->honeysql [:athena :seconds]
- [_driver _seconds-or-milliseconds expr]
- [:from_unixtime expr])
+ [_driver _precision expr]
+ (-> [:from_unixtime expr]
+ (h2x/with-database-type-info "timestamp")))
+
+(defn- from-unixtime-nanos [expr]
+ (-> [:from_unixtime_nanos expr]
+ (h2x/with-database-type-info "timestamp")))
+
+(defmethod sql.qp/unix-timestamp->honeysql [:athena :milliseconds]
+ [_driver _precision expr]
+ (from-unixtime-nanos (h2x/* expr 1000000)))
+
+(defmethod sql.qp/unix-timestamp->honeysql [:athena :microseconds]
+ [_driver _precision expr]
+ (from-unixtime-nanos (h2x/* expr 1000)))
+
+(defmethod sql.qp/unix-timestamp->honeysql [:athena :nanoseconds]
+ [_driver _precision expr]
+ (from-unixtime-nanos expr))
(defmethod sql.qp/add-interval-honeysql-form :athena
[_driver hsql-form amount unit]
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
index 50fd8bec2d4..fa43b510918 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
@@ -317,10 +317,10 @@
[:field "x" {:base-type :type/DateTime, :temporal-unit :day-of-week}] nil
(meta/field-metadata :checkins :date) :date)))
-(deftest reconcile-temporal-types-test
+(deftest ^:parallel reconcile-temporal-types-test
(doseq [test-case (bigquery.qp.reconciliation-tu/test-cases)]
- (testing (str \newline (u/pprint-to-str (list `bigquery.qp.reconciliation-tu/test-temporal-type-reconciliation! test-case)))
- (bigquery.qp.reconciliation-tu/test-temporal-type-reconciliation! test-case))))
+ (testing (str \newline (u/pprint-to-str (list `bigquery.qp.reconciliation-tu/test-temporal-type-reconciliation test-case)))
+ (bigquery.qp.reconciliation-tu/test-temporal-type-reconciliation test-case))))
(deftest reconcile-temporal-types-date-extraction-filters-test
(mt/with-report-timezone-id! nil
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj
index c732fd44022..8250b3534f6 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj
@@ -9,6 +9,7 @@
[metabase.lib.test-util :as lib.tu]
[metabase.lib.util.match :as lib.util.match]
[metabase.query-processor.store :as qp.store]
+ [metabase.query-processor.timezone :as qp.timezone]
[metabase.query-processor.util.add-alias-info :as add]
[metabase.test :as mt]
[metabase.util.honey-sql-2 :as h2x]))
@@ -48,11 +49,15 @@
:database-type "integer"})]
(lib.tu/mock-metadata-provider
meta/metadata-provider
- {:fields [date-field
- datetime-field
- timestamp-field
- unix-seconds-field
- unix-millis-field]})))
+ {:database (assoc meta/database
+ :engine :bigquery-cloud-sdk
+ :details {:project-id-from-credentials "mock-project"
+ :service-account-json "{}"})
+ :fields [date-field
+ datetime-field
+ timestamp-field
+ unix-seconds-field
+ unix-millis-field]})))
(def mock-temporal-fields
{:date (lib.metadata/field mock-temporal-fields-metadata-provider 1)
@@ -77,16 +82,16 @@
:type :timestamp
:as {:date (t/local-date "2019-12-10")
:datetime (t/local-date-time "2019-12-10T14:47:00")}}
- :unix-timestamp-seconds {:value [:field 4 nil]
+ :unix-timestamp-seconds {:value [:field 4 {::add/source-table (meta/id :checkins)}]
:type :timestamp
- :as (let [expected (-> [:timestamp_seconds (h2x/identifier :field "PUBLIC" "CHECKINS" "unix_seconds")]
+ :as (let [expected (-> [:timestamp_seconds (h2x/identifier :field "PUBLIC.CHECKINS" "unix_seconds")]
(h2x/with-database-type-info "timestamp"))]
{:date [:date expected]
:datetime [:datetime expected]
:timestamp expected})}
- :unix-timestamp-milliseconds {:value [:field 5 nil]
+ :unix-timestamp-milliseconds {:value [:field 5 {::add/source-table (meta/id :checkins)}]
:type :timestamp
- :as (let [expected (-> [:timestamp_millis (h2x/identifier :field "PUBLIC" "CHECKINS" "unix_milliseconds")]
+ :as (let [expected (-> [:timestamp_millis (h2x/identifier :field "PUBLIC.CHECKINS" "unix_milliseconds")]
(h2x/with-database-type-info "timestamp"))]
{:date [:date expected]
:datetime [:datetime expected]
@@ -181,12 +186,12 @@
(repeat (dec num-args) filter-value))]
(sql.qp/->honeysql :bigquery-cloud-sdk filter-clause)))
-(defn test-temporal-type-reconciliation!
- [test-case]
+(defn test-temporal-type-reconciliation [test-case]
(mt/test-driver :bigquery-cloud-sdk
(qp.store/with-metadata-provider mock-temporal-fields-metadata-provider
- (mt/with-report-timezone-id! nil
- (binding [*print-meta* true]
- (when-let [test-case (expand-test-case test-case)]
- (is (= (temporal-type-reconciliation-expected-value test-case)
- (temporal-type-reconciliation-actual-value test-case)))))))))
+ ;; just need some sort of invalid timezone ID to override everything else
+ (binding [qp.timezone/*report-timezone-id-override* ::nil
+ *print-meta* true]
+ (when-let [test-case (expand-test-case test-case)]
+ (is (= (temporal-type-reconciliation-expected-value test-case)
+ (temporal-type-reconciliation-actual-value test-case))))))))
diff --git a/modules/drivers/hive-like/src/metabase/driver/hive_like.clj b/modules/drivers/hive-like/src/metabase/driver/hive_like.clj
index 1a0c0ef4e2a..c18c395b78a 100644
--- a/modules/drivers/hive-like/src/metabase/driver/hive_like.clj
+++ b/modules/drivers/hive-like/src/metabase/driver/hive_like.clj
@@ -82,6 +82,11 @@
[_ _ expr]
(h2x/->timestamp [:from_unixtime expr]))
+(defmethod sql.qp/unix-timestamp->honeysql [:hive-like :milliseconds]
+ [_ _ expr]
+ (-> [:timestamp_millis expr]
+ (h2x/with-database-type-info "timestamp")))
+
(defn- date-format [format-str expr]
[:date_format expr (h2x/literal format-str)])
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index d9f4bb603e3..4f0cafdb430 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -241,8 +241,10 @@ function(bin) {
")
(defmethod ->rvalue :metadata/column
- [{coercion :coercion-strategy, ::keys [source-alias join-field] :as field}]
- (let [field-name (str \$ (scope-with-join-field (field->name field) join-field source-alias))]
+ [{coercion :coercion-strategy, ::keys [source-alias join-field inherited?] :as field}]
+ (let [field-name (str \$ (scope-with-join-field (field->name field) join-field source-alias))
+ coercion (when-not inherited?
+ coercion)]
(cond
(isa? coercion :Coercion/UNIXNanoSeconds->DateTime)
{:$dateFromParts {:millisecond {$divide [field-name 1000000]}, :year 1970, :timezone "UTC"}}
@@ -438,14 +440,15 @@ function(bin) {
(defmethod ->rvalue :field
[[_ id-or-name {:keys [temporal-unit join-alias] :as opts} :as field]]
- (let [join-field (get-join-alias join-alias)
+ (let [join-field (get-join-alias join-alias)
source-alias (driver-api/qp.add.source-alias opts)]
(cond-> (if (integer? id-or-name)
(if-let [mapped (find-mapped-field-name field)]
(str \$ mapped)
(->rvalue (assoc (driver-api/field (driver-api/metadata-provider) id-or-name)
::source-alias source-alias
- ::join-field join-field)))
+ ::join-field join-field
+ ::inherited? (not (pos-int? (driver-api/qp.add.source-table opts))))))
(if-let [mapped (find-mapped-field-name field)]
(str \$ mapped)
(str \$ (scope-with-join-field (name id-or-name) join-field source-alias))))
diff --git a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj b/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
index 77e4390d7b4..f661fd5023d 100644
--- a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
+++ b/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
@@ -39,19 +39,19 @@
(let [parent-method (get-method sql.qp/->honeysql [:hive-like :field])
field-clause (driver-api/update-field-options
field-clause
- update
- driver-api/qp.add.source-table
- (fn [source-table]
- (cond
- ;; DO NOT qualify fields from field filters with `t1`, that won't
- ;; work unless the user-written SQL query is doing the same
- ;; thing.
- compiling-field-filter? driver-api/qp.add.none
- ;; for all other fields from the source table qualify them with
- ;; `t1`
- (integer? source-table) source-table-alias
- ;; no changes for anyone else.
- :else source-table)))]
+ (fn [{source-table driver-api/qp.add.source-table, :as options}]
+ (-> options
+ (cond-> (pos-int? source-table) (assoc :qp/allow-coercion-for-columns-without-integer-qp.add.source-table true))
+ (assoc driver-api/qp.add.source-table (cond
+ ;; DO NOT qualify fields from field filters with `t1`, that won't
+ ;; work unless the user-written SQL query is doing the same
+ ;; thing.
+ compiling-field-filter? driver-api/qp.add.none
+ ;; for all other fields from the source table qualify them with
+ ;; `t1`
+ (pos-int? source-table) source-table-alias
+ ;; no changes for anyone else.
+ :else source-table)))))]
(parent-method driver field-clause)))
(defn- format-over
diff --git a/modules/drivers/sparksql/test/metabase/test/data/sparksql.clj b/modules/drivers/sparksql/test/metabase/test/data/sparksql.clj
index a9cc665fd46..71f47c4e666 100644
--- a/modules/drivers/sparksql/test/metabase/test/data/sparksql.clj
+++ b/modules/drivers/sparksql/test/metabase/test/data/sparksql.clj
@@ -25,14 +25,19 @@
[_driver _feature _database]
false))
-(doseq [[base-type database-type] {:type/BigInteger "BIGINT"
- :type/Boolean "BOOLEAN"
- :type/Date "DATE"
- :type/DateTime "TIMESTAMP"
- :type/Decimal "DECIMAL"
- :type/Float "DOUBLE"
- :type/Integer "INTEGER"
- :type/Text "STRING"}]
+(doseq [[base-type database-type] {:type/BigInteger "BIGINT"
+ :type/Boolean "BOOLEAN"
+ :type/Date "DATE"
+ ;; TODO (Cam 9/25/25) -- in Spark SQL 3.4+ (which we don't test against yet)
+ ;; there's the new `TIMESTAMP_NTZ` type that stores timestamps as wall-clock time
+ ;; without timezone adjustment.
+ :type/DateTime "TIMESTAMP"
+ ;; stores timestamps as UTC but displays them in session timezone
+ :type/DateTimeWithTZ "TIMESTAMP"
+ :type/Decimal "DECIMAL"
+ :type/Float "DOUBLE"
+ :type/Integer "INTEGER"
+ :type/Text "STRING"}]
(defmethod sql.tx/field-base-type->sql-type [:sparksql base-type] [_ _] database-type))
;; If someone tries to run Time column tests with SparkSQL give them a heads up that SparkSQL does not support it
diff --git a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj b/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
index 0acd069ea21..a55a9f67b7b 100644
--- a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
+++ b/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
@@ -2,6 +2,7 @@
(:refer-clojure :exclude [mapv])
(:require
[clojure.java.io :as io]
+ [clojure.math :as math]
[clojure.set :as set]
[clojure.string :as str]
[java-time.api :as t]
@@ -12,8 +13,7 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
- [metabase.driver.sql.parameters.substitution
- :as sql.params.substitution]
+ [metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
@@ -263,19 +263,33 @@
(->datetime hsql-form (h2x/literal (format "%+d %s" (* amount multiplier) sqlite-unit)))))
(defmethod sql.qp/unix-timestamp->honeysql [:sqlite :seconds]
- [_ _ expr]
+ [_driver _precision expr]
(->datetime expr (h2x/literal "unixepoch")))
+(defn- unix-timestamp->honeysql [expr power]
+ (let [divisor (long (math/pow 10 power))
+ format-string (format "%%0%dd" power)]
+ [:concat
+ (->datetime (h2x// expr divisor) (h2x/literal "unixepoch"))
+ (h2x/literal ".")
+ [:printf
+ (h2x/literal format-string)
+ (h2x/mod expr divisor)]]))
+
+(defmethod sql.qp/unix-timestamp->honeysql [:sqlite :milliseconds] [_driver _precision expr] (unix-timestamp->honeysql expr 3))
+(defmethod sql.qp/unix-timestamp->honeysql [:sqlite :microseconds] [_driver _precision expr] (unix-timestamp->honeysql expr 6))
+(defmethod sql.qp/unix-timestamp->honeysql [:sqlite :nanoseconds] [_driver _precision expr] (unix-timestamp->honeysql expr 9))
+
(defmethod sql.qp/cast-temporal-string [:sqlite :Coercion/ISO8601->DateTime]
- [_driver _semantic_type expr]
+ [_driver _semantic-type expr]
(->datetime expr))
(defmethod sql.qp/cast-temporal-string [:sqlite :Coercion/ISO8601->Date]
- [_driver _semantic_type expr]
+ [_driver _semantic-type expr]
(->date expr))
(defmethod sql.qp/cast-temporal-string [:sqlite :Coercion/ISO8601->Time]
- [_driver _semantic_type expr]
+ [_driver _semantic-type expr]
(->time expr))
(defmethod sql.qp/cast-temporal-string [:sqlite :Coercion/YYYYMMDDHHMMSSString->Temporal]
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index aa50db2166b..ca8d7b07ff4 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -18,8 +18,7 @@
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
- [metabase.driver.sql.parameters.substitution
- :as sql.params.substitution]
+ [metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison]
[metabase.driver.sql.util :as sql.u]
@@ -29,19 +28,8 @@
[metabase.util.log :as log]
[metabase.util.performance :as perf :refer [mapv]])
(:import
- (java.sql
- Connection
- DatabaseMetaData
- PreparedStatement
- ResultSet
- Time)
- (java.time
- LocalDate
- LocalDateTime
- LocalTime
- OffsetDateTime
- OffsetTime
- ZonedDateTime)
+ (java.sql Connection DatabaseMetaData PreparedStatement ResultSet Time)
+ (java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
(java.time.format DateTimeFormatter)
(java.util UUID)
(net.sf.jsqlparser.schema Table)
@@ -348,12 +336,60 @@
[_ hsql-form amount unit]
(date-add unit amount hsql-form))
+;; The second argument to DATEADD() gets casted to a 32-bit integer. BIGINT is 64 bites, so we tend to run into
+;; integer overflow errors (especially for millisecond timestamps).
+;; Work around this by converting the timestamps to minutes instead before calling DATEADD().
+;;
+;; NOTE: In SQL Server 2025 (17.x) + this is no longer
+;; true (https://learn.microsoft.com/en-us/sql/t-sql/functions/dateadd-transact-sql?view=sql-server-ver17):
+;;
+;; > In SQL Server 2025 (17.x) Preview, Azure SQL Database, Azure SQL Managed Instance and SQL database in Microsoft
+;; > Fabric Preview, number can be expressed as a bigint. This feature is in preview.
+
+(defn- supports-bigint-date-add? []
+ (some-> (driver-api/metadata-provider)
+ driver-api/database
+ :dbms-version
+ :semantic-version
+ first
+ (>= 17)))
+
+(def ^:private nineteen-seventy (h2x/cast :datetime2 (h2x/literal "1970-01-01 00:00:00.000")))
+
+(defn- unix-timestamp->honeysql [expr unit num-per-minute]
+ (if (supports-bigint-date-add?)
+ (date-add unit expr nineteen-seventy)
+ (->> nineteen-seventy
+ (date-add :minute (h2x// expr num-per-minute))
+ (date-add unit (h2x/mod expr num-per-minute)))))
+
(defmethod sql.qp/unix-timestamp->honeysql [:sqlserver :seconds]
- [_ _ expr]
- ;; The second argument to DATEADD() gets casted to a 32-bit integer. BIGINT is 64 bites, so we tend to run into
- ;; integer overflow errors (especially for millisecond timestamps).
- ;; Work around this by converting the timestamps to minutes instead before calling DATEADD().
- (date-add :minute (h2x// expr 60) (h2x/literal "1970-01-01")))
+ [_driver _precision expr]
+ (unix-timestamp->honeysql expr :second 60))
+
+(defmethod sql.qp/unix-timestamp->honeysql [:sqlserver :milliseconds]
+ [_driver _precision expr]
+ (unix-timestamp->honeysql expr :millisecond (* 60 1000)))
+
+(defmethod sql.qp/unix-timestamp->honeysql [:sqlserver :microseconds]
+ [_driver _precision expr]
+ (unix-timestamp->honeysql expr :microsecond (* 60 1000 1000)))
+
+;;; 60 ✕ 10^9 is still larger than `Integer/MAX_VALUE` (i.e., the max value for a 32-bit integer) so it STILL results
+;;; in integer overflows even if we modulo by the number per minute... we actually have to split this up into three
+;;; parts to avoid the overflow.
+(defmethod sql.qp/unix-timestamp->honeysql [:sqlserver :nanoseconds]
+ [_driver _precision expr]
+ (if (supports-bigint-date-add?)
+ (date-add :nanosecond expr nineteen-seventy)
+ (let [num-per-minute (* 60 1000 1000 1000)
+ num-per-second (* 1000 1000 1000)]
+ (->> nineteen-seventy
+ (date-add :minute (h2x// expr num-per-minute))
+ (date-add :second (-> expr
+ (h2x/mod num-per-minute)
+ (h2x// num-per-second)))
+ (date-add :nanosecond (h2x/mod expr num-per-second))))))
(defmethod sql.qp/float-dbtype :sqlserver
[_]
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 9d405beff0c..322d5b8d900 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -522,15 +522,15 @@
(defmethod unix-timestamp->honeysql [:sql :milliseconds]
[driver _ expr]
- (unix-timestamp->honeysql driver :seconds (h2x// expr 1000)))
+ (unix-timestamp->honeysql driver :seconds (h2x// expr 1000.0)))
(defmethod unix-timestamp->honeysql [:sql :microseconds]
[driver _ expr]
- (unix-timestamp->honeysql driver :seconds (h2x// expr 1000000)))
+ (unix-timestamp->honeysql driver :seconds (h2x// expr 1000000.0)))
(defmethod unix-timestamp->honeysql [:sql :nanoseconds]
[driver _ expr]
- (unix-timestamp->honeysql driver :seconds (h2x// expr 1000000000)))
+ (unix-timestamp->honeysql driver :seconds (h2x// expr 1000000000.0)))
(defmulti cast-temporal-byte
"Cast a byte field"
@@ -887,6 +887,8 @@
field-metadata (when (integer? id-or-name)
(driver-api/field (driver-api/metadata-provider) id-or-name))
allow-casting? (and field-metadata
+ (or (pos-int? (driver-api/qp.add.source-table options))
+ (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table options))
(not (:qp/ignore-coercion options)))
;; preserve metadata attached to the original field clause, for example BigQuery temporal type information.
identifier (-> (apply h2x/identifier :field
diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.clj
index 3810b2accd4..8706e581bd1 100644
--- a/test/metabase/driver/sql/query_processor_test.clj
+++ b/test/metabase/driver/sql/query_processor_test.clj
@@ -1471,7 +1471,7 @@
(update :query #(str/split-lines (driver/prettify-native-form :h2 %))))))))))
(deftest ^:parallel no-double-coercion-when-joining-coerced-fields-test
- (testing "Should generate correct SQL when joining a field that has coercion applied"
+ (testing "Should generate correct SQL when joining a field that has coercion applied (#62099)"
(let [mp (lib.tu/merged-mock-metadata-provider
meta/metadata-provider
{:fields [(merge (meta/field-metadata :products :id)
diff --git a/test/metabase/query_processor_test/alternative_date_test.clj b/test/metabase/query_processor_test/alternative_date_test.clj
index 153a4751c1a..abc366328f6 100644
--- a/test/metabase/query_processor_test/alternative_date_test.clj
+++ b/test/metabase/query_processor_test/alternative_date_test.clj
@@ -69,8 +69,8 @@
(defmethod microseconds-test-expected-rows :sqlite
[_driver]
- [[1 4 "2015-06-06 10:40:00"]
- [2 0 "2015-06-10 19:51:00"]])
+ [[1 4 "2015-06-06 10:40:00.000000"]
+ [2 0 "2015-06-10 19:51:00.000000"]])
(deftest ^:parallel microseconds-test
(mt/test-drivers (mt/normal-drivers)
diff --git a/test/metabase/query_processor_test/cast_test.clj b/test/metabase/query_processor_test/cast_test.clj
index f698d149edb..96e3a3914b4 100644
--- a/test/metabase/query_processor_test/cast_test.clj
+++ b/test/metabase/query_processor_test/cast_test.clj
@@ -1003,22 +1003,42 @@
(defmulti datetime-number-cast-expected
"Expected datetime string for [[datetime-number-cast]] test."
- {:arglists '([driver])}
- tx/dispatch-on-driver-with-test-extensions
+ {:arglists '([driver] [driver mode])}
+ (fn
+ ([driver]
+ (tx/dispatch-on-driver-with-test-extensions driver))
+ ([driver mode]
+ [(tx/dispatch-on-driver-with-test-extensions driver)
+ mode]))
:hierarchy #'driver/hierarchy)
(defmethod datetime-number-cast-expected :default
- [_]
+ [_driver]
"2025-07-02T18:33:35Z")
-(defmethod datetime-number-cast-expected :sqlite
- [_]
+(defmethod datetime-number-cast-expected [:sqlite :unix-seconds]
+ [_driver _mode]
"2025-07-02 18:33:35")
-;; sqlserver's sql.qp/unix-timestamp->honeysql truncates to minutes
-(defmethod datetime-number-cast-expected :sqlserver
- [_]
- "2025-07-02T18:33:00Z")
+(defmethod datetime-number-cast-expected [:sqlite :unix-milliseconds]
+ [_driver _mode]
+ "2025-07-02 18:33:35.000")
+
+(defmethod datetime-number-cast-expected [:sqlite :unix-microseconds]
+ [_driver _mode]
+ "2025-07-02 18:33:35.000000")
+
+(defmethod datetime-number-cast-expected [:sqlite :unix-nanoseconds]
+ [_driver _mode]
+ "2025-07-02 18:33:35.000000000")
+
+(defn- datetime-number-cast-expected*
+ [driver mode]
+ (let [default-method (get-method datetime-number-cast-expected :default)
+ driver (tx/dispatch-on-driver-with-test-extensions driver)]
+ (if (= (get-method datetime-number-cast-expected [driver mode]) default-method)
+ (datetime-number-cast-expected driver)
+ (datetime-number-cast-expected driver mode))))
(deftest ^:parallel datetime-number-cast
(let [seconds-timestamp 1751481215]
@@ -1043,8 +1063,9 @@
(lib/limit 1))
result (-> query qp/process-query)
rows (mt/rows result)]
- (is (= (datetime-number-cast-expected driver/*driver*)
- (-> rows first (get 2))))))))))
+ (mt/with-native-query-testing-context query
+ (is (= (datetime-number-cast-expected* driver/*driver* mode)
+ (-> rows first (get 2)))))))))))
;; today()
(deftest ^:parallel today-test
diff --git a/test/metabase/query_processor_test/coercion_test.clj b/test/metabase/query_processor_test/coercion_test.clj
index e80c32bf1d1..eb12a845036 100644
--- a/test/metabase/query_processor_test/coercion_test.clj
+++ b/test/metabase/query_processor_test/coercion_test.clj
@@ -4,14 +4,17 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-util :as lib.tu]
+ [metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.query-processor :as qp]
[metabase.query-processor.store :as qp.store]
+ [metabase.query-processor.test-util :as qp.test-util]
[metabase.test :as mt]
- [metabase.types.core :as types]))
+ [metabase.types.core :as types]
+ [metabase.util.date-2 :as u.date]))
(set! *warn-on-reflection* true)
-(deftest string-to-float-coercion-test
+(deftest ^:parallel string-to-float-coercion-test
(mt/test-drivers
(mt/normal-drivers)
(mt/dataset
@@ -29,7 +32,7 @@
(mt/formatted-rows [3.0])
ffirst)))))))))
-(deftest string-to-integer-coercion-test
+(deftest ^:parallel string-to-integer-coercion-test
(mt/test-drivers
(mt/normal-drivers)
(mt/dataset
@@ -49,7 +52,7 @@
(is (= res
(biginteger coerced-number))))))))))
-(deftest float-to-integer-coercion-test
+(deftest ^:parallel float-to-integer-coercion-test
(mt/test-drivers
(mt/normal-drivers)
(mt/dataset
@@ -91,7 +94,7 @@
(catch Exception _
(-> s java.time.LocalDate/parse))))
-(deftest datetime-to-date-coercion-test
+(deftest ^:parallel datetime-to-date-coercion-test
(mt/test-drivers (mt/normal-drivers)
(doseq [[human-col table col] [["orders created_at (timestamptz)" :orders :created_at]
["users last_login (timestamp)" :users :last_login]]]
@@ -111,3 +114,57 @@
(is (date-type? col))
(doseq [[date-col] rows]
(is (parse-date date-col))))))))
+
+(deftest ^:parallel coerced-custom-column-from-join-test
+ (testing "Should be able to used a coerced column from a join in expressions (#63890)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join :expressions)
+ (let [mp (-> (mt/metadata-provider)
+ (lib.tu/merged-mock-metadata-provider
+ {:fields [{:id (mt/id :orders :quantity)
+ :coercion-strategy :Coercion/UNIXMilliSeconds->DateTime
+ :effective-type :type/DateTime}]})
+ qp.test-util/mock-fks-application-database-metadata-provider)
+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :products)))
+ (lib/join (-> (lib/join-clause (lib.metadata/table mp (mt/id :orders)))
+ (lib/with-join-alias "Orders")
+ (lib/with-join-fields :none)))
+ (as-> $query (lib/expression $query "test" (lib.tu.notebook/find-col-with-spec
+ $query
+ (lib/visible-columns $query)
+ {:display-name #".*Orders"}
+ {:display-name "Quantity"})))
+ (as-> $query (lib/with-fields $query [(lib.tu.notebook/find-col-with-spec
+ $query
+ (lib/visible-columns $query)
+ {:display-name #".*Products"}
+ {:display-name "ID"})
+ (lib.tu.notebook/find-col-with-spec
+ $query
+ (lib/visible-columns $query)
+ {:display-name #".*Orders"}
+ {:display-name "ID"})
+ (lib.tu.notebook/find-col-with-spec
+ $query
+ (lib/visible-columns $query)
+ {}
+ {:display-name "test"})]))
+ (as-> $query (lib/order-by $query (lib.tu.notebook/find-col-with-spec
+ $query
+ (lib/orderable-columns $query)
+ {:display-name #".*Products"}
+ {:display-name "ID"})))
+ (as-> $query (lib/order-by $query (lib.tu.notebook/find-col-with-spec
+ $query
+ (lib/orderable-columns $query)
+ {:display-name #".*Orders"}
+ {:display-name "ID"})))
+ (lib/limit 3))]
+ (mt/with-native-query-testing-context query
+ (is (= [[1 448 "1970-01-01T00:00:00.002Z"]
+ [1 493 "1970-01-01T00:00:00.001Z"]
+ [1 607 "1970-01-01T00:00:00.007Z"]]
+ (mt/formatted-rows [int int (fn [s]
+ (if (string? s)
+ (u.date/format "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" (u.date/parse s))
+ s))]
+ (qp/process-query query)))))))))
diff --git a/test/metabase/query_processor_test/date_bucketing_test.clj b/test/metabase/query_processor_test/date_bucketing_test.clj
index fe8e0711f8d..faffab44c40 100644
--- a/test/metabase/query_processor_test/date_bucketing_test.clj
+++ b/test/metabase/query_processor_test/date_bucketing_test.clj
@@ -99,9 +99,23 @@
[id s cnt]))))
(defmethod sanity-check-test-expected-rows :sqlite
- [_driver _timezone]
- (for [[id s cnt] sanity-check-test-utc-results]
- [id (u.date/format-sql (t/local-date-time (u.date/parse s))) cnt]))
+ [_driver timezone]
+ (case timezone
+ :pacific [[1 "2015-06-06 10:40:00.000" 4]
+ [2 "2015-06-10 19:51:00.000" 0]
+ [3 "2015-06-09 15:42:00.000" 5]
+ [4 "2015-06-22 23:49:00.000" 3]
+ [5 "2015-06-20 01:45:00.000" 3]]
+ :utc [[1 "2015-06-06 10:40:00.000" 4]
+ [2 "2015-06-10 19:51:00.000" 0]
+ [3 "2015-06-09 15:42:00.000" 5]
+ [4 "2015-06-22 23:49:00.000" 3]
+ [5 "2015-06-20 01:45:00.000" 3]]
+ :eastern [[1 "2015-06-06 10:40:00.000" 4]
+ [2 "2015-06-10 19:51:00.000" 0]
+ [3 "2015-06-09 15:42:00.000" 5]
+ [4 "2015-06-22 23:49:00.000" 3]
+ [5 "2015-06-20 01:45:00.000" 3]]))
(deftest sanity-check-test
(mt/test-drivers (mt/normal-drivers-with-feature ::sanity-check-test)
@@ -240,16 +254,16 @@
;;; return them directly. This is less than ideal. TIMEZONE FIXME
(defmethod group-by-default-test-expected-rows :sqlite
[_driver]
- [["2015-06-01 10:31:00" 1]
- ["2015-06-01 16:06:00" 1]
- ["2015-06-01 17:23:00" 1]
- ["2015-06-01 18:55:00" 1]
- ["2015-06-01 21:04:00" 1]
- ["2015-06-01 21:19:00" 1]
- ["2015-06-02 02:13:00" 1]
- ["2015-06-02 05:37:00" 1]
- ["2015-06-02 08:20:00" 1]
- ["2015-06-02 11:11:00" 1]])
+ [["2015-06-01 10:31:00.000" 1]
+ ["2015-06-01 16:06:00.000" 1]
+ ["2015-06-01 17:23:00.000" 1]
+ ["2015-06-01 18:55:00.000" 1]
+ ["2015-06-01 21:04:00.000" 1]
+ ["2015-06-01 21:19:00.000" 1]
+ ["2015-06-02 02:13:00.000" 1]
+ ["2015-06-02 05:37:00.000" 1]
+ ["2015-06-02 08:20:00.000" 1]
+ ["2015-06-02 11:11:00.000" 1]])
(deftest group-by-default-test
(mt/test-drivers (mt/normal-drivers)
@@ -307,16 +321,16 @@
;;; Always in UTC so isn't impacted by changes in report-timezone
(defmethod group-by-default-test-2-expected-rows :sqlite
[_driver]
- [["2015-06-01 10:31:00" 1]
- ["2015-06-01 16:06:00" 1]
- ["2015-06-01 17:23:00" 1]
- ["2015-06-01 18:55:00" 1]
- ["2015-06-01 21:04:00" 1]
- ["2015-06-01 21:19:00" 1]
- ["2015-06-02 02:13:00" 1]
- ["2015-06-02 05:37:00" 1]
- ["2015-06-02 08:20:00" 1]
- ["2015-06-02 11:11:00" 1]])
+ [["2015-06-01 10:31:00.000" 1]
+ ["2015-06-01 16:06:00.000" 1]
+ ["2015-06-01 17:23:00.000" 1]
+ ["2015-06-01 18:55:00.000" 1]
+ ["2015-06-01 21:04:00.000" 1]
+ ["2015-06-01 21:19:00.000" 1]
+ ["2015-06-02 02:13:00.000" 1]
+ ["2015-06-02 05:37:00.000" 1]
+ ["2015-06-02 08:20:00.000" 1]
+ ["2015-06-02 11:11:00.000" 1]])
(deftest group-by-default-test-2
(mt/test-drivers (mt/normal-drivers)
@@ -345,7 +359,16 @@
(defmethod group-by-default-test-3-expected-rows :sqlite
[_driver]
- (sad-toucan-result (default-timezone-parse-fn :utc) (comp u.date/format-sql t/local-date-time)))
+ [["2015-06-01 10:31:00.000" 1]
+ ["2015-06-01 16:06:00.000" 1]
+ ["2015-06-01 17:23:00.000" 1]
+ ["2015-06-01 18:55:00.000" 1]
+ ["2015-06-01 21:04:00.000" 1]
+ ["2015-06-01 21:19:00.000" 1]
+ ["2015-06-02 02:13:00.000" 1]
+ ["2015-06-02 05:37:00.000" 1]
+ ["2015-06-02 08:20:00.000" 1]
+ ["2015-06-02 11:11:00.000" 1]])
(deftest group-by-default-test-3
;; Changes the JVM timezone from UTC to Pacific, this test isn't run on H2 as the database stores it's timezones in
diff --git a/test/metabase/query_processor_test/expressions_test.clj b/test/metabase/query_processor_test/expressions_test.clj
index c9a7e3d9682..200bcf3eb2a 100644
--- a/test/metabase/query_processor_test/expressions_test.clj
+++ b/test/metabase/query_processor_test/expressions_test.clj
@@ -7,7 +7,6 @@
[metabase.driver :as driver]
[metabase.query-processor :as qp]
[metabase.test :as mt]
- [metabase.test.data.interface :as tx]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[toucan2.core :as t2]))
@@ -1186,11 +1185,3 @@
(mt/formatted-rows
[u.date/temporal-str->iso8601-str int int]
(qp/process-query query))))))))))
-
-(deftest ^:parallel null-array-test
- (testing "a null array should be handled gracefully and return nil"
- (mt/test-drivers (mt/normal-drivers-with-feature :test/null-arrays)
- (is (= [[nil]]
- (-> (mt/native-query {:query (tx/native-null-array-query driver/*driver*)})
- mt/process-query
- mt/rows))))))
diff --git a/test/metabase/query_processor_test/null_array_test.clj b/test/metabase/query_processor_test/null_array_test.clj
new file mode 100644
index 00000000000..05f18129b90
--- /dev/null
+++ b/test/metabase/query_processor_test/null_array_test.clj
@@ -0,0 +1,14 @@
+(ns ^:mb/driver-tests metabase.query-processor-test.null-array-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.driver :as driver]
+ [metabase.test :as mt]
+ [metabase.test.data.interface :as tx]))
+
+(deftest ^:parallel null-array-test
+ (testing "a null array should be handled gracefully and return nil"
+ (mt/test-drivers (mt/normal-drivers-with-feature :test/null-arrays)
+ (is (= [[nil]]
+ (-> (mt/native-query {:query (tx/native-null-array-query driver/*driver*)})
+ mt/process-query
+ mt/rows))))))