Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix bigquery timestamp and timezone casting in aggregation (#67089)

Riley Thompsoncde52a68ec03
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index cd7ad85beec..46e13a5d974 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -267,7 +267,8 @@
(defmethod temporal-type ::h2x/identifier
[identifier]
- (:bigquery-cloud-sdk/temporal-type (meta identifier)))
+ (or (:bigquery-cloud-sdk/temporal-type (meta identifier))
+ (:bigquery-cloud-sdk/base-temporal-type (meta identifier))))
(defmethod temporal-type :absolute-datetime
[[_ t _]]
@@ -674,6 +675,12 @@
[_driver [_ nfc-path]]
nfc-path)
+(defn- with-base-temporal-type
+ [[_ _id-or-name {:keys [base-type]} :as clause]]
+ (if (not (instance? clojure.lang.IObj clause))
+ clause
+ (vary-meta clause assoc :bigquery-cloud-sdk/base-temporal-type (base-type->temporal-type base-type))))
+
(defmethod sql.qp/->honeysql [:bigquery-cloud-sdk :field]
[driver [_field id-or-name opts :as field-clause]]
(let [source-table (get opts driver-api/qp.add.source-table)
@@ -685,7 +692,9 @@
;; attach temporal type info to the field clause, this will get attached to the resulting [[h2x/identifier]] by
;; SQL QP parent method, and we can access that inside other things like [[sql.qp/date]] implementations which it
;; may call in turn.
- (let [field-clause (with-temporal-type field-clause (temporal-type field-clause))
+ (let [field-clause (-> field-clause
+ (with-temporal-type (temporal-type field-clause))
+ with-base-temporal-type)
stored-field (when (integer? id-or-name)
(driver-api/field (driver-api/metadata-provider) id-or-name))
result (parent-method driver field-clause)
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
index fa602a62c64..dbaf2fa3e18 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
@@ -10,6 +10,7 @@
[metabase.driver.bigquery-cloud-sdk.query-processor-test.reconciliation-test-util :as bigquery.qp.reconciliation-tu]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.lib.core :as lib]
+ [metabase.lib.expression :as lib.expression]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
@@ -1293,3 +1294,63 @@
[[[:= $name "Won"] [:datetime-add $created_at 0 :month]]]
{:default [:datetime-add $birth_date 0 :month]}]}
:filter [:time-interval [:expression "asdfdsa"] -12 :month]}))))))
+
+(mt/defdataset timestamp-dataset
+ [["ts_table" [{:field-name "user_id"
+ :base-type :type/Integer}
+ {:field-name "created_at"
+ :base-type :type/DateTimeWithLocalTZ}]
+ [[1 #t "2025-03-01 23:30:00Z"]
+ [2 #t "2025-03-01 23:30:00Z"]
+ [3 #t "2025-03-01 23:30:00Z"]
+ [1 #t "2025-03-08 23:30:00Z"]
+ [2 #t "2025-03-08 23:30:00Z"]
+ [1 #t "2025-03-15 23:30:00Z"]
+ [3 #t "2025-03-15 23:30:00Z"]
+ [2 #t "2025-03-22 23:30:00Z"]
+ [3 #t "2025-03-22 23:30:00Z"]
+ [1 #t "2025-03-29 23:30:00Z"]
+ [2 #t "2025-03-29 23:30:00Z"]
+ [3 #t "2025-03-29 23:30:00Z"]]]
+ ["dt_table" [{:field-name "user_id"
+ :base-type :type/Integer}
+ {:field-name "created_at"
+ :base-type :type/DateTime}]
+ [[1 #t "2025-03-01 23:30:00"]
+ [2 #t "2025-03-01 23:30:00"]
+ [3 #t "2025-03-01 23:30:00"]
+ [1 #t "2025-03-08 23:30:00"]
+ [2 #t "2025-03-08 23:30:00"]
+ [1 #t "2025-03-15 23:30:00"]
+ [3 #t "2025-03-15 23:30:00"]
+ [2 #t "2025-03-22 23:30:00"]
+ [3 #t "2025-03-22 23:30:00"]
+ [1 #t "2025-03-29 23:30:00"]
+ [2 #t "2025-03-29 23:30:00"]
+ [3 #t "2025-03-29 23:30:00"]]]])
+
+(deftest timestamp-aggregation-with-timezone-test
+ (mt/test-driver :bigquery-cloud-sdk
+ (mt/dataset timestamp-dataset
+ (doseq [table [:ts_table :dt_table]
+ timezone ["Europe/Paris" "Database default"]]
+ (let [mp (mt/metadata-provider)
+ the-table (lib.metadata/table mp (mt/id table))
+ user-id-col (lib.metadata/field mp (mt/id table :user_id))
+ created-at-col (lib.metadata/field mp (mt/id table :created_at))
+ base-query (-> (lib/query mp the-table)
+ (lib/expression "created_at_year" (lib/get-year created-at-col)))
+ query (-> base-query
+ (lib/aggregate (lib/distinct user-id-col))
+ (lib/breakout (lib/with-temporal-bucket created-at-col :week-of-year))
+ (lib/breakout (lib.expression/expression-ref base-query 0 "created_at_year")))]
+ (mt/with-report-timezone-id! timezone
+ (let [exp-rows (cond->> [[8 2025 3]
+ [9 2025 2]
+ [10 2025 2]
+ [11 2025 2]
+ [12 2025 3]]
+ (and (= table :ts_table) (= timezone "Europe/Paris"))
+ (mapv #(update % 0 inc)))]
+ (is (= exp-rows
+ (mt/formatted-rows [int int int] (qp/process-query query)))))))))))