Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Optimize temporal expressions the same as filters (#74875)

Riley Thompson71e66269c2d7
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index 4d0a5d73bde..e9f1355ef97 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -194,7 +194,7 @@
:filter [:and
;; This still gets :default bucketing! auto-bucket-datetimes
;; puts :day bucketing on both parts of this filter, since it's
- ;; matching a YYYY-mm-dd string. Then optimize-temporal-filters
+ ;; matching a YYYY-mm-dd string. Then optimize-temporal-clauses
;; sees that the
;; :type/Date column already has :day
;; granularity, and switches both to :default
diff --git a/modules/drivers/mongo/test/metabase/test/data/mongo.clj b/modules/drivers/mongo/test/metabase/test/data/mongo.clj
index 3706fb79149..49634216f7f 100644
--- a/modules/drivers/mongo/test/metabase/test/data/mongo.clj
+++ b/modules/drivers/mongo/test/metabase/test/data/mongo.clj
@@ -19,7 +19,8 @@
(tx/add-test-extensions! :mongo)
(doseq [feature [:test/time-type
- :test/timestamptz-type]]
+ :test/timestamptz-type
+ :test/date-type]]
(defmethod driver/database-supports? [:mongo feature]
[_driver _feature _database]
false))
diff --git a/modules/drivers/oracle/test/metabase/test/data/oracle.clj b/modules/drivers/oracle/test/metabase/test/data/oracle.clj
index f5893d557f5..41bf0b5bcbe 100644
--- a/modules/drivers/oracle/test/metabase/test/data/oracle.clj
+++ b/modules/drivers/oracle/test/metabase/test/data/oracle.clj
@@ -105,6 +105,10 @@
[_driver _feature _database]
false)
+(defmethod driver/database-supports? [:oracle :test/date-type]
+ [_driver _feature _database]
+ false)
+
(doseq [[base-type sql-type] {:type/BigInteger "NUMBER(*,0)"
:type/Boolean "NUMBER(1)"
:type/Date "DATE"
diff --git a/src/metabase/query_processor/middleware/optimize_temporal_filters.clj b/src/metabase/query_processor/middleware/optimize_temporal_filters.clj
index a1b62cbd47d..6e8f1a0718d 100644
--- a/src/metabase/query_processor/middleware/optimize_temporal_filters.clj
+++ b/src/metabase/query_processor/middleware/optimize_temporal_filters.clj
@@ -1,10 +1,11 @@
(ns metabase.query-processor.middleware.optimize-temporal-filters
"Middleware that optimizes equality filter clauses against bucketed temporal fields. See docstring for
- `optimize-temporal-filters` for more details."
- (:refer-clojure :exclude [get-in])
+ `optimize-temporal-clauses` for more details."
+ (:refer-clojure :exclude [get-in mapv])
(:require
[better-cond.core :as b]
[metabase.lib.core :as lib]
+ [metabase.lib.options :as lib.options]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.expression :as lib.schema.expression]
@@ -16,7 +17,7 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.match :as match]
- [metabase.util.performance :refer [get-in]]))
+ [metabase.util.performance :refer [get-in mapv]]))
(def ^:private optimizable-units
#{:second :minute :hour :day :week :month :quarter :year})
@@ -36,7 +37,7 @@
(or (= unit :default)
(contains? optimizable-units unit))))))
-(defmulti ^:private can-optimize-filter?
+(defmulti ^:private can-optimize-clause?
{:arglists '([mbql-clause])}
lib/dispatch-value)
@@ -54,7 +55,7 @@
(defn- field-and-temporal-value-have-compatible-units?
"Do datetime `field` clause and `temporal-value` clause have 'compatible' units that mean we'll be able to optimize
- the filter clause they're in?"
+ the clause they're in?"
[field temporal-value]
(match/match-one temporal-value
[:relative-datetime _opts #{0 :current}]
@@ -68,9 +69,9 @@
(= value-unit :default) (contains? optimizable-units field-unit)
:else (= field-unit value-unit)))))
-(defmethod can-optimize-filter? :default
- [filter-clause]
- (match/match-one filter-clause
+(defmethod can-optimize-clause? :default
+ [clause]
+ (match/match-one clause
[_tag
_opts
(field :guard optimizable-expr?)
@@ -82,15 +83,15 @@
(and (vector? clause)
(not= :default (get-in clause [2 :temporal-unit]))))
-;; TODO: I believe we do not generate __filter clauses that have default temporal bucket on column arg which should be
-;; optimized__. Unfortunately I'm not certain about that. If I was, the following `can-optimize-filter? :>=` and
-;; `can-optimize-filter? :>=` definitions would be redundant after update of `optimizable-expr?`, ie. changing
+;; TODO: I believe we do not generate __clauses that have default temporal bucket on column arg which should be
+;; optimized__. Unfortunately I'm not certain about that. If I was, the following `can-optimize-clause? :>=` and
+;; `can-optimize-clause? :>=` definitions would be redundant after update of `optimizable-expr?`, ie. changing
;; the logic to something along "if `expr` has default temporal unit we should not optimize".
-(defmethod can-optimize-filter? :>=
- [filter-clause]
+(defmethod can-optimize-clause? :>=
+ [clause]
(match/match-one
- filter-clause
+ clause
[_tag
_opts
;; Don't optimize >= with column that has default temporal bucket
@@ -98,9 +99,9 @@
(temporal-value :guard optimizable-temporal-value?)]
(field-and-temporal-value-have-compatible-units? field temporal-value)))
-(defmethod can-optimize-filter? :<
- [filter-clause]
- (match/match-one filter-clause
+(defmethod can-optimize-clause? :<
+ [clause]
+ (match/match-one clause
[_tag
_opts
;; Don't optimize < with column that has default temporal bucket
@@ -108,9 +109,9 @@
(temporal-value :guard optimizable-temporal-value?)]
(field-and-temporal-value-have-compatible-units? field temporal-value)))
-(defmethod can-optimize-filter? :between
- [filter-clause]
- (match/match-one filter-clause
+(defmethod can-optimize-clause? :between
+ [clause]
+ (match/match-one clause
[:between
_opts
[(_offset :guard #{:+ :-})
@@ -215,14 +216,14 @@
(and (isa? (lib.walk/apply-f-for-stage-at-path lib/type-of query path expr) :type/Date)
(= (lib/raw-temporal-bucket expr) :day)))
-(defmulti ^:private optimize-filter
- "Optimize a filter clause against a temporal-bucketed `:field` or `:expression` clause and `:absolute-datetime` or
+(defmulti ^:private optimize-clause
+ "Optimize a clause against a temporal-bucketed `:field` or `:expression` clause and `:absolute-datetime` or
`:relative-datetime`value by converting to an unbucketed range."
{:arglists '([query path clause])}
(fn [_query _path clause]
(lib/dispatch-value clause)))
-(defmethod optimize-filter :=
+(defmethod optimize-clause :=
[query path [_tag _opts field temporal-value]]
(if (date-field-with-day-bucketing? query path field)
(lib/= (change-temporal-unit-to-default field) (change-temporal-unit-to-default temporal-value))
@@ -235,22 +236,22 @@
(lib/>= field' lower-bound)
(lib/< (lib/fresh-uuids field') upper-bound)))))))))
-(defmethod optimize-filter :!=
- [query path [_tag _opts field temporal-value :as filter-clause]]
+(defmethod optimize-clause :!=
+ [query path [_tag _opts field temporal-value :as clause]]
(if (date-field-with-day-bucketing? query path field)
(lib/!= (change-temporal-unit-to-default field) (change-temporal-unit-to-default temporal-value))
- (when-let [optimized ((get-method optimize-filter :=) query path filter-clause)]
+ (when-let [optimized ((get-method optimize-clause :=) query path clause)]
(lib/negate-boolean-expression optimized))))
-(mu/defn- optimize-comparison-filter :- [:maybe ::lib.schema.mbql-clause/clause]
- [query path optimize-temporal-value-fn [tag opts field temporal-value] new-filter-type :- [:enum :< :>=]]
+(mu/defn- optimize-comparison-clause :- [:maybe ::lib.schema.mbql-clause/clause]
+ [query path optimize-temporal-value-fn [tag opts field temporal-value] new-clause-type :- [:enum :< :>=]]
(b/cond
(date-field-with-day-bucketing? query path field)
[tag opts (change-temporal-unit-to-default field) (change-temporal-unit-to-default temporal-value)]
:let [new-bound (optimize-temporal-value-fn temporal-value (lib/raw-temporal-bucket field))]
new-bound
- ((case new-filter-type
+ ((case new-clause-type
:< lib/<
:>= lib/>=)
(change-temporal-unit-to-default field)
@@ -259,23 +260,23 @@
:else
(log/errorf "optimize-temporal-value-fn %s did not return a new bound" optimize-temporal-value-fn)))
-(defmethod optimize-filter :<
- [query path filter-clause]
- (optimize-comparison-filter query path #'temporal-value-lower-bound filter-clause :<))
+(defmethod optimize-clause :<
+ [query path clause]
+ (optimize-comparison-clause query path #'temporal-value-lower-bound clause :<))
-(defmethod optimize-filter :<=
- [query path filter-clause]
- (optimize-comparison-filter query path #'temporal-value-upper-bound filter-clause :<))
+(defmethod optimize-clause :<=
+ [query path clause]
+ (optimize-comparison-clause query path #'temporal-value-upper-bound clause :<))
-(defmethod optimize-filter :>
- [query path filter-clause]
- (optimize-comparison-filter query path #'temporal-value-upper-bound filter-clause :>=))
+(defmethod optimize-clause :>
+ [query path clause]
+ (optimize-comparison-clause query path #'temporal-value-upper-bound clause :>=))
-(defmethod optimize-filter :>=
- [query path filter-clause]
- (optimize-comparison-filter query path #'temporal-value-lower-bound filter-clause :>=))
+(defmethod optimize-clause :>=
+ [query path clause]
+ (optimize-comparison-clause query path #'temporal-value-lower-bound clause :>=))
-(defmethod optimize-filter :between
+(defmethod optimize-clause :between
[query path [_tag _opts field lower-bound upper-bound]]
(if (date-field-with-day-bucketing? query path field)
(lib/between
@@ -289,25 +290,25 @@
(lib/>= field' new-lower-bound)
(lib/< (lib/fresh-uuids field') new-upper-bound)))))))
-(def ^:private optimizable-filter-types
- (set (keys (methods optimize-filter))))
+(def ^:private optimizable-clause-types
+ (set (keys (methods optimize-clause))))
-(defn- optimize-temporal-filters* [query path clause]
- (when (lib/clause-of-type? clause optimizable-filter-types)
- (or (when (can-optimize-filter? clause)
- (u/prog1 (optimize-filter query path clause)
+(defn- optimize-temporal-clauses* [query path clause]
+ (when (lib/clause-of-type? clause optimizable-clause-types)
+ (or (when (can-optimize-clause? clause)
+ (u/prog1 (optimize-clause query path clause)
(if <>
(when-not (= clause <>)
- (log/tracef "Optimized filter %s to %s" (pr-str clause) (pr-str <>)))
- ;; if for some reason `optimize-filter` doesn't return an optimized filter clause, log and error and use
- ;; the original. `can-optimize-filter?` shouldn't have said we could optimize this filter in the first
+ (log/tracef "Optimized clause %s to %s" (pr-str clause) (pr-str <>)))
+ ;; if for some reason `optimize-clause` doesn't return an optimized clause, log and error and use
+ ;; the original. `can-optimize-clause?` shouldn't have said we could optimize this clause in the first
;; place
- (log/error "Error optimizing temporal filter clause: optimize-filter unexpectedly returned nil" (pr-str clause)))))
+ (log/error "Error optimizing temporal clause: optimize-clause unexpectedly returned nil" (pr-str clause)))))
clause)))
-(mu/defn optimize-temporal-filters :- ::lib.schema/query
- "Middleware that optimizes equality (`=` and `!=`) and comparison (`<`, `between`, etc.) filter clauses against
- bucketed datetime fields. Rewrites those filter clauses as logically equivalent filter clauses that do not use
+(mu/defn optimize-temporal-clauses :- ::lib.schema/query
+ "Middleware that optimizes equality (`=` and `!=`) and comparison (`<`, `between`, etc.) clauses against
+ bucketed datetime fields. Rewrites those clauses as logically equivalent clauses that do not use
bucketing (i.e., their datetime unit is `:default`, meaning no bucketing functions need be applied).
[:= [:field 1 {:temporal-unit :month}] [:absolute-datetime #t \"2019-09-01\" :month]]
@@ -333,11 +334,24 @@
(lib.walk/walk-stages
query
(fn [query path stage]
- (when (seq (:filters stage))
- (letfn [(update-filters [filters]
- (let [filters' (lib.walk/walk-clauses* filters #(optimize-temporal-filters* query path %))]
+ (cond-> stage
+ (seq (:filters stage))
+ (update :filters
+ (fn [filters]
+ (let [filters' (lib.walk/walk-clauses* filters #(optimize-temporal-clauses* query path %))]
(if (= filters' filters)
filters
;; if we did some optimizations, we should flatten/deduplicate the filter clauses afterwards.
- (lib/simplify-filters filters'))))]
- (update stage :filters update-filters))))))
+ (lib/simplify-filters filters')))))
+
+ (seq (:expressions stage))
+ (update :expressions
+ (fn [expressions]
+ (mapv (fn [[_ opts & _rest :as expr]]
+ (let [expr' (lib.walk/walk-clause expr #(optimize-temporal-clauses* query path %))]
+ (if (= expr' expr)
+ expr
+ (let [expr-name (:lib/expression-name opts)]
+ (cond-> expr'
+ expr-name (lib.options/update-options assoc :lib/expression-name expr-name))))))
+ expressions)))))))
diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.clj
index 68a18eaad72..6b1a580d380 100644
--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj
+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj
@@ -3,7 +3,6 @@
information; parses datetime string literals when appropriate."
(:refer-clojure :exclude [select-keys])
(:require
- [clojure.string :as str]
[java-time.api :as t]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -22,6 +21,8 @@
(:import
(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)))
+(set! *warn-on-reflection* true)
+
(mu/defn- value :- :mbql.clause/value
[info :- :map v]
[:value (assoc info :lib/uuid (str (random-uuid))) v])
@@ -185,31 +186,25 @@
(let [t (parse-temporal-string-literal-to-class s OffsetTime)]
(lib/time t target-unit)))
-(defmethod parse-temporal-string-literal :type/DateTime
- [_effective-type s target-unit]
- (let [t (parse-temporal-string-literal-to-class s LocalDateTime)]
+(defn- parse-datetime-string-literal [s target-unit klass]
+ (let [t (parse-temporal-string-literal-to-class s klass)
+ target-unit (if (and (= target-unit :default)
+ (try (LocalDate/parse s) true (catch Exception _ false)))
+ :day
+ target-unit)]
(lib/absolute-datetime t target-unit)))
-(defn- date-literal-string? [s]
- (not (str/includes? s "T")))
+(defmethod parse-temporal-string-literal :type/DateTime
+ [_effective-type s target-unit]
+ (parse-datetime-string-literal s target-unit LocalDateTime))
(defmethod parse-temporal-string-literal :type/DateTimeWithTZ
[_effective-type s target-unit]
- (let [t (parse-temporal-string-literal-to-class s OffsetDateTime)
- target-unit (if (and (= target-unit :default)
- (date-literal-string? s))
- :day
- target-unit)]
- (lib/absolute-datetime t target-unit)))
+ (parse-datetime-string-literal s target-unit OffsetDateTime))
(defmethod parse-temporal-string-literal :type/DateTimeWithZoneID
[_effective-type s target-unit]
- (let [target-unit (if (and (= target-unit :default)
- (date-literal-string? s))
- :day
- target-unit)
- t (parse-temporal-string-literal-to-class s ZonedDateTime)]
- (lib/absolute-datetime t target-unit)))
+ (parse-datetime-string-literal s target-unit ZonedDateTime))
(defmethod add-type-info String
[s {:keys [unit], :as col} & {:keys [parse-datetime-strings?]
diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.clj
index 3c81e60ebe0..bb3a215637b 100644
--- a/src/metabase/query_processor/preprocess.clj
+++ b/src/metabase/query_processor/preprocess.clj
@@ -27,7 +27,7 @@
[metabase.query-processor.middleware.measures :as measures]
[metabase.query-processor.middleware.metrics :as metrics]
[metabase.query-processor.middleware.normalize-query :as normalize]
- [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-filters]
+ [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses]
[metabase.query-processor.middleware.parameters :as parameters]
[metabase.query-processor.middleware.permissions :as qp.perms]
[metabase.query-processor.middleware.persistence :as qp.persistence]
@@ -101,7 +101,7 @@
#'qp.wrap-value-literals/wrap-value-literals
#'auto-parse-filter-values/auto-parse-filter-values
#'validate-temporal-bucketing/validate-temporal-bucketing
- #'optimize-temporal-filters/optimize-temporal-filters
+ #'optimize-temporal-clauses/optimize-temporal-clauses
#'limit/add-default-limit
#'qp.middleware.enterprise/apply-download-limit
#'qp.middleware.enterprise/apply-workspace-remapping
diff --git a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
index 31082c36324..e96bf366b9f 100644
--- a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
+++ b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
@@ -504,19 +504,26 @@
{:short_json {"b" "y"}, :long_json nil}}
(sample)))
(testing "If driver.sql/json-field-length is not implemented for the driver don't omit the long value"
- (letfn [(do-with-removed-method [thunk]
- (let [original-method (get-method driver.sql/json-field-length driver/*driver*)]
- (if (= original-method (get-method driver.sql/json-field-length :default))
- (thunk)
- (do (remove-method driver.sql/json-field-length driver/*driver*)
- (thunk)
- (defmethod driver.sql/json-field-length driver/*driver* [driver field]
- (original-method driver field))))))]
- (do-with-removed-method
- (fn []
- (is (= #{{:short_json {"a" "x"}, :long_json {"a" "x"}}
- {:short_json {"b" "y"}, :long_json {"b" "yyyyyyyyyy"}}}
- (sample)))))))
+ ;; Temporarily override json-field-length with the default (nyi) implementation.
+ ;; We can't just remove-method — for child drivers (e.g. :postgres-mbql5) the
+ ;; parent's method would still be inherited.
+ (let [default-method (get-method driver.sql/json-field-length :default)
+ original-method (get-method driver.sql/json-field-length driver/*driver*)
+ had-explicit? (contains? (methods driver.sql/json-field-length) driver/*driver*)]
+ (defmethod driver.sql/json-field-length driver/*driver* [driver field]
+ (default-method driver field))
+ (try
+ (is (= #{{:short_json {"a" "x"}, :long_json {"a" "x"}}
+ {:short_json {"b" "y"}, :long_json {"b" "yyyyyyyyyy"}}}
+ (sample)))
+ (finally
+ (remove-method driver.sql/json-field-length driver/*driver*)
+ ;; Re-install the original if there was an explicit method (not just inherited).
+ ;; For inherited methods (e.g. :postgres-mbql5), remove-method is enough —
+ ;; the parent's method resumes via hierarchy dispatch.
+ (when had-explicit?
+ (defmethod driver.sql/json-field-length driver/*driver* [driver field]
+ (original-method driver field)))))))
(testing "The resulting synced fields exclude the field that corresponds to the long value"
(is (= #{"id"
"short_json"
diff --git a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
index ed01854bc41..406c8288089 100644
--- a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
+++ b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
@@ -1,16 +1,20 @@
-(ns metabase.query-processor.middleware.optimize-temporal-filters-test
+(ns ^:mb/driver-tests metabase.query-processor.middleware.optimize-temporal-filters-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.driver :as driver]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.util :as lib.schema.util]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.attempted-murders-metadata-provider :as lib.tu.attempted-murders-metadata-provider]
[metabase.lib.test-util.macros :as lib.tu.macros]
[metabase.query-processor.compile :as qp.compile]
- [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-filters]
+ [metabase.query-processor.core :as qp]
+ [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses]
+ [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.test :as mt]
[metabase.util.date-2 :as u.date]))
@@ -22,7 +26,7 @@
(let [query (lib/query meta/metadata-provider (meta/table-metadata :orders))]
(or (-> clause
lib/->mbql5
- (->> (#'optimize-temporal-filters/optimize-filter query [:stages 0]))
+ (->> (#'optimize-temporal-clauses/optimize-clause query [:stages 0]))
lib/->legacy-MBQL)
clause)))
@@ -33,7 +37,7 @@
([metadata-provider query]
(let [query (lib/query metadata-provider query)]
(-> query
- optimize-temporal-filters/optimize-temporal-filters
+ optimize-temporal-clauses/optimize-temporal-clauses
lib/->legacy-MBQL))))
(defn- optimize-filters
@@ -120,7 +124,7 @@
(defn- assoc-field-options [legacy-ref & kvs]
(apply update (vec legacy-ref) 2 assoc kvs))
-(deftest ^:parallel optimize-temporal-filters-test
+(deftest ^:parallel optimize-temporal-clauses-test
(doseq [field-or-expr [[:field (meta/id :orders :created-at) {}]
[:expression "date" {}]]
{:keys [unit filter-value lower upper]} test-units-and-values]
@@ -244,10 +248,10 @@
(mt/with-report-timezone-id! timezone-id
(testing "lower-bound and upper-bound util fns"
(is (= lower
- (#'optimize-temporal-filters/temporal-literal-lower-bound :day t))
+ (#'optimize-temporal-clauses/temporal-literal-lower-bound :day t))
(format "lower bound of day(%s) in the %s timezone should be %s" t timezone-id lower))
(is (= upper
- (#'optimize-temporal-filters/temporal-literal-upper-bound :day t))
+ (#'optimize-temporal-clauses/temporal-literal-upper-bound :day t))
(format "upper bound of day(%s) in the %s timezone should be %s" t timezone-id upper)))
(testing "optimize-with-datetime"
(let [expected [:and
@@ -517,7 +521,7 @@
[:interval 10 :minute]]
[:relative-datetime 1 :minute]]]}}
(-> query
- optimize-temporal-filters/optimize-temporal-filters
+ optimize-temporal-clauses/optimize-temporal-clauses
lib/->legacy-MBQL))))))
(deftest ^:parallel optimize-filter-with-nested-compatible-field-3
@@ -537,7 +541,7 @@
[:relative-datetime -1 :week]
[:relative-datetime 0 :week]]}}
(-> query
- optimize-temporal-filters/optimize-temporal-filters
+ optimize-temporal-clauses/optimize-temporal-clauses
lib/->legacy-MBQL))))))
(deftest ^:parallel optimize-date-equals-date-filters-test
@@ -642,3 +646,116 @@
[:field (meta/id :products :created-at) {:base-type :type/DateTime, :temporal-unit :day}]]]
(is (= clause
(optimize-filter clause)))))))
+
+(defn- local-date= [expected-date]
+ (fn [actual]
+ (= (t/local-date expected-date) (t/local-date actual))))
+
+(defn- temporal? [x]
+ (instance? java.time.temporal.Temporal x))
+
+(deftest ^:parallel optimize-temporal-between-clauses-test
+ (mt/test-drivers (mt/normal-driver-select {:+features [:test/date-type :test/timestamptz-type]})
+ (mt/dataset attempted-murders-no-time
+ (are [start-date end-date expect-expressions expect-rows]
+ (let [mp (mt/metadata-provider)
+ attempts (lib.metadata/table mp (mt/id :attempts))
+ date (lib.metadata/field mp (mt/id :attempts :date))
+ datetime (lib.metadata/field mp (mt/id :attempts :datetime))
+ datetime-tz (lib.metadata/field mp (mt/id :attempts :datetime_tz))
+ query (-> (lib/query mp attempts)
+ (lib/expression "date_between" (lib/between date start-date end-date))
+ (lib/expression "datetime_between" (lib/between datetime start-date end-date))
+ (lib/expression "datetime_tz_between" (lib/between datetime-tz start-date end-date))
+ (lib/limit 10)
+ (as-> q (lib/with-fields q [(lib/expression-ref q "date_between")
+ (lib/expression-ref q "datetime_between")
+ (lib/expression-ref q "datetime_tz_between")])))
+ opt-expressions (-> query
+ qp.preprocess/preprocess
+ lib/expressions
+ lib.schema.util/remove-lib-uuids)]
+ (is (=? expect-expressions opt-expressions))
+ (is (= expect-rows (mt/formatted-rows [mt/boolish->bool mt/boolish->bool mt/boolish->bool]
+ (qp/process-query query)))))
+
+ ;; Date strings with no time component should have bucketing of `:day` and get optimized
+ "2019-11-04" "2019-11-07"
+ [[:between {:lib/expression-name "date_between"}
+ [:field map? int?]
+ [:absolute-datetime {} #t "2019-11-04" :default]
+ [:absolute-datetime {} #t "2019-11-07" :default]]
+ [:and {:lib/expression-name "datetime_between"}
+ [:>= {}
+ [:field map? int?]
+ [:absolute-datetime {} (local-date= "2019-11-04") :default]]
+ [:< {}
+ [:field map? int?]
+ [:absolute-datetime {} (local-date= "2019-11-08") :default]]]
+ [:and {:lib/expression-name "datetime_tz_between"}
+ [:>= {}
+ [:field map? int?]
+ [:absolute-datetime {} (local-date= "2019-11-04") :default]]
+ [:< {}
+ [:field map? int?]
+ [:absolute-datetime {} (local-date= "2019-11-08") :default]]]]
+ [[false false false]
+ [false false false]
+ [false false true]
+ [true true true]
+ [true true true]
+ [true true true]
+ [true true true]
+ [false false false]
+ [false false false]
+ [false false false]]
+
+ ;; Datetime strings should have bucketing of `:default` and not get optimized
+ "2019-11-04T12:00:00" "2019-11-07T01:00:00"
+ [[:between {:lib/expression-name "date_between"}
+ [:field map? int?]
+ [:absolute-datetime {} #t "2019-11-04" :default]
+ [:absolute-datetime {} #t "2019-11-07" :default]]
+ [:between {:lib/expression-name "datetime_between"}
+ [:field map? int?]
+ [:absolute-datetime {} temporal? :default]
+ [:absolute-datetime {} temporal? :default]]
+ [:between {:lib/expression-name "datetime_tz_between"}
+ [:field map? int?]
+ [:absolute-datetime {} temporal? :default]
+ [:absolute-datetime {} temporal? :default]]]
+ [[false false false]
+ [false false false]
+ [false false false]
+ [true false false]
+ [true true true]
+ [true true false]
+ [true false false]
+ [false false false]
+ [false false false]
+ [false false false]]
+
+ ;; Datetime strings without a T should have bucketing of `:default` and not get optimized
+ "2019-11-04 12:00:00" "2019-11-07 01:00:00"
+ [[:between {:lib/expression-name "date_between"}
+ [:field map? int?]
+ [:absolute-datetime {} #t "2019-11-04" :default]
+ [:absolute-datetime {} #t "2019-11-07" :default]]
+ [:between {:lib/expression-name "datetime_between"}
+ [:field map? int?]
+ [:absolute-datetime {} temporal? :default]
+ [:absolute-datetime {} temporal? :default]]
+ [:between {:lib/expression-name "datetime_tz_between"}
+ [:field map? int?]
+ [:absolute-datetime {} temporal? :default]
+ [:absolute-datetime {} temporal? :default]]]
+ [[false false false]
+ [false false false]
+ [false false false]
+ [true false false]
+ [true true true]
+ [true true false]
+ [true false false]
+ [false false false]
+ [false false false]
+ [false false false]]))))
diff --git a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
index d254ad4ea13..f727a3b0469 100644
--- a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
+++ b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
@@ -224,7 +224,7 @@
{:expressions {"foo" $date}
:filter [:>
[:expression "foo" {:base-type :type/DateTime}]
- [:absolute-datetime #t "2014-01-01T00:00" :default]]})
+ [:absolute-datetime #t "2014-01-01T00:00" :day]]})
(wrap-value-literals
(lib.tu.macros/mbql-query checkins
{:expressions {"foo" $date}
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index c543d8dfaf5..24a5c0ad52b 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -688,6 +688,10 @@
[_driver _feature _database]
true)
+(defmethod driver/database-supports? [::driver/driver :test/date-type]
+ [_driver _feature _database]
+ true)
+
(defmethod driver/database-supports? [::driver/driver :test/timestamptz-type]
[_driver _feature _database]
true)
diff --git a/test/metabase/test/data/sql.clj b/test/metabase/test/data/sql.clj
index 41b1a3fb9c1..6fe90a6eaf0 100644
--- a/test/metabase/test/data/sql.clj
+++ b/test/metabase/test/data/sql.clj
@@ -172,12 +172,19 @@
(fn [driver base-type] [(tx/dispatch-on-driver-with-test-extensions driver) base-type])
:hierarchy #'driver/hierarchy)
+(defn- find-method-for-ancestor-type
+ "Find a registered method for `driver` (or any of its ancestors) with `ancestor-type`."
+ [driver ancestor-type]
+ (let [methods* (methods field-base-type->sql-type)]
+ (some #(get methods* [% ancestor-type])
+ (cons driver (ancestors driver/hierarchy driver)))))
+
(defmethod field-base-type->sql-type :default
[driver base-type]
(or (some
(fn [ancestor-type]
(when-not (= ancestor-type :type/*)
- (when-let [method (get (methods field-base-type->sql-type) [driver ancestor-type])]
+ (when-let [method (find-method-for-ancestor-type driver ancestor-type)]
(log/infof "No test data type mapping for driver %s for base type %s, falling back to ancestor base type %s"
driver base-type ancestor-type)
(method driver base-type))))
diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.clj
index 2e5d360e989..aa6b415fc16 100644
--- a/test/metabase/upload/impl_test.clj
+++ b/test/metabase/upload/impl_test.clj
@@ -1308,7 +1308,7 @@
(mt/with-dynamic-fn-redefs [driver.u/supports? (constantly false)]
(is (thrown-with-msg?
java.lang.Exception
- #"^Uploads are not supported on \w+ databases\."
+ #"^Uploads are not supported on [\w-]+ databases\."
(do-with-uploaded-example-csv!
{:schema-name "public", :table-prefix "uploaded_magic_"}
identity)))))