The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Optimize temporal expressions the same as filters (#74875)
Riley Thompson71e66269c2d7
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 4d0a5d73bde..e9f1355ef97 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -194,7 +194,7 @@:filter [:and;; This still gets :default bucketing! auto-bucket-datetimes;; puts :day bucketing on both parts of this filter, since it's- ;; matching a YYYY-mm-dd string. Then optimize-temporal-filters+ ;; matching a YYYY-mm-dd string. Then optimize-temporal-clauses;; sees that the;; :type/Date column already has :day;; granularity, and switches both to :defaultdiff --git a/modules/drivers/mongo/test/metabase/test/data/mongo.clj b/modules/drivers/mongo/test/metabase/test/data/mongo.cljindex 3706fb79149..49634216f7f 100644--- a/modules/drivers/mongo/test/metabase/test/data/mongo.clj+++ b/modules/drivers/mongo/test/metabase/test/data/mongo.clj@@ -19,7 +19,8 @@(tx/add-test-extensions! :mongo)(doseq [feature [:test/time-type- :test/timestamptz-type]]+ :test/timestamptz-type+ :test/date-type]](defmethod driver/database-supports? [:mongo feature][_driver _feature _database]false))diff --git a/modules/drivers/oracle/test/metabase/test/data/oracle.clj b/modules/drivers/oracle/test/metabase/test/data/oracle.cljindex f5893d557f5..41bf0b5bcbe 100644--- a/modules/drivers/oracle/test/metabase/test/data/oracle.clj+++ b/modules/drivers/oracle/test/metabase/test/data/oracle.clj@@ -105,6 +105,10 @@[_driver _feature _database]false)+(defmethod driver/database-supports? [:oracle :test/date-type]+ [_driver _feature _database]+ false)+(doseq [[base-type sql-type] {:type/BigInteger "NUMBER(*,0)":type/Boolean "NUMBER(1)":type/Date "DATE"diff --git a/src/metabase/query_processor/middleware/optimize_temporal_filters.clj b/src/metabase/query_processor/middleware/optimize_temporal_filters.cljindex a1b62cbd47d..6e8f1a0718d 100644--- a/src/metabase/query_processor/middleware/optimize_temporal_filters.clj+++ b/src/metabase/query_processor/middleware/optimize_temporal_filters.clj@@ -1,10 +1,11 @@(ns metabase.query-processor.middleware.optimize-temporal-filters"Middleware that optimizes equality filter clauses against bucketed temporal fields. See docstring for- `optimize-temporal-filters` for more details."- (:refer-clojure :exclude [get-in])+ `optimize-temporal-clauses` for more details."+ (:refer-clojure :exclude [get-in mapv])(:require[better-cond.core :as b][metabase.lib.core :as lib]+ [metabase.lib.options :as lib.options][metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as lib.schema.expression]@@ -16,7 +17,7 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.match :as match]- [metabase.util.performance :refer [get-in]]))+ [metabase.util.performance :refer [get-in mapv]]))(def ^:private optimizable-units#{:second :minute :hour :day :week :month :quarter :year})@@ -36,7 +37,7 @@(or (= unit :default)(contains? optimizable-units unit))))))-(defmulti ^:private can-optimize-filter?+(defmulti ^:private can-optimize-clause?{:arglists '([mbql-clause])}lib/dispatch-value)@@ -54,7 +55,7 @@(defn- field-and-temporal-value-have-compatible-units?"Do datetime `field` clause and `temporal-value` clause have 'compatible' units that mean we'll be able to optimize- the filter clause they're in?"+ the clause they're in?"[field temporal-value](match/match-one temporal-value[:relative-datetime _opts #{0 :current}]@@ -68,9 +69,9 @@(= value-unit :default) (contains? optimizable-units field-unit):else (= field-unit value-unit)))))-(defmethod can-optimize-filter? :default- [filter-clause]- (match/match-one filter-clause+(defmethod can-optimize-clause? :default+ [clause]+ (match/match-one clause[_tag_opts(field :guard optimizable-expr?)@@ -82,15 +83,15 @@(and (vector? clause)(not= :default (get-in clause [2 :temporal-unit]))))-;; TODO: I believe we do not generate __filter clauses that have default temporal bucket on column arg which should be-;; optimized__. Unfortunately I'm not certain about that. If I was, the following `can-optimize-filter? :>=` and-;; `can-optimize-filter? :>=` definitions would be redundant after update of `optimizable-expr?`, ie. changing+;; TODO: I believe we do not generate __clauses that have default temporal bucket on column arg which should be+;; optimized__. Unfortunately I'm not certain about that. If I was, the following `can-optimize-clause? :>=` and+;; `can-optimize-clause? :>=` definitions would be redundant after update of `optimizable-expr?`, ie. changing;; the logic to something along "if `expr` has default temporal unit we should not optimize".-(defmethod can-optimize-filter? :>=- [filter-clause]+(defmethod can-optimize-clause? :>=+ [clause](match/match-one- filter-clause+ clause[_tag_opts;; Don't optimize >= with column that has default temporal bucket@@ -98,9 +99,9 @@(temporal-value :guard optimizable-temporal-value?)](field-and-temporal-value-have-compatible-units? field temporal-value)))-(defmethod can-optimize-filter? :<- [filter-clause]- (match/match-one filter-clause+(defmethod can-optimize-clause? :<+ [clause]+ (match/match-one clause[_tag_opts;; Don't optimize < with column that has default temporal bucket@@ -108,9 +109,9 @@(temporal-value :guard optimizable-temporal-value?)](field-and-temporal-value-have-compatible-units? field temporal-value)))-(defmethod can-optimize-filter? :between- [filter-clause]- (match/match-one filter-clause+(defmethod can-optimize-clause? :between+ [clause]+ (match/match-one clause[:between_opts[(_offset :guard #{:+ :-})@@ -215,14 +216,14 @@(and (isa? (lib.walk/apply-f-for-stage-at-path lib/type-of query path expr) :type/Date)(= (lib/raw-temporal-bucket expr) :day)))-(defmulti ^:private optimize-filter- "Optimize a filter clause against a temporal-bucketed `:field` or `:expression` clause and `:absolute-datetime` or+(defmulti ^:private optimize-clause+ "Optimize a clause against a temporal-bucketed `:field` or `:expression` clause and `:absolute-datetime` or`:relative-datetime`value by converting to an unbucketed range."{:arglists '([query path clause])}(fn [_query _path clause](lib/dispatch-value clause)))-(defmethod optimize-filter :=+(defmethod optimize-clause :=[query path [_tag _opts field temporal-value]](if (date-field-with-day-bucketing? query path field)(lib/= (change-temporal-unit-to-default field) (change-temporal-unit-to-default temporal-value))@@ -235,22 +236,22 @@(lib/>= field' lower-bound)(lib/< (lib/fresh-uuids field') upper-bound)))))))))-(defmethod optimize-filter :!=- [query path [_tag _opts field temporal-value :as filter-clause]]+(defmethod optimize-clause :!=+ [query path [_tag _opts field temporal-value :as clause]](if (date-field-with-day-bucketing? query path field)(lib/!= (change-temporal-unit-to-default field) (change-temporal-unit-to-default temporal-value))- (when-let [optimized ((get-method optimize-filter :=) query path filter-clause)]+ (when-let [optimized ((get-method optimize-clause :=) query path clause)](lib/negate-boolean-expression optimized))))-(mu/defn- optimize-comparison-filter :- [:maybe ::lib.schema.mbql-clause/clause]- [query path optimize-temporal-value-fn [tag opts field temporal-value] new-filter-type :- [:enum :< :>=]]+(mu/defn- optimize-comparison-clause :- [:maybe ::lib.schema.mbql-clause/clause]+ [query path optimize-temporal-value-fn [tag opts field temporal-value] new-clause-type :- [:enum :< :>=]](b/cond(date-field-with-day-bucketing? query path field)[tag opts (change-temporal-unit-to-default field) (change-temporal-unit-to-default temporal-value)]:let [new-bound (optimize-temporal-value-fn temporal-value (lib/raw-temporal-bucket field))]new-bound- ((case new-filter-type+ ((case new-clause-type:< lib/<:>= lib/>=)(change-temporal-unit-to-default field)@@ -259,23 +260,23 @@:else(log/errorf "optimize-temporal-value-fn %s did not return a new bound" optimize-temporal-value-fn)))-(defmethod optimize-filter :<- [query path filter-clause]- (optimize-comparison-filter query path #'temporal-value-lower-bound filter-clause :<))+(defmethod optimize-clause :<+ [query path clause]+ (optimize-comparison-clause query path #'temporal-value-lower-bound clause :<))-(defmethod optimize-filter :<=- [query path filter-clause]- (optimize-comparison-filter query path #'temporal-value-upper-bound filter-clause :<))+(defmethod optimize-clause :<=+ [query path clause]+ (optimize-comparison-clause query path #'temporal-value-upper-bound clause :<))-(defmethod optimize-filter :>- [query path filter-clause]- (optimize-comparison-filter query path #'temporal-value-upper-bound filter-clause :>=))+(defmethod optimize-clause :>+ [query path clause]+ (optimize-comparison-clause query path #'temporal-value-upper-bound clause :>=))-(defmethod optimize-filter :>=- [query path filter-clause]- (optimize-comparison-filter query path #'temporal-value-lower-bound filter-clause :>=))+(defmethod optimize-clause :>=+ [query path clause]+ (optimize-comparison-clause query path #'temporal-value-lower-bound clause :>=))-(defmethod optimize-filter :between+(defmethod optimize-clause :between[query path [_tag _opts field lower-bound upper-bound]](if (date-field-with-day-bucketing? query path field)(lib/between@@ -289,25 +290,25 @@(lib/>= field' new-lower-bound)(lib/< (lib/fresh-uuids field') new-upper-bound)))))))-(def ^:private optimizable-filter-types- (set (keys (methods optimize-filter))))+(def ^:private optimizable-clause-types+ (set (keys (methods optimize-clause))))-(defn- optimize-temporal-filters* [query path clause]- (when (lib/clause-of-type? clause optimizable-filter-types)- (or (when (can-optimize-filter? clause)- (u/prog1 (optimize-filter query path clause)+(defn- optimize-temporal-clauses* [query path clause]+ (when (lib/clause-of-type? clause optimizable-clause-types)+ (or (when (can-optimize-clause? clause)+ (u/prog1 (optimize-clause query path clause)(if <>(when-not (= clause <>)- (log/tracef "Optimized filter %s to %s" (pr-str clause) (pr-str <>)))- ;; if for some reason `optimize-filter` doesn't return an optimized filter clause, log and error and use- ;; the original. `can-optimize-filter?` shouldn't have said we could optimize this filter in the first+ (log/tracef "Optimized clause %s to %s" (pr-str clause) (pr-str <>)))+ ;; if for some reason `optimize-clause` doesn't return an optimized clause, log and error and use+ ;; the original. `can-optimize-clause?` shouldn't have said we could optimize this clause in the first;; place- (log/error "Error optimizing temporal filter clause: optimize-filter unexpectedly returned nil" (pr-str clause)))))+ (log/error "Error optimizing temporal clause: optimize-clause unexpectedly returned nil" (pr-str clause)))))clause)))-(mu/defn optimize-temporal-filters :- ::lib.schema/query- "Middleware that optimizes equality (`=` and `!=`) and comparison (`<`, `between`, etc.) filter clauses against- bucketed datetime fields. Rewrites those filter clauses as logically equivalent filter clauses that do not use+(mu/defn optimize-temporal-clauses :- ::lib.schema/query+ "Middleware that optimizes equality (`=` and `!=`) and comparison (`<`, `between`, etc.) clauses against+ bucketed datetime fields. Rewrites those clauses as logically equivalent clauses that do not usebucketing (i.e., their datetime unit is `:default`, meaning no bucketing functions need be applied).[:= [:field 1 {:temporal-unit :month}] [:absolute-datetime #t \"2019-09-01\" :month]]@@ -333,11 +334,24 @@(lib.walk/walk-stagesquery(fn [query path stage]- (when (seq (:filters stage))- (letfn [(update-filters [filters]- (let [filters' (lib.walk/walk-clauses* filters #(optimize-temporal-filters* query path %))]+ (cond-> stage+ (seq (:filters stage))+ (update :filters+ (fn [filters]+ (let [filters' (lib.walk/walk-clauses* filters #(optimize-temporal-clauses* query path %))](if (= filters' filters)filters;; if we did some optimizations, we should flatten/deduplicate the filter clauses afterwards.- (lib/simplify-filters filters'))))]- (update stage :filters update-filters))))))+ (lib/simplify-filters filters')))))++ (seq (:expressions stage))+ (update :expressions+ (fn [expressions]+ (mapv (fn [[_ opts & _rest :as expr]]+ (let [expr' (lib.walk/walk-clause expr #(optimize-temporal-clauses* query path %))]+ (if (= expr' expr)+ expr+ (let [expr-name (:lib/expression-name opts)]+ (cond-> expr'+ expr-name (lib.options/update-options assoc :lib/expression-name expr-name))))))+ expressions)))))))diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.cljindex 68a18eaad72..6b1a580d380 100644--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj@@ -3,7 +3,6 @@information; parses datetime string literals when appropriate."(:refer-clojure :exclude [select-keys])(:require- [clojure.string :as str][java-time.api :as t][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -22,6 +21,8 @@(:import(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)))+(set! *warn-on-reflection* true)+(mu/defn- value :- :mbql.clause/value[info :- :map v][:value (assoc info :lib/uuid (str (random-uuid))) v])@@ -185,31 +186,25 @@(let [t (parse-temporal-string-literal-to-class s OffsetTime)](lib/time t target-unit)))-(defmethod parse-temporal-string-literal :type/DateTime- [_effective-type s target-unit]- (let [t (parse-temporal-string-literal-to-class s LocalDateTime)]+(defn- parse-datetime-string-literal [s target-unit klass]+ (let [t (parse-temporal-string-literal-to-class s klass)+ target-unit (if (and (= target-unit :default)+ (try (LocalDate/parse s) true (catch Exception _ false)))+ :day+ target-unit)](lib/absolute-datetime t target-unit)))-(defn- date-literal-string? [s]- (not (str/includes? s "T")))+(defmethod parse-temporal-string-literal :type/DateTime+ [_effective-type s target-unit]+ (parse-datetime-string-literal s target-unit LocalDateTime))(defmethod parse-temporal-string-literal :type/DateTimeWithTZ[_effective-type s target-unit]- (let [t (parse-temporal-string-literal-to-class s OffsetDateTime)- target-unit (if (and (= target-unit :default)- (date-literal-string? s))- :day- target-unit)]- (lib/absolute-datetime t target-unit)))+ (parse-datetime-string-literal s target-unit OffsetDateTime))(defmethod parse-temporal-string-literal :type/DateTimeWithZoneID[_effective-type s target-unit]- (let [target-unit (if (and (= target-unit :default)- (date-literal-string? s))- :day- target-unit)- t (parse-temporal-string-literal-to-class s ZonedDateTime)]- (lib/absolute-datetime t target-unit)))+ (parse-datetime-string-literal s target-unit ZonedDateTime))(defmethod add-type-info String[s {:keys [unit], :as col} & {:keys [parse-datetime-strings?]diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex 3c81e60ebe0..bb3a215637b 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -27,7 +27,7 @@[metabase.query-processor.middleware.measures :as measures][metabase.query-processor.middleware.metrics :as metrics][metabase.query-processor.middleware.normalize-query :as normalize]- [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-filters]+ [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses][metabase.query-processor.middleware.parameters :as parameters][metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.middleware.persistence :as qp.persistence]@@ -101,7 +101,7 @@#'qp.wrap-value-literals/wrap-value-literals#'auto-parse-filter-values/auto-parse-filter-values#'validate-temporal-bucketing/validate-temporal-bucketing- #'optimize-temporal-filters/optimize-temporal-filters+ #'optimize-temporal-clauses/optimize-temporal-clauses#'limit/add-default-limit#'qp.middleware.enterprise/apply-download-limit#'qp.middleware.enterprise/apply-workspace-remappingdiff --git a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_table_test.cljindex 31082c36324..e96bf366b9f 100644--- a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj+++ b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj@@ -504,19 +504,26 @@{:short_json {"b" "y"}, :long_json nil}}(sample)))(testing "If driver.sql/json-field-length is not implemented for the driver don't omit the long value"- (letfn [(do-with-removed-method [thunk]- (let [original-method (get-method driver.sql/json-field-length driver/*driver*)]- (if (= original-method (get-method driver.sql/json-field-length :default))- (thunk)- (do (remove-method driver.sql/json-field-length driver/*driver*)- (thunk)- (defmethod driver.sql/json-field-length driver/*driver* [driver field]- (original-method driver field))))))]- (do-with-removed-method- (fn []- (is (= #{{:short_json {"a" "x"}, :long_json {"a" "x"}}- {:short_json {"b" "y"}, :long_json {"b" "yyyyyyyyyy"}}}- (sample)))))))+ ;; Temporarily override json-field-length with the default (nyi) implementation.+ ;; We can't just remove-method — for child drivers (e.g. :postgres-mbql5) the+ ;; parent's method would still be inherited.+ (let [default-method (get-method driver.sql/json-field-length :default)+ original-method (get-method driver.sql/json-field-length driver/*driver*)+ had-explicit? (contains? (methods driver.sql/json-field-length) driver/*driver*)]+ (defmethod driver.sql/json-field-length driver/*driver* [driver field]+ (default-method driver field))+ (try+ (is (= #{{:short_json {"a" "x"}, :long_json {"a" "x"}}+ {:short_json {"b" "y"}, :long_json {"b" "yyyyyyyyyy"}}}+ (sample)))+ (finally+ (remove-method driver.sql/json-field-length driver/*driver*)+ ;; Re-install the original if there was an explicit method (not just inherited).+ ;; For inherited methods (e.g. :postgres-mbql5), remove-method is enough —+ ;; the parent's method resumes via hierarchy dispatch.+ (when had-explicit?+ (defmethod driver.sql/json-field-length driver/*driver* [driver field]+ (original-method driver field)))))))(testing "The resulting synced fields exclude the field that corresponds to the long value"(is (= #{"id""short_json"diff --git a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.cljindex ed01854bc41..406c8288089 100644--- a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj+++ b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj@@ -1,16 +1,20 @@-(ns metabase.query-processor.middleware.optimize-temporal-filters-test+(ns ^:mb/driver-tests metabase.query-processor.middleware.optimize-temporal-filters-test(:require[clojure.string :as str][clojure.test :refer :all][java-time.api :as t][metabase.driver :as driver][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.util :as lib.schema.util][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.attempted-murders-metadata-provider :as lib.tu.attempted-murders-metadata-provider][metabase.lib.test-util.macros :as lib.tu.macros][metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-filters]+ [metabase.query-processor.core :as qp]+ [metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.test :as mt][metabase.util.date-2 :as u.date]))@@ -22,7 +26,7 @@(let [query (lib/query meta/metadata-provider (meta/table-metadata :orders))](or (-> clauselib/->mbql5- (->> (#'optimize-temporal-filters/optimize-filter query [:stages 0]))+ (->> (#'optimize-temporal-clauses/optimize-clause query [:stages 0]))lib/->legacy-MBQL)clause)))@@ -33,7 +37,7 @@([metadata-provider query](let [query (lib/query metadata-provider query)](-> query- optimize-temporal-filters/optimize-temporal-filters+ optimize-temporal-clauses/optimize-temporal-clauseslib/->legacy-MBQL))))(defn- optimize-filters@@ -120,7 +124,7 @@(defn- assoc-field-options [legacy-ref & kvs](apply update (vec legacy-ref) 2 assoc kvs))-(deftest ^:parallel optimize-temporal-filters-test+(deftest ^:parallel optimize-temporal-clauses-test(doseq [field-or-expr [[:field (meta/id :orders :created-at) {}][:expression "date" {}]]{:keys [unit filter-value lower upper]} test-units-and-values]@@ -244,10 +248,10 @@(mt/with-report-timezone-id! timezone-id(testing "lower-bound and upper-bound util fns"(is (= lower- (#'optimize-temporal-filters/temporal-literal-lower-bound :day t))+ (#'optimize-temporal-clauses/temporal-literal-lower-bound :day t))(format "lower bound of day(%s) in the %s timezone should be %s" t timezone-id lower))(is (= upper- (#'optimize-temporal-filters/temporal-literal-upper-bound :day t))+ (#'optimize-temporal-clauses/temporal-literal-upper-bound :day t))(format "upper bound of day(%s) in the %s timezone should be %s" t timezone-id upper)))(testing "optimize-with-datetime"(let [expected [:and@@ -517,7 +521,7 @@[:interval 10 :minute]][:relative-datetime 1 :minute]]]}}(-> query- optimize-temporal-filters/optimize-temporal-filters+ optimize-temporal-clauses/optimize-temporal-clauseslib/->legacy-MBQL))))))(deftest ^:parallel optimize-filter-with-nested-compatible-field-3@@ -537,7 +541,7 @@[:relative-datetime -1 :week][:relative-datetime 0 :week]]}}(-> query- optimize-temporal-filters/optimize-temporal-filters+ optimize-temporal-clauses/optimize-temporal-clauseslib/->legacy-MBQL))))))(deftest ^:parallel optimize-date-equals-date-filters-test@@ -642,3 +646,116 @@[:field (meta/id :products :created-at) {:base-type :type/DateTime, :temporal-unit :day}]]](is (= clause(optimize-filter clause)))))))++(defn- local-date= [expected-date]+ (fn [actual]+ (= (t/local-date expected-date) (t/local-date actual))))++(defn- temporal? [x]+ (instance? java.time.temporal.Temporal x))++(deftest ^:parallel optimize-temporal-between-clauses-test+ (mt/test-drivers (mt/normal-driver-select {:+features [:test/date-type :test/timestamptz-type]})+ (mt/dataset attempted-murders-no-time+ (are [start-date end-date expect-expressions expect-rows]+ (let [mp (mt/metadata-provider)+ attempts (lib.metadata/table mp (mt/id :attempts))+ date (lib.metadata/field mp (mt/id :attempts :date))+ datetime (lib.metadata/field mp (mt/id :attempts :datetime))+ datetime-tz (lib.metadata/field mp (mt/id :attempts :datetime_tz))+ query (-> (lib/query mp attempts)+ (lib/expression "date_between" (lib/between date start-date end-date))+ (lib/expression "datetime_between" (lib/between datetime start-date end-date))+ (lib/expression "datetime_tz_between" (lib/between datetime-tz start-date end-date))+ (lib/limit 10)+ (as-> q (lib/with-fields q [(lib/expression-ref q "date_between")+ (lib/expression-ref q "datetime_between")+ (lib/expression-ref q "datetime_tz_between")])))+ opt-expressions (-> query+ qp.preprocess/preprocess+ lib/expressions+ lib.schema.util/remove-lib-uuids)]+ (is (=? expect-expressions opt-expressions))+ (is (= expect-rows (mt/formatted-rows [mt/boolish->bool mt/boolish->bool mt/boolish->bool]+ (qp/process-query query)))))++ ;; Date strings with no time component should have bucketing of `:day` and get optimized+ "2019-11-04" "2019-11-07"+ [[:between {:lib/expression-name "date_between"}+ [:field map? int?]+ [:absolute-datetime {} #t "2019-11-04" :default]+ [:absolute-datetime {} #t "2019-11-07" :default]]+ [:and {:lib/expression-name "datetime_between"}+ [:>= {}+ [:field map? int?]+ [:absolute-datetime {} (local-date= "2019-11-04") :default]]+ [:< {}+ [:field map? int?]+ [:absolute-datetime {} (local-date= "2019-11-08") :default]]]+ [:and {:lib/expression-name "datetime_tz_between"}+ [:>= {}+ [:field map? int?]+ [:absolute-datetime {} (local-date= "2019-11-04") :default]]+ [:< {}+ [:field map? int?]+ [:absolute-datetime {} (local-date= "2019-11-08") :default]]]]+ [[false false false]+ [false false false]+ [false false true]+ [true true true]+ [true true true]+ [true true true]+ [true true true]+ [false false false]+ [false false false]+ [false false false]]++ ;; Datetime strings should have bucketing of `:default` and not get optimized+ "2019-11-04T12:00:00" "2019-11-07T01:00:00"+ [[:between {:lib/expression-name "date_between"}+ [:field map? int?]+ [:absolute-datetime {} #t "2019-11-04" :default]+ [:absolute-datetime {} #t "2019-11-07" :default]]+ [:between {:lib/expression-name "datetime_between"}+ [:field map? int?]+ [:absolute-datetime {} temporal? :default]+ [:absolute-datetime {} temporal? :default]]+ [:between {:lib/expression-name "datetime_tz_between"}+ [:field map? int?]+ [:absolute-datetime {} temporal? :default]+ [:absolute-datetime {} temporal? :default]]]+ [[false false false]+ [false false false]+ [false false false]+ [true false false]+ [true true true]+ [true true false]+ [true false false]+ [false false false]+ [false false false]+ [false false false]]++ ;; Datetime strings without a T should have bucketing of `:default` and not get optimized+ "2019-11-04 12:00:00" "2019-11-07 01:00:00"+ [[:between {:lib/expression-name "date_between"}+ [:field map? int?]+ [:absolute-datetime {} #t "2019-11-04" :default]+ [:absolute-datetime {} #t "2019-11-07" :default]]+ [:between {:lib/expression-name "datetime_between"}+ [:field map? int?]+ [:absolute-datetime {} temporal? :default]+ [:absolute-datetime {} temporal? :default]]+ [:between {:lib/expression-name "datetime_tz_between"}+ [:field map? int?]+ [:absolute-datetime {} temporal? :default]+ [:absolute-datetime {} temporal? :default]]]+ [[false false false]+ [false false false]+ [false false false]+ [true false false]+ [true true true]+ [true true false]+ [true false false]+ [false false false]+ [false false false]+ [false false false]]))))diff --git a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj b/test/metabase/query_processor/middleware/wrap_value_literals_test.cljindex d254ad4ea13..f727a3b0469 100644--- a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj+++ b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj@@ -224,7 +224,7 @@{:expressions {"foo" $date}:filter [:>[:expression "foo" {:base-type :type/DateTime}]- [:absolute-datetime #t "2014-01-01T00:00" :default]]})+ [:absolute-datetime #t "2014-01-01T00:00" :day]]})(wrap-value-literals(lib.tu.macros/mbql-query checkins{:expressions {"foo" $date}diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.cljindex c543d8dfaf5..24a5c0ad52b 100644--- a/test/metabase/test/data/interface.clj+++ b/test/metabase/test/data/interface.clj@@ -688,6 +688,10 @@[_driver _feature _database]true)+(defmethod driver/database-supports? [::driver/driver :test/date-type]+ [_driver _feature _database]+ true)+(defmethod driver/database-supports? [::driver/driver :test/timestamptz-type][_driver _feature _database]true)diff --git a/test/metabase/test/data/sql.clj b/test/metabase/test/data/sql.cljindex 41b1a3fb9c1..6fe90a6eaf0 100644--- a/test/metabase/test/data/sql.clj+++ b/test/metabase/test/data/sql.clj@@ -172,12 +172,19 @@(fn [driver base-type] [(tx/dispatch-on-driver-with-test-extensions driver) base-type]):hierarchy #'driver/hierarchy)+(defn- find-method-for-ancestor-type+ "Find a registered method for `driver` (or any of its ancestors) with `ancestor-type`."+ [driver ancestor-type]+ (let [methods* (methods field-base-type->sql-type)]+ (some #(get methods* [% ancestor-type])+ (cons driver (ancestors driver/hierarchy driver)))))+(defmethod field-base-type->sql-type :default[driver base-type](or (some(fn [ancestor-type](when-not (= ancestor-type :type/*)- (when-let [method (get (methods field-base-type->sql-type) [driver ancestor-type])]+ (when-let [method (find-method-for-ancestor-type driver ancestor-type)](log/infof "No test data type mapping for driver %s for base type %s, falling back to ancestor base type %s"driver base-type ancestor-type)(method driver base-type))))diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.cljindex 2e5d360e989..aa6b415fc16 100644--- a/test/metabase/upload/impl_test.clj+++ b/test/metabase/upload/impl_test.clj@@ -1308,7 +1308,7 @@(mt/with-dynamic-fn-redefs [driver.u/supports? (constantly false)](is (thrown-with-msg?java.lang.Exception- #"^Uploads are not supported on \w+ databases\."+ #"^Uploads are not supported on [\w-]+ databases\."(do-with-uploaded-example-csv!{:schema-name "public", :table-prefix "uploaded_magic_"}identity)))))