Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[MBQL lib] Use `:between` filters for bucketed time fields (#64218)

Braden Shepherdson21e9d33c668d
diff --git a/src/metabase/lib/drill_thru/underlying_records.cljc b/src/metabase/lib/drill_thru/underlying_records.cljc
index f462e1fa939..8bf3107da9e 100644
--- a/src/metabase/lib/drill_thru/underlying_records.cljc
+++ b/src/metabase/lib/drill_thru/underlying_records.cljc
@@ -34,6 +34,7 @@
[metabase.lib.aggregation :as lib.aggregation]
[metabase.lib.binning :as lib.binning]
[metabase.lib.drill-thru.common :as lib.drill-thru.common]
+ [metabase.lib.fe-util :as lib.fe-util]
[metabase.lib.filter :as lib.filter]
[metabase.lib.join :as lib.join]
[metabase.lib.metadata :as lib.metadata]
@@ -43,6 +44,7 @@
[metabase.lib.schema.drill-thru :as lib.schema.drill-thru]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.ref :as lib.schema.ref]
+ [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]
[metabase.lib.temporal-bucket :as lib.temporal-bucket]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.lib.underlying :as lib.underlying]
@@ -142,13 +144,18 @@
;; instead of
;;
;; month(col) = March 2023
- (let [column (if-let [temporal-unit (or (::lib.underlying/temporal-unit column)
- (lib.temporal-bucket/temporal-bucket column))]
- (lib.temporal-bucket/with-temporal-bucket filter-column temporal-unit)
+ (let [bucket (or (::lib.underlying/temporal-unit column)
+ (lib.temporal-bucket/temporal-bucket column))
+ unit (cond-> bucket
+ (map? bucket) :unit)
+ column (if unit
+ (lib.temporal-bucket/with-temporal-bucket filter-column unit)
filter-column)]
- (if (some? value)
- [(lib.filter/= column value)]
- [(lib.filter/is-null column)])))]
+ (if (nil? value)
+ [(lib.filter/is-null column)]
+ [(cond-> (lib.filter/= column value)
+ (and unit (lib.schema.temporal-bucketing/datetime-truncation-units unit))
+ lib.fe-util/expand-temporal-expression)])))]
(reduce
(fn [query filter-clause]
(lib.filter/filter query stage-number filter-clause))
diff --git a/src/metabase/lib/fe_util.cljc b/src/metabase/lib/fe_util.cljc
index a097fa1e2ee..dffcfe1308c 100644
--- a/src/metabase/lib/fe_util.cljc
+++ b/src/metabase/lib/fe_util.cljc
@@ -73,7 +73,7 @@
(:temporal-unit (lib.options/options maybe-clause-arg)))
(u.time/timestamp-coercible? other-arg))))
-(defn- expand-temporal-expression
+(defn expand-temporal-expression
"Modify expression in a way, that its resulting [[expression-parts]] are digestable by filter picker.
Current filter picker implementation is unable to handle expression parts of expressions of a form
diff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.clj
index baaaaed9bde..c5d67a254b1 100644
--- a/src/metabase/xrays/automagic_dashboards/combination.clj
+++ b/src/metabase/xrays/automagic_dashboards/combination.clj
@@ -163,8 +163,8 @@
[{:keys [cell-query]}]
(letfn [(collect-dimensions [[op & args]]
(case (some-> op qp.util/normalize-token)
- :and (mapcat collect-dimensions args)
- := (magic.util/collect-field-references args)
+ :and (mapcat collect-dimensions args)
+ (:between :=) (magic.util/collect-field-references args)
nil))]
(->> cell-query
collect-dimensions
diff --git a/src/metabase/xrays/automagic_dashboards/filters.clj b/src/metabase/xrays/automagic_dashboards/filters.clj
index a01cddb4d90..ff4e7ffdf2f 100644
--- a/src/metabase/xrays/automagic_dashboards/filters.clj
+++ b/src/metabase/xrays/automagic_dashboards/filters.clj
@@ -13,7 +13,8 @@
(defn- temporal?
"Does `field` represent a temporal value, i.e. a date, time, or datetime?"
[{base-type :base_type, effective-type :effective_type, unit :unit}]
- ;; TODO -- not sure why we're excluding year here? Is it because we normally returned it as an integer in the past?
+ ;; Excluding :year because it's (currently) both an extraction and truncation unit.
+ ;; For the purposes of this check, :year is an interesting :unit which yields a time interval, not just a number.
(and (not ((disj u.date/extract-units :year) unit))
(isa? (or effective-type base-type) :type/Temporal)))
diff --git a/src/metabase/xrays/automagic_dashboards/names.clj b/src/metabase/xrays/automagic_dashboards/names.clj
index 0973a863945..82e045bcbc4 100644
--- a/src/metabase/xrays/automagic_dashboards/names.clj
+++ b/src/metabase/xrays/automagic_dashboards/names.clj
@@ -204,7 +204,13 @@
(mu/defmethod humanize-filter-value :between
[root :- ::ads/root
[_ field-reference min-value max-value]]
- (tru "{0} is between {1} and {2}" (item-name root field-reference) min-value max-value))
+ (let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
+ (or (when (isa? (or effective_type base_type) :type/Temporal)
+ (let [humanized-min (humanize-datetime min-value unit)
+ humanized-max (humanize-datetime max-value unit)]
+ (when (= humanized-min humanized-max)
+ (tru "{0} is {1}" item-name humanized-min))))
+ (tru "{0} is between {1} and {2}" item-name min-value max-value))))
(mu/defmethod humanize-filter-value :inside
[root :- ::ads/root
diff --git a/test/metabase/lib/drill_thru/automatic_insights_test.cljc b/test/metabase/lib/drill_thru/automatic_insights_test.cljc
index b44f6f7299b..ec3f38c0ec4 100644
--- a/test/metabase/lib/drill_thru/automatic_insights_test.cljc
+++ b/test/metabase/lib/drill_thru/automatic_insights_test.cljc
@@ -82,7 +82,8 @@
(lib/drill-thru query -1 nil drill)))))
(deftest ^:parallel automatic-insights-apply-test
- (let [filters [[:= {} [:field {} (meta/id :orders :created-at)] "2023-12-01"]]]
+ (let [filters [[:between {} [:field {} (meta/id :orders :created-at)]
+ "2023-12-01" "2023-12-31"]]]
(testing "breakouts are turned to filters, aggregations dropped"
(auto-insights (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
(lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal)))
@@ -109,7 +110,7 @@
(lib/breakout (lib/with-temporal-bucket
(meta/field-metadata :orders :created-at)
:month)))
- [[:= {} [:field {} (meta/id :orders :created-at)] "2023-12-01"]
+ [[:between {} [:field {} (meta/id :orders :created-at)] "2023-12-01" "2023-12-31"]
[:= {} [:field {} (meta/id :products :category)] "Doohickey"]])))
(deftest ^:parallel automatic-insights-apply-test-3
@@ -129,7 +130,8 @@
(lib/breakout (lib/with-temporal-bucket
(meta/field-metadata :orders :created-at)
:month)))
- [[:= {} [:field {} (meta/id :orders :created-at)] "2023-12-01"]]))))
+ [[:between {} [:field {} (meta/id :orders :created-at)]
+ "2023-12-01" "2023-12-31"]]))))
(deftest ^:parallel binned-column-test
(testing "Automatic insights for a binned column should generate filters for current bin's min/max values"
diff --git a/test/metabase/lib/drill_thru/underlying_records_test.cljc b/test/metabase/lib/drill_thru/underlying_records_test.cljc
index 60eb00eb120..63d656cfa85 100644
--- a/test/metabase/lib/drill_thru/underlying_records_test.cljc
+++ b/test/metabase/lib/drill_thru/underlying_records_test.cljc
@@ -115,16 +115,24 @@
:value "2022-12-01T00:00:00+02:00"}]}}))
(def ^:private last-month
- #?(:cljs (let [now (js/Date.)
- year (.getFullYear now)
- month (.getMonth now)]
- (-> (js/Date.UTC year (dec month))
- (js/Date.)
- (.toISOString)))
+ ;; The JS date libraries suck. Rather than working hard to get this right in both environments for a unit test,
+ ;; here's a hard-coded value. This can be any month, provided it is in the range of the Sample Database, which is
+ ;; roughly +/- 2 years. I've picked June 2026 in October 2025, so this value should be good for a few years.
+ #?(:cljs "2026-06-01"
:clj (let [last-month (-> (t/zoned-date-time (t/year) (t/month))
(t/minus (t/months 1)))]
(t/format :iso-offset-date-time last-month))))
+;; See the note above under [[last-month]] about the hard-coded values in CLJS.
+(let [[start end] #?(:cljs ["2026-06-01" "2026-06-30"]
+ :clj (let [this-month (t/local-date (t/year) (t/month))]
+ [(t/minus this-month (t/months 1))
+ (t/minus this-month (t/days 1))]))
+ ->str #?(:cljs identity ; They're already just strings.
+ :clj #(t/format :iso-date %))]
+ (def ^:private last-month-start (->str start))
+ (def ^:private last-month-end (->str end)))
+
(defn- underlying-state [query agg-index agg-value breakout-values exp-filters-fn]
(let [columns (lib/returned-columns query)
aggs (filter #(= (:lib/source %) :source/aggregations)
@@ -167,10 +175,11 @@
42295.12
[last-month]
(fn [_agg-dim [breakout-dim]]
- [[:= {}
+ [[:between {}
(-> (:column-ref breakout-dim)
(lib.options/with-options {:temporal-unit :month}))
- last-month]]))))
+ last-month-start
+ last-month-end]]))))
(deftest ^:parallel underlying-records-apply-test-2
(testing "sum_where(subtotal, products.category = \"Doohickey\") over time"
@@ -186,10 +195,11 @@
6572.12
[last-month]
(fn [_agg-dim [breakout-dim]]
- [[:= {}
+ [[:between {}
(-> (:column-ref breakout-dim)
(lib.options/with-options {:temporal-unit :month}))
- last-month]
+ last-month-start
+ last-month-end]
[:= {} (-> (meta/field-metadata :products :category)
lib/ref
(lib.options/with-options {}))
@@ -220,7 +230,7 @@
(fn [_agg-dim [breakout-dim]]
(let [monthly-breakout (-> (:column-ref breakout-dim)
(lib.options/with-options {:temporal-unit :month}))]
- [[:= {} monthly-breakout last-month]]))))))
+ [[:between {} monthly-breakout last-month-start last-month-end]]))))))
(deftest ^:parallel multiple-aggregations-multiple-breakouts-test
(let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
@@ -239,10 +249,11 @@
(-> (:column-ref product-id-dim)
(lib.options/update-options dissoc :lib/uuid))
120]
- [:= {}
+ [:between {}
(-> (:column-ref created-at-dim)
(lib.options/with-options {:temporal-unit :month}))
- last-month]])))))
+ last-month-start
+ last-month-end]])))))
(deftest ^:parallel temporal-unit-breakouts-test
(let [column (-> (meta/field-metadata :orders :created-at)
@@ -368,9 +379,10 @@
(is (=? {:stages [{:filters [[:> {}
[:field {} (meta/id :orders :total)]
50]
- [:= {}
+ [:between {}
[:field {:temporal-unit :month} (meta/id :orders :created-at)]
- "2023-03-01T00:00:00Z"]]}]}
+ "2023-03-01"
+ "2023-03-31"]]}]}
(lib/drill-thru query drill))))))
(deftest ^:parallel negative-aggregation-values-display-info-test
@@ -512,11 +524,12 @@
:column-ref [:aggregation {:lib/source-name "sum_where_SUBTOTAL"} string?]}
drill))
(is (=? {:lib/type :mbql/query
- :stages [{:filters [[:= {}
+ :stages [{:filters [[:between {}
(-> (meta/field-metadata :orders :created-at)
lib/ref
(lib.options/with-options {}))
- "2023-12-01"]
+ "2023-12-01"
+ "2023-12-31"]
[:= {} (-> (meta/field-metadata :products :category)
lib/ref
(lib.options/with-options {}))
@@ -688,14 +701,16 @@
(lib/available-drill-thrus query context))]
(is (some? drill))
(is (=? [{:source-card 1
- :filters [[:=
+ :filters [[:between
{}
[:field {:temporal-unit :month} "CREATED_AT"]
- "2023-07-01T00:00:00Z"]
- [:=
+ "2023-07-01"
+ "2023-07-31"]
+ [:between
{}
[:field {:temporal-unit :year} "CREATED_AT"]
- "2023-01-01T00:00:00Z"]]}]
+ "2023-01-01"
+ "2023-12-31"]]}]
(:stages (lib/drill-thru query drill))))))
(deftest ^:parallel remove-original-binning-test