The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[MBQL lib] Use `:between` filters for bucketed time fields (#64218)
Braden Shepherdson21e9d33c668d
diff --git a/src/metabase/lib/drill_thru/underlying_records.cljc b/src/metabase/lib/drill_thru/underlying_records.cljcindex f462e1fa939..8bf3107da9e 100644--- a/src/metabase/lib/drill_thru/underlying_records.cljc+++ b/src/metabase/lib/drill_thru/underlying_records.cljc@@ -34,6 +34,7 @@[metabase.lib.aggregation :as lib.aggregation][metabase.lib.binning :as lib.binning][metabase.lib.drill-thru.common :as lib.drill-thru.common]+ [metabase.lib.fe-util :as lib.fe-util][metabase.lib.filter :as lib.filter][metabase.lib.join :as lib.join][metabase.lib.metadata :as lib.metadata]@@ -43,6 +44,7 @@[metabase.lib.schema.drill-thru :as lib.schema.drill-thru][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.ref :as lib.schema.ref]+ [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing][metabase.lib.temporal-bucket :as lib.temporal-bucket][metabase.lib.types.isa :as lib.types.isa][metabase.lib.underlying :as lib.underlying]@@ -142,13 +144,18 @@;; instead of;;;; month(col) = March 2023- (let [column (if-let [temporal-unit (or (::lib.underlying/temporal-unit column)- (lib.temporal-bucket/temporal-bucket column))]- (lib.temporal-bucket/with-temporal-bucket filter-column temporal-unit)+ (let [bucket (or (::lib.underlying/temporal-unit column)+ (lib.temporal-bucket/temporal-bucket column))+ unit (cond-> bucket+ (map? bucket) :unit)+ column (if unit+ (lib.temporal-bucket/with-temporal-bucket filter-column unit)filter-column)]- (if (some? value)- [(lib.filter/= column value)]- [(lib.filter/is-null column)])))]+ (if (nil? value)+ [(lib.filter/is-null column)]+ [(cond-> (lib.filter/= column value)+ (and unit (lib.schema.temporal-bucketing/datetime-truncation-units unit))+ lib.fe-util/expand-temporal-expression)])))](reduce(fn [query filter-clause](lib.filter/filter query stage-number filter-clause))diff --git a/src/metabase/lib/fe_util.cljc b/src/metabase/lib/fe_util.cljcindex a097fa1e2ee..dffcfe1308c 100644--- a/src/metabase/lib/fe_util.cljc+++ b/src/metabase/lib/fe_util.cljc@@ -73,7 +73,7 @@(:temporal-unit (lib.options/options maybe-clause-arg)))(u.time/timestamp-coercible? other-arg))))-(defn- expand-temporal-expression+(defn expand-temporal-expression"Modify expression in a way, that its resulting [[expression-parts]] are digestable by filter picker.Current filter picker implementation is unable to handle expression parts of expressions of a formdiff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.cljindex baaaaed9bde..c5d67a254b1 100644--- a/src/metabase/xrays/automagic_dashboards/combination.clj+++ b/src/metabase/xrays/automagic_dashboards/combination.clj@@ -163,8 +163,8 @@[{:keys [cell-query]}](letfn [(collect-dimensions [[op & args]](case (some-> op qp.util/normalize-token)- :and (mapcat collect-dimensions args)- := (magic.util/collect-field-references args)+ :and (mapcat collect-dimensions args)+ (:between :=) (magic.util/collect-field-references args)nil))](->> cell-querycollect-dimensionsdiff --git a/src/metabase/xrays/automagic_dashboards/filters.clj b/src/metabase/xrays/automagic_dashboards/filters.cljindex a01cddb4d90..ff4e7ffdf2f 100644--- a/src/metabase/xrays/automagic_dashboards/filters.clj+++ b/src/metabase/xrays/automagic_dashboards/filters.clj@@ -13,7 +13,8 @@(defn- temporal?"Does `field` represent a temporal value, i.e. a date, time, or datetime?"[{base-type :base_type, effective-type :effective_type, unit :unit}]- ;; TODO -- not sure why we're excluding year here? Is it because we normally returned it as an integer in the past?+ ;; Excluding :year because it's (currently) both an extraction and truncation unit.+ ;; For the purposes of this check, :year is an interesting :unit which yields a time interval, not just a number.(and (not ((disj u.date/extract-units :year) unit))(isa? (or effective-type base-type) :type/Temporal)))diff --git a/src/metabase/xrays/automagic_dashboards/names.clj b/src/metabase/xrays/automagic_dashboards/names.cljindex 0973a863945..82e045bcbc4 100644--- a/src/metabase/xrays/automagic_dashboards/names.clj+++ b/src/metabase/xrays/automagic_dashboards/names.clj@@ -204,7 +204,13 @@(mu/defmethod humanize-filter-value :between[root :- ::ads/root[_ field-reference min-value max-value]]- (tru "{0} is between {1} and {2}" (item-name root field-reference) min-value max-value))+ (let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]+ (or (when (isa? (or effective_type base_type) :type/Temporal)+ (let [humanized-min (humanize-datetime min-value unit)+ humanized-max (humanize-datetime max-value unit)]+ (when (= humanized-min humanized-max)+ (tru "{0} is {1}" item-name humanized-min))))+ (tru "{0} is between {1} and {2}" item-name min-value max-value))))(mu/defmethod humanize-filter-value :inside[root :- ::ads/rootdiff --git a/test/metabase/lib/drill_thru/automatic_insights_test.cljc b/test/metabase/lib/drill_thru/automatic_insights_test.cljcindex b44f6f7299b..ec3f38c0ec4 100644--- a/test/metabase/lib/drill_thru/automatic_insights_test.cljc+++ b/test/metabase/lib/drill_thru/automatic_insights_test.cljc@@ -82,7 +82,8 @@(lib/drill-thru query -1 nil drill)))))(deftest ^:parallel automatic-insights-apply-test- (let [filters [[:= {} [:field {} (meta/id :orders :created-at)] "2023-12-01"]]]+ (let [filters [[:between {} [:field {} (meta/id :orders :created-at)]+ "2023-12-01" "2023-12-31"]]](testing "breakouts are turned to filters, aggregations dropped"(auto-insights (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))(lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal)))@@ -109,7 +110,7 @@(lib/breakout (lib/with-temporal-bucket(meta/field-metadata :orders :created-at):month)))- [[:= {} [:field {} (meta/id :orders :created-at)] "2023-12-01"]+ [[:between {} [:field {} (meta/id :orders :created-at)] "2023-12-01" "2023-12-31"][:= {} [:field {} (meta/id :products :category)] "Doohickey"]])))(deftest ^:parallel automatic-insights-apply-test-3@@ -129,7 +130,8 @@(lib/breakout (lib/with-temporal-bucket(meta/field-metadata :orders :created-at):month)))- [[:= {} [:field {} (meta/id :orders :created-at)] "2023-12-01"]]))))+ [[:between {} [:field {} (meta/id :orders :created-at)]+ "2023-12-01" "2023-12-31"]]))))(deftest ^:parallel binned-column-test(testing "Automatic insights for a binned column should generate filters for current bin's min/max values"diff --git a/test/metabase/lib/drill_thru/underlying_records_test.cljc b/test/metabase/lib/drill_thru/underlying_records_test.cljcindex 60eb00eb120..63d656cfa85 100644--- a/test/metabase/lib/drill_thru/underlying_records_test.cljc+++ b/test/metabase/lib/drill_thru/underlying_records_test.cljc@@ -115,16 +115,24 @@:value "2022-12-01T00:00:00+02:00"}]}}))(def ^:private last-month- #?(:cljs (let [now (js/Date.)- year (.getFullYear now)- month (.getMonth now)]- (-> (js/Date.UTC year (dec month))- (js/Date.)- (.toISOString)))+ ;; The JS date libraries suck. Rather than working hard to get this right in both environments for a unit test,+ ;; here's a hard-coded value. This can be any month, provided it is in the range of the Sample Database, which is+ ;; roughly +/- 2 years. I've picked June 2026 in October 2025, so this value should be good for a few years.+ #?(:cljs "2026-06-01":clj (let [last-month (-> (t/zoned-date-time (t/year) (t/month))(t/minus (t/months 1)))](t/format :iso-offset-date-time last-month))))+;; See the note above under [[last-month]] about the hard-coded values in CLJS.+(let [[start end] #?(:cljs ["2026-06-01" "2026-06-30"]+ :clj (let [this-month (t/local-date (t/year) (t/month))]+ [(t/minus this-month (t/months 1))+ (t/minus this-month (t/days 1))]))+ ->str #?(:cljs identity ; They're already just strings.+ :clj #(t/format :iso-date %))]+ (def ^:private last-month-start (->str start))+ (def ^:private last-month-end (->str end)))+(defn- underlying-state [query agg-index agg-value breakout-values exp-filters-fn](let [columns (lib/returned-columns query)aggs (filter #(= (:lib/source %) :source/aggregations)@@ -167,10 +175,11 @@42295.12[last-month](fn [_agg-dim [breakout-dim]]- [[:= {}+ [[:between {}(-> (:column-ref breakout-dim)(lib.options/with-options {:temporal-unit :month}))- last-month]]))))+ last-month-start+ last-month-end]]))))(deftest ^:parallel underlying-records-apply-test-2(testing "sum_where(subtotal, products.category = \"Doohickey\") over time"@@ -186,10 +195,11 @@6572.12[last-month](fn [_agg-dim [breakout-dim]]- [[:= {}+ [[:between {}(-> (:column-ref breakout-dim)(lib.options/with-options {:temporal-unit :month}))- last-month]+ last-month-start+ last-month-end][:= {} (-> (meta/field-metadata :products :category)lib/ref(lib.options/with-options {}))@@ -220,7 +230,7 @@(fn [_agg-dim [breakout-dim]](let [monthly-breakout (-> (:column-ref breakout-dim)(lib.options/with-options {:temporal-unit :month}))]- [[:= {} monthly-breakout last-month]]))))))+ [[:between {} monthly-breakout last-month-start last-month-end]]))))))(deftest ^:parallel multiple-aggregations-multiple-breakouts-test(let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))@@ -239,10 +249,11 @@(-> (:column-ref product-id-dim)(lib.options/update-options dissoc :lib/uuid))120]- [:= {}+ [:between {}(-> (:column-ref created-at-dim)(lib.options/with-options {:temporal-unit :month}))- last-month]])))))+ last-month-start+ last-month-end]])))))(deftest ^:parallel temporal-unit-breakouts-test(let [column (-> (meta/field-metadata :orders :created-at)@@ -368,9 +379,10 @@(is (=? {:stages [{:filters [[:> {}[:field {} (meta/id :orders :total)]50]- [:= {}+ [:between {}[:field {:temporal-unit :month} (meta/id :orders :created-at)]- "2023-03-01T00:00:00Z"]]}]}+ "2023-03-01"+ "2023-03-31"]]}]}(lib/drill-thru query drill))))))(deftest ^:parallel negative-aggregation-values-display-info-test@@ -512,11 +524,12 @@:column-ref [:aggregation {:lib/source-name "sum_where_SUBTOTAL"} string?]}drill))(is (=? {:lib/type :mbql/query- :stages [{:filters [[:= {}+ :stages [{:filters [[:between {}(-> (meta/field-metadata :orders :created-at)lib/ref(lib.options/with-options {}))- "2023-12-01"]+ "2023-12-01"+ "2023-12-31"][:= {} (-> (meta/field-metadata :products :category)lib/ref(lib.options/with-options {}))@@ -688,14 +701,16 @@(lib/available-drill-thrus query context))](is (some? drill))(is (=? [{:source-card 1- :filters [[:=+ :filters [[:between{}[:field {:temporal-unit :month} "CREATED_AT"]- "2023-07-01T00:00:00Z"]- [:=+ "2023-07-01"+ "2023-07-31"]+ [:between{}[:field {:temporal-unit :year} "CREATED_AT"]- "2023-01-01T00:00:00Z"]]}]+ "2023-01-01"+ "2023-12-31"]]}](:stages (lib/drill-thru query drill))))))(deftest ^:parallel remove-original-binning-test