Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Turn whole-day dates into ranges in Metabot aggregations (#84853)

Andrei Simionescu148ec3932803
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.clj
index a7bab7f9159..0ce033a5642 100644
--- a/src/metabase/agent_lib/representations/repair.clj
+++ b/src/metabase/agent_lib/representations/repair.clj
@@ -978,6 +978,7 @@
;;; literal (resolve's `:invalid-temporal-literal` check still needs to see it). The `between`
;;; example drops the wrapper anyway: `day` on a date adds nothing, and the bound ends up as the
;;; query builder writes it.
+;;; Pass 2.6 expands date-only `day` / `default` bounds into ranges in conditional aggregations.
;;;
;;; `::query` / `::stage` are `:closed false`, so a stray `filters:` on the query root or a join map
;;; sits in no stage: E7 names the clause, never the stray key, and the model's fix is dropped again.
@@ -1294,6 +1295,98 @@
node))
form))
+;;; ============================================================
+;;; Pass 2.6 -- expand whole-day comparisons outside of filters, expressions and join conditions.
+;;;
+;;; Date-only bounds with the `day` or `default` unit become ranges on the unbucketed column,
+;;; with an exclusive upper bound on the next day. The expression editor can save these ranges;
+;;; it cannot save a bucketed column, and dropping the unit would leave out times after midnight
+;;; on the last day. Timestamps with the `default` unit only lose their wrapper, and so does the
+;;; whole-day start of a `between` that ends at `now` or at a timestamp. E7 rejects date
+;;; negations because `!=` filters keep rows with no date for Date columns and drop them for
+;;; DateTime columns, and this pass does not know the column's type.
+;;; ============================================================
+
+(defn- next-day [literal]
+ (str (u.date/add (u.date/parse literal) :day 1)))
+
+(defn- expand-day-comparison [[head opts ref & dates :as node]]
+ (let [range-ref (update ref 1 dissoc "lib/uuid")]
+ (cond
+ (= head "between")
+ ["and" opts [">=" {} range-ref (first dates)] ["<" {} range-ref (next-day (second dates))]]
+
+ (contains? #{"=" "in"} head)
+ (let [ranges (mapv (fn [date] ["and" {} [">=" {} range-ref date] ["<" {} range-ref (next-day date)]]) dates)]
+ (if (= 1 (count ranges))
+ (assoc (first ranges) 1 opts)
+ (into ["or" opts] ranges)))
+
+ (= 1 (count dates))
+ (case head
+ "<=" ["<" opts ref (next-day (first dates))]
+ ">" [">=" opts ref (next-day (first dates))]
+ node)
+
+ :else node)))
+
+(defn- whole-day-bound?
+ "Whether `v` is a `day` or `default` bound written as a full date, which a filter reads as the whole day.
+ `yyyy-MM` and `yyyy` name a month or a year, though [[hoistable-bucket]] widens them to their first day."
+ [v]
+ (and (redundant-bucket-literal v)
+ (re-matches date-only-pattern (str/trim (nth v 2)))))
+
+(defn- now-or-timestamp? [v]
+ (or (and (vector? v) (= "now" (nth v 0 nil)))
+ (and (iso-date-string? v) (not (re-matches date-only-pattern (str/trim v))))))
+
+(defn- t-separated
+ "`timestamp` with a `T` between its date and time.
+ A space there breaks the column name of an aggregation, though a filter accepts it."
+ [timestamp]
+ (str/replace-first (str/trim timestamp) " " "T"))
+
+(defn- unwrap-whole-day-start
+ "Unwrap the whole-day start of a `between` that ends at `now` or at a timestamp.
+
+ SQL `BETWEEN` then counts from midnight, as the same clause does in a filter. Any other end leaves the
+ clause to E7, since `BETWEEN` stops at the start of a plain date or of a `relative-datetime` period."
+ [[_ _ _ start end :as node]]
+ (if (and (between-clause? node) (whole-day-bound? start) (now-or-timestamp? end))
+ (cond-> (assoc node 3 (redundant-bucket-literal start))
+ (string? end) (assoc 4 (t-separated end)))
+ node))
+
+(defn- repair-day-and-default-comparison [node]
+ (if (hoistable-comparison-node? node)
+ (let [literals (mapv redundant-bucket-literal (subvec node 3))]
+ (if (every? whole-day-bound? (subvec node 3))
+ (if (contains? #{"!=" "not-in"} (first node))
+ node
+ (expand-day-comparison (into (subvec node 0 3) literals)))
+ (unwrap-whole-day-start
+ (into (subvec node 0 3)
+ (map (fn [operand literal]
+ (if (and literal (not (re-matches date-only-pattern literal))) (t-separated literal) operand))
+ (subvec node 3) literals)))))
+ node))
+
+(defn- repair-day-and-default-comparisons*
+ "Repair temporal comparisons outside of filters, expressions and join conditions."
+ [form]
+ (if (and (map? form) (vector? (get form "stages")))
+ (letfn [(rewrite [node]
+ (if (map? node)
+ (reduce-kv (fn [result k v]
+ (assoc result k (if (or (= k "conditions") (some #{k} bucket-hoist-stage-keys))
+ v
+ (rewrite v))))
+ (empty node) node)
+ (walk/walk rewrite repair-day-and-default-comparison node)))]
+ (rewrite form))
+ form))
+
;;; ============================================================
;;; Pass 1.867 -- rewrite `temporal-extract` to the `get-*` clause that means the same thing.
;;;
@@ -3229,22 +3322,23 @@
(not (parseable-temporal-literal? literal)))))
(defn- unencodable-temporal-clause-error!
- "Detect an `absolute-datetime`, `during` or unit-carrying `value` clause that Pass 2.95 could not
- rewrite.
+ "Reject temporal clauses that the repair passes could not rewrite.
[\"count-where\", {}, [\">\", {}, <col>, [\"absolute-datetime\", {}, \"2025-01-01\", \"month\"]]]
- All three hold a temporal unit in a slot the legacy schema types with `::DateUnit` /
- `::DateTimeUnit` (`metabase.legacy-mbql.schema`, via `::ValueTypeInfo` for `value`), and neither
- enum carries a `:decode/normalize`: after a JSON hop as legacy MBQL the unit is a string and the
+ The `absolute-datetime`, `during` and unit-carrying `value` clauses store their units in
+ `::DateUnit` / `::DateTimeUnit` slots in the legacy schema (`metabase.legacy-mbql.schema`, via
+ `::ValueTypeInfo` for `value`), and neither enum carries a `:decode/normalize`: after a JSON hop as
+ legacy MBQL the unit is a string and the
clause never normalizes back. The `/question#` link carries a structured query as MBQL 5, where the
clause survives the hop, but the query engine runs some of these clauses wrong by comparing the
column with the start of the literal's period. On the Sample Database, `count-where` with
`[\">\" CREATED_AT [\"absolute-datetime\" \"2026-01-01\" \"month\"]]` counts 18016 orders, though
17811 are after January 2026, and
`[\"=\" [\"absolute-datetime\" \"2026-03-01\" \"month\"] CREATED_AT]` counts none of the 256 orders
- from March 2026. Pass 2.95 rewrites every shape it can rewrite without changing the answer, and
- this detector has the model rewrite the rest.
+ from March 2026. Pass 2.95 repairs comparisons in filters and expressions.
+ Pass 2.6 expands date-only `day` / `default` bounds into ranges and unwraps `default` timestamps
+ outside of filters, expressions and join conditions. This detector has the model rewrite the rest.
`relative-datetime` and `datetime-diff` units do carry `:decode/normalize`, so they survive the hop
and are not listed. `temporal-extract`'s `::TemporalExtractUnit` has no decoder either, but Pass
@@ -3272,7 +3366,7 @@
(some? (get (nth node 1) "unit")))
false))
(throw (ex-info
- (tru "Move the temporal unit in `{0}` off the literal. Put it on the column - a `temporal-unit` option on the `field` / `expression` reference, compared to the first day of the period as a plain date string like \"2025-01-01\" - or write the range out as `>=` its first day and `<` the first day after it, both plain date strings (not `between`, which drops everything after midnight on its last day). Use the range for `year` on an `expression` reference, which the query engine reads as a number, and inside `aggregation:` (`count-where`, `sum-where`, `share`) and join `conditions:`, where moving the unit onto the column would change the answer."
+ (tru "Move the temporal unit in `{0}` off the literal. Write a range with `>=` the first day of the period and `<` the first day after the period, both as plain date strings (not `between`, which drops everything after midnight on its last day). To exclude `2025-01-31`, use an `or` of `< \"2025-01-31\"` and `>= \"2025-02-01\"` on the column. These comparisons drop rows with no date; add `is-null` on the column to the `or` if they should count. Join `conditions:` cannot hold an `or`, so a date exclusion cannot be written there; tell the user that it cannot be added to the join. You can instead put the unit on the column with `temporal-unit` in `filters:` and `expressions:`, then compare the column to the first day of the period as a plain date string. Do not put the unit on the column in `aggregation:` or join `conditions:`, where the query engine does not turn it into a range. Use the range for `year` on an `expression` reference, which the query engine reads as a number."
(pr-str node))
{:agent-error? true
:error :unencodable-temporal-clause
@@ -3355,16 +3449,19 @@
1.88. merge a trailing extra options-map back into position-1 options on fixed-arity
tuple clauses (e.g. `[\"time-interval\" {} <expr> -1 \"month\" {\"include-current\" true}]`);
2. fill in missing `\"lib/type\"` markers on the query, joins, and stages;
- 2.95. inside a stage's `filters` and `expressions` only, hoist a temporal bucket off an
- `absolute-datetime` literal and onto the ref it is compared to: the hoisted form survives a
- fallback to legacy MBQL, and the query engine runs it right where it runs the literal-side
- form wrong (examples in the pass header; runs after Pass 2.9, which creates new `filters`);
+ 2.6. expand date-only `day` / `default` comparisons into whole-day ranges on the unbucketed column outside of
+ `filters`, `expressions` and join `conditions`; leave date negations for E7; unwrap `default` timestamps
+ and the whole-day start of a `between` that ends at `now` or at a timestamp;
3. rewrite inline aggregation expressions in `order-by` to aggregation references when
they match an aggregation in the same stage's `aggregation:` list (synthesising the
referenced aggregation's `lib/uuid` if needed);
3.5. resolve 0-based integer aggregation references (`[aggregation, {}, <int>]`) in a
stage to the canonical UUID-keyed MBQL 5 form, stamping `lib/uuid` on the target
aggregation clause and `base-type`/`effective-type` on the ref's options;
+ 2.95. inside a stage's `filters` and `expressions` only, hoist a temporal bucket off an
+ `absolute-datetime` literal and onto the ref it is compared to: the hoisted form survives a
+ fallback to legacy MBQL, and the query engine runs it right where it runs the literal-side
+ form wrong (examples in the pass header; runs after Pass 2.9, which creates new `filters`);
3.7. auto-wire `source-field-join-alias` (and the accompanying portable `source-field`)
on field clauses whose target table is reachable through **exactly one** explicit
join on the stage via a single unambiguous FK on the joined table. Skips clauses
@@ -3428,6 +3525,8 @@
(-> parsed
normalize-shape*
(stamp-top-level-database* mp)
+ ;; Before Pass 3: order-by must equal its aggregation, so date bounds are normalized first.
+ repair-day-and-default-comparisons*
rewrite-order-by-inline-aggs*
resolve-aggregation-ref-indexes*
split-post-agg-filters*
diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.clj
index 48e93c31816..131c32a10b4 100644
--- a/test/metabase/agent_lib/representations/repair_test.clj
+++ b/test/metabase/agent_lib/representations/repair_test.clj
@@ -1075,6 +1075,148 @@
(let [once (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "month")])]
(is (= once (repair-filter once))))))
+(deftest ^:parallel conditional-aggregations-with-temporal-buckets-test
+ (doseq [operator ["count-where" "sum-where" "distinct-where" "share"]
+ [start-unit end-unit] [["day" "day"] ["default" "default"] ["day" "default"] ["default" "day"]]]
+ (testing (str operator " with " start-unit "/" end-unit)
+ (let [args (cond-> [operator {}]
+ (contains? #{"sum-where" "distinct-where"} operator)
+ (conj ["field" {} ["Sample" "PUBLIC" "ORDERS" "USER_ID"]]))
+ input (conj args ["between" {} created-at (abs-dt "2026-01-01" start-unit) (abs-dt "2026-01-31" end-unit)])
+ expected (conj args ["and" {} [">=" {} created-at "2026-01-01"] ["<" {} created-at "2026-02-01"]])
+ stage (repair-in-stage {"aggregation" [input]})]
+ (is (= [expected] (get stage "aggregation")))
+ (is (= stage (repair-in-stage stage)))))))
+
+(deftest ^:parallel date-only-comparisons-use-whole-day-ranges-test
+ (doseq [unit ["day" "default"]
+ :let [date (fn [literal] (abs-dt literal unit))
+ jan31 ["and" {} [">=" {} created-at "2017-01-31"] ["<" {} created-at "2017-02-01"]]
+ leap-day ["and" {} [">=" {} created-at "2024-02-29"] ["<" {} created-at "2024-03-01"]]]
+ [input expected] [[["=" {} created-at (date "2017-01-31")] jan31]
+ [["=" {} created-at (date "2017-01-31") (date "2024-02-29")] ["or" {} jan31 leap-day]]
+ [["in" {} created-at (date "2017-01-31") (date "2024-02-29")] ["or" {} jan31 leap-day]]
+ [["<=" {} created-at (date "2017-01-31")] ["<" {} created-at "2017-02-01"]]
+ [[">" {} created-at (date "2017-12-31")] [">=" {} created-at "2018-01-01"]]
+ [["<" {} created-at (date "2017-01-31")] ["<" {} created-at "2017-01-31"]]
+ [[">=" {} created-at (date "2017-01-31")] [">=" {} created-at "2017-01-31"]]]]
+ (testing (pr-str input)
+ (let [stage (repair-in-stage {"aggregation" [["count-where" {} input]]})]
+ (is (= [["count-where" {} expected]] (get stage "aggregation")))
+ (is (= stage (repair-in-stage stage)))))))
+
+(deftest ^:parallel whole-day-ranges-in-order-by-and-joined-aggregations-test
+ (let [predicate ["between" {} created-at (abs-dt "2017-01-01" "day") (abs-dt "2017-01-31" "default")]
+ expected ["and" {} [">=" {} created-at "2017-01-01"] ["<" {} created-at "2017-02-01"]]]
+ (testing "an inline order-by aggregation"
+ (let [stage (repair-in-stage {"order-by" [["desc" {} ["count-where" {} predicate]]]})]
+ (is (= ["count-where" {} expected] (get-in stage ["order-by" 0 2])))))
+ (testing "an aggregation in a join's source query"
+ (let [stage (repair-in-stage
+ {"joins" [{"lib/type" "mbql/join" "alias" "O2"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "aggregation" [["count-where" {} predicate]]}]}]})]
+ (is (= [["count-where" {} expected]] (get-in stage ["joins" 0 "stages" 0 "aggregation"])))))))
+
+(deftest ^:parallel inline-order-by-with-equivalent-temporal-aggregations-test
+ (let [range-clause ["and" {} [">=" {} created-at "2017-01-01"] ["<" {} created-at "2017-02-01"]]]
+ (doseq [[aggregation-predicate order-predicate]
+ [[["between" {} created-at (abs-dt "2017-01-01" "day") (abs-dt "2017-01-31" "day")]
+ ["between" {} created-at (abs-dt "2017-01-01" "default") (abs-dt "2017-01-31" "default")]]
+ [["between" {} created-at (abs-dt "2017-01-01" "day") (abs-dt "2017-01-31" "day")]
+ range-clause]
+ [range-clause
+ ["between" {} created-at (abs-dt "2017-01-01" "day") (abs-dt "2017-01-31" "day")]]
+ [[">=" {} created-at (abs-dt "2017-01-01T12:00:00" "default")]
+ [">=" {} created-at "2017-01-01T12:00:00"]]]]
+ (testing (pr-str [aggregation-predicate order-predicate])
+ (let [stage (repair-in-stage {"aggregation" [["count-where" {} aggregation-predicate]]
+ "order-by" [["desc" {} ["count-where" {} order-predicate]]]})
+ uuid (get-in stage ["aggregation" 0 1 "lib/uuid"])]
+ (is (some? uuid))
+ (is (= ["aggregation" {} uuid] (get-in stage ["order-by" 0 2])))
+ (is (= stage (repair-in-stage stage))))))))
+
+(deftest ^:parallel join-conditions-with-temporal-literals-stay-refused-test
+ (doseq [operator ["in" "=" "between"]
+ [start-unit end-unit] [["day" "day"] ["default" "default"] ["day" "default"] ["default" "day"]]]
+ (testing (str operator " with " start-unit "/" end-unit)
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Move the temporal unit"
+ (repair-in-stage
+ {"joins" [{"lib/type" "mbql/join" "alias" "O2"
+ "stages" [{"lib/type" "mbql.stage/mbql" "source-table" ["Sample" "PUBLIC" "ORDERS"]}]
+ "conditions" [[operator {} created-at
+ (abs-dt "2017-01-30" start-unit)
+ (abs-dt "2017-01-31" end-unit)]]}]}))))))
+
+(deftest ^:parallel default-timestamps-keep-their-time-test
+ (doseq [separator ["T" " "]]
+ (let [input ["between" {} created-at (abs-dt (str "2026-01-01" separator "12:30:00") "default")
+ (abs-dt (str "2026-01-31" separator "12:30:00") "default")]
+ expected ["between" {} created-at "2026-01-01T12:30:00" "2026-01-31T12:30:00"]]
+ (is (= [["count-where" {} expected]]
+ (get (repair-in-stage {"aggregation" [["count-where" {} input]]}) "aggregation"))
+ (pr-str input)))))
+
+(deftest ^:parallel between-a-whole-day-and-now-or-a-timestamp-test
+ (doseq [unit ["day" "default"]
+ [end end'] [[["now" {}] ["now" {}]]
+ ["now" ["now" {}]]
+ [(abs-dt "2026-01-31T12:30:00" "default") "2026-01-31T12:30:00"]
+ ["2026-01-31T12:30:00" "2026-01-31T12:30:00"]
+ [(abs-dt "2026-01-31 12:30:00" "default") "2026-01-31T12:30:00"]
+ ["2026-01-31 12:30:00" "2026-01-31T12:30:00"]]]
+ (testing (pr-str [unit end])
+ (let [stage (repair-in-stage
+ {"aggregation" [["count-where" {} ["between" {} created-at (abs-dt "2026-01-01" unit) end]]]})]
+ (is (= [["count-where" {} ["between" {} created-at "2026-01-01" end']]] (get stage "aggregation")))
+ (is (= stage (repair-in-stage stage)))))))
+
+(deftest ^:parallel conditional-aggregations-with-refused-temporal-clauses-test
+ (doseq [predicate (concat
+ (for [[literal units] [["2026-01-01" ["week" "month" "quarter" "year"]]
+ ["2026-01-01T10:00:00" ["hour" "minute" "second"]]]
+ unit units]
+ [">" {} created-at (abs-dt literal unit)])
+ [["!=" {} created-at (abs-dt "2026-01-31" "day")]
+ ["not-in" {} created-at (abs-dt "2026-01-30" "default") (abs-dt "2026-01-31" "day")]
+ [">" {} created-at (abs-dt "2026-01-01T12:30:00" "day")]
+ ["during" {} created-at "2026-01-01" "day"]
+ ["between" {} created-at (abs-dt "2026-01-01" "day") (abs-dt "2026-01-31" "month")]
+ ["between" {} created-at (abs-dt "2026-01-01" "day") "2026-01-31"]
+ ["between" {} created-at (abs-dt "2026-01-01" "day") ["relative-datetime" {} -1 "day"]]
+ ["between" {} created-at (abs-dt "2026-01-01" "day") ["today" {}]]
+ ["between" {} created-at (abs-dt "2026-01-01T12:30:00" "default") (abs-dt "2026-01-31" "default")]
+ ["=" {} (created-at-bucketed "day") (abs-dt "2026-01-01" "day")]
+ ["=" {} (abs-dt "2026-01-01" "day") created-at]])]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Move the temporal unit"
+ (repair-in-stage {"aggregation" [["count-where" {} predicate]]})))))
+
+(deftest ^:parallel partial-dates-in-conditional-aggregations-stay-refused-test
+ (let [user-id ["field" {} ["Sample" "PUBLIC" "ORDERS" "USER_ID"]]]
+ (doseq [unit ["day" "default"]
+ [start end] [["2026-01" "2026-03"] ["2025" "2026"]]
+ predicate [["=" {} created-at (abs-dt start unit)]
+ ["between" {} created-at (abs-dt start unit) (abs-dt end unit)]
+ ["between" {} created-at (abs-dt start unit) ["now" {}]]]
+ aggregation [["count-where" {} predicate]
+ ["sum-where" {} user-id predicate]
+ ["distinct-where" {} user-id predicate]
+ ["share" {} predicate]]]
+ (testing (pr-str aggregation)
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Move the temporal unit"
+ (repair-in-stage {"aggregation" [aggregation]})))))))
+
+(deftest ^:parallel standalone-temporal-literals-keep-their-type-test
+ (let [literal (abs-dt "2026-01-01" "day")]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Move the temporal unit"
+ (repair-in-stage {"aggregation" [["min" {} literal]]})))))
+
(deftest ^:parallel unencodable-temporal-clause-error-test
(testing "a temporal clause the hoist pass could not move becomes a retryable agent error"
(doseq [[label stage-kvs]
@@ -1140,8 +1282,26 @@
;; plain upper date keeps only midnight of the last day
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
- #"`>=` its first day and `<` the first day after it.*not `between`.*`year` on an `expression` reference"
- (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-01" "year")])))))
+ #"`>=` the first day of the period and `<` the first day after the period.*not `between`.*`year` on an `expression` reference"
+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-01" "year")]))))
+ (testing "date negations leave the choice of counting NULL dates to the model"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"These comparisons drop rows with no date; add `is-null` on the column to the `or` if they should count"
+ (repair-in-stage {"aggregation" [["count-where" {} ["!=" {} created-at (abs-dt "2025-01-31" "day")]]]}))))
+ (testing "a date exclusion in a join has no rewrite, so the model tells the user"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Join `conditions:` cannot hold an `or`, so a date exclusion cannot be written there; tell the user that it cannot be added to the join."
+ (repair-in-stage
+ {"joins" [{"lib/type" "mbql/join" "alias" "O2"
+ "stages" [{"lib/type" "mbql.stage/mbql" "source-table" ["Sample" "PUBLIC" "ORDERS"]}]
+ "conditions" [["!=" {} created-at (abs-dt "2025-01-31" "day")]]}]}))))
+ (testing "a bucketed column is ruled out where the query engine does not turn it into a range"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Do not put the unit on the column in `aggregation:` or join `conditions:`"
+ (repair-in-stage {"aggregation" [["count-where" {} [">" {} created-at (abs-dt "2025-01-01" "month")]]]})))))
;;; ============================================================
;;; Pass 1.867 - rewrite temporal-extract to its get-* equivalent
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index 729bd2e3aa6..87b46597d17 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -1944,14 +1944,105 @@
;; assertion below could hold while proving nothing.
(is (= "query" (:type wire))
"->legacy-mbql fell back to MBQL 5 instead of converting")
- ;; ... and `lib/query` does not throw on a wire whose clause `clean-stage-schema-errors` deleted,
- ;; so "no throw" is not survival. Compare the clauses - by count, since uuids and idents are
- ;; regenerated across the hop and deep equality is not available. A clause swapped for a different
- ;; clause of the same arity would slip through; deletion, the failure mode here, would not.
- (let [after (lib/query mp-temporal wire)]
+ ;; `lib/query` silently drops invalid clauses, so reopening alone does not prove survival.
+ (let [after (lib/query mp-temporal (lib.convert/js-legacy-query->mbql5 wire))]
(is (map? after) "did not survive the JSON round-trip")
(is (= (stage-clause-counts before) (stage-clause-counts after))
- "a clause was silently dropped by the round-trip")))
+ "a clause was silently dropped by the round-trip")
+ after))
+
+(deftest conditional-aggregations-with-date-ranges-test
+ (with-temporal-mp-and-stubs!
+ (fn []
+ (doseq [operator ["count-where" "sum-where" "distinct-where" "share"]
+ [start-unit end-unit] [["day" "day"] ["default" "default"] ["day" "default"] ["default" "day"]]]
+ (testing (str operator " with " start-unit "/" end-unit)
+ (let [aggregations (mapv (fn [[label start end]]
+ (let [predicate ["between" {}
+ ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ ["absolute-datetime" {} start start-unit]
+ ["absolute-datetime" {} end end-unit]]]
+ (cond-> [operator {"display-name" label}]
+ (contains? #{"sum-where" "distinct-where"} operator)
+ (conj ["field" {} ["Sample" "PUBLIC" "ORDERS" "USER_ID"]])
+ true (conj predicate))))
+ [["Jan 2026" "2026-01-01" "2026-01-31"]
+ ["Mar 2026" "2026-03-01" "2026-03-31"]])
+ [before :as hop] (legacy-json-wire
+ {"aggregation" aggregations
+ "breakout" [["field" {} ["Sample" "PUBLIC" "ORDERS" "STATUS"]]]
+ "order-by" [["desc" {} ["aggregation" {} 0]]]})
+ after (assert-survives-json-hop! hop)]
+ (is (= ["Jan 2026" "Mar 2026"]
+ (mapv :display-name (take-last 2 (lib/returned-columns before)))))
+ (is (= (lib/remove-lib-uuids (lib/aggregations before))
+ (lib/remove-lib-uuids (lib/aggregations after))))))))))
+
+(deftest conditional-date-range-with-column-uuid-test
+ (with-temporal-mp-and-stubs!
+ (fn []
+ (let [[before :as hop] (legacy-json-wire
+ {"aggregation"
+ [["count-where" {}
+ ["between" {}
+ ["field" {"lib/uuid" "8b1a3b2c-0000-4000-8000-000000000001"}
+ ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ ["absolute-datetime" {} "2017-01-01" "day"]
+ ["absolute-datetime" {} "2017-01-31" "day"]]]]})
+ after (assert-survives-json-hop! hop)]
+ (is (= (lib/remove-lib-uuids (lib/aggregations before))
+ (lib/remove-lib-uuids (lib/aggregations after))))))))
+
+(deftest conditional-aggregations-count-whole-days-test
+ (mt/dataset test-data
+ (mt/with-test-user :crowberto
+ (mt/with-dynamic-fn-redefs [construct/resolve-database-id-from-first-stage (fn [_] (mt/id))]
+ (let [db-name (:name (mt/db))
+ created-at ["field" {} [db-name "PUBLIC" "ORDERS" "CREATED_AT"]]
+ user-id ["field" {} [db-name "PUBLIC" "ORDERS" "USER_ID"]]
+ result (fn [stage-kvs]
+ (-> (construct/execute-representations-query
+ (query-data
+ {"lib/type" "mbql/query"
+ "stages" [(merge {"lib/type" "mbql.stage/mbql"
+ "source-table" [db-name "PUBLIC" "ORDERS"]}
+ stage-kvs)]}))
+ (get-in [:structured-output :query])
+ qp/process-query mt/rows first))
+ total (first (result {"aggregation" [["count" {}]]}))]
+ (doseq [[start-unit end-unit] [["day" "day"] ["default" "default"] ["day" "default"] ["default" "day"]]
+ :let [start (fn [literal] ["absolute-datetime" {} literal start-unit])
+ end (fn [literal] ["absolute-datetime" {} literal end-unit])]
+ [predicate expected-count]
+ [[["between" {} created-at (start "2017-01-01") (end "2017-01-31")] 205]
+ [["=" {} created-at (end "2017-01-31")] 8]
+ [["=" {} created-at (start "2017-01-30") (end "2017-01-31")] 14]
+ [["in" {} created-at (start "2017-01-30") (end "2017-01-31")] 14]
+ [["<=" {} created-at (end "2017-01-31")] 949]
+ [[">" {} created-at (start "2017-01-30")] 17819]
+ [["<" {} created-at (end "2017-02-01")] 949]
+ [[">=" {} created-at (start "2017-01-31")] 17819]
+ [["between" {} created-at (start "2017-01-31") ["now" {}]] 17819]
+ [["between" {} created-at (start "2017-01-01") ["absolute-datetime" {} "2017-01-31T12:30:00" "default"]] 203]
+ [["between" {} created-at (start "2017-01-01") ["absolute-datetime" {} "2017-01-31 12:30:00" "default"]] 203]
+ [["between" {} created-at (start "2017-01-01") "2017-01-31 12:30:00"] 203]]]
+ (testing (pr-str predicate)
+ (let [;; Separate filters let Pass 2.95 handle dates with different units.
+ filter-predicate (if (and (contains? #{"=" "in"} (first predicate)) (> (count predicate) 4))
+ (into ["or" {}] (map (fn [date] ["=" {} created-at date]) (drop 3 predicate)))
+ predicate)
+ [filtered-count filtered-sum filtered-distinct]
+ (result {"aggregation" [["count" {}] ["sum" {} user-id] ["distinct" {} user-id]]
+ "filters" [filter-predicate]})
+ [conditional-count conditional-sum conditional-distinct conditional-share]
+ (result {"aggregation" [["count-where" {} predicate]
+ ["sum-where" {} user-id predicate]
+ ["distinct-where" {} user-id predicate]
+ ["share" {} predicate]]})]
+ (is (== expected-count filtered-count conditional-count))
+ (is (== filtered-sum conditional-sum))
+ (is (== filtered-distinct conditional-distinct))
+ (is (< (abs (- (double (/ filtered-count total)) (double conditional-share))) 1e-9))))))))))
;; A Metabot-built question reaches the frontend as legacy MBQL inside a base64 `/question#` hash,
;; so it makes a JSON hop that turns every keyword into a string. Legacy normalization has no