Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Stop Metabot emitting filters legacy MBQL can't normalize (#81947)

Paolo Rascunàe85197580628
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.clj
index f4bd9025b93..b24b3cde08c 100644
--- a/src/metabase/agent_lib/representations/repair.clj
+++ b/src/metabase/agent_lib/representations/repair.clj
@@ -34,9 +34,12 @@
[metabase.lib.expression :as lib.expression]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.schema.mbql-clause :as mbql-clause]
+ [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]
+ [metabase.lib.walk :as lib.walk]
[metabase.models.serialization.resolve :as resolve]
[metabase.models.serialization.resolve.mp :as resolve.mp]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
@@ -311,7 +314,7 @@
;;; ============================================================
;;; Pass 1.87 -- rewrite known misspelled `lib/type` markers to their canonical value.
;;;
-;;; Pass 2's `infer-*` helpers only FILL a missing marker; they never rewrite a present one.
+;;; Pass 2's `ensure-lib-types*` only FILLS a missing marker; it never rewrites a present one.
;;; This pass handles the present-but-wrong case via a small alias table, exactly like the
;;; `rewrite-operator-name-aliases*` / `rewrite-temporal-bucket-aliases*` passes.
;;; ============================================================
@@ -802,29 +805,16 @@
(&recur (-> &match (assoc 3 (nth &match 4)) (assoc 4 (nth &match 3))))))
;;; ============================================================
-;;; Pass 1.86 -- wrap bare ISO-date string bounds in `between` clauses as
-;;; `[absolute-datetime, {}, <iso-str>, "day"]`.
-;;;
-;;; LLMs frequently write `[between, {}, <date-field>, "2024-01-01", "2024-12-31"]`,
-;;; using bare strings as the bounds. lib's `:between` schema demands a temporal
-;;; expression on each side once any side is temporal; bare strings won't satisfy
-;;; `:type/Date`. We detect the case where at least one of the two bounds matches the
-;;; ISO-8601 `yyyy-mm-dd` pattern and wrap each matching string as an
-;;; `["absolute-datetime" {} <iso-str> "day"]` clause. Carried over from the sexp
-;;; pipeline's `wrap-iso-date-as-absolute-datetime` (see
-;;; `repr-deletion-followups.md` § 1.6). High-frequency LLM pattern.
-;;;
-;;; Idempotency: after wrap, bounds are vectors, so the predicate (string + ISO regex)
-;;; no longer matches.
+;;; Temporal-literal helpers, shared by the passes below.
;;; ============================================================
-(def ^:private iso-date-pattern
+(def ^:private iso-date-or-datetime-pattern
"Recognises an ISO-8601 calendar-date string (yyyy-mm-dd, optionally with a time portion)."
#"\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+\-]\d{2}:?\d{2})?)?")
(defn- iso-date-string? [v]
(and (string? v)
- (re-matches iso-date-pattern (str/trim v))))
+ (re-matches iso-date-or-datetime-pattern (str/trim v))))
(defn- temporal-clause-head?
"Heads that are unambiguously temporal-shaped clauses."
@@ -840,37 +830,6 @@
(string? (nth v 0))
(temporal-clause-head? (nth v 0))))
-(defn- wrap-iso-date [v]
- (if (iso-date-string? v)
- ["absolute-datetime" {} (str/trim v) "day"]
- v))
-
-(defn- between-needs-iso-wrap?
- "Trigger the ISO-wrap when at least one bound is already a temporal-shaped clause OR
- at least one bound is an ISO-date string. The first case mirrors sexp's behaviour
- (`temporal-expression?` on either side), the second case is a small extension - if
- both bounds are bare ISO date strings the structure is unambiguous and bare strings
- would fail validation anyway."
- [lo hi]
- (or (temporal-shaped-clause? lo)
- (temporal-shaped-clause? hi)
- (iso-date-string? lo)
- (iso-date-string? hi)))
-
-(defn- between-needs-iso-wrap-clause? [node]
- (and (between-clause? node)
- (between-needs-iso-wrap? (nth node 3) (nth node 4))))
-
-(defn- wrap-iso-date-bounds*
- [form]
- ;; No `&recur`: a wrapped ISO bound becomes an `absolute-datetime` clause, which is itself
- ;; temporal-shaped, so `between-needs-iso-wrap-clause?` would fire on the result forever.
- (match/replace form
- (_ :guard between-needs-iso-wrap-clause?)
- (-> &match
- (assoc 3 (wrap-iso-date (nth &match 3)))
- (assoc 4 (wrap-iso-date (nth &match 4))))))
-
;;; ============================================================
;;; Pass 1.865 -- wrap bare `"now"` string literals in temporal contexts as the canonical
;;; `["now" {}]` clause.
@@ -900,7 +859,11 @@
(defn- temporal-context-operand? [v]
(or (temporal-shaped-clause? v)
- (field-with-temporal-unit? v)))
+ (field-with-temporal-unit? v)
+ ;; A bare ISO-date bound counts as a temporal sibling: in `["between" {} <field>
+ ;; "2024-01-01" "now"]` neither operand is a clause, and without this the `"now"`
+ ;; would survive as a bare string and blow up in `wrap-value-literals`.
+ (iso-date-string? v)))
(def ^:private temporal-comparison-heads
"Comparison-style heads for which we'll wrap a bare `\"now\"` literal in another operand
@@ -939,6 +902,458 @@
node))
form))
+;;; ============================================================
+;;; Structural predicates -- what a bare map looks like, with no `lib/type` marker to go on.
+;;;
+;;; Shared by Pass 2.95, which hoists only inside a stage, and Pass 2, which stamps a missing
+;;; marker on one. One definition, so the hoist's notion of a stage is the one that stamps the
+;;; marker -- a marker the model wrote can be wrong, a derivation cannot disagree with itself.
+;;; ============================================================
+
+(defn- top-level-query-map?
+ [m]
+ (match/matches? m {"database" _, "stages" _}))
+
+(defn- join-like-map?
+ "A map that looks like an explicit join: it carries join-only keys (`conditions`, or
+ `alias`+`stages`) that never appear on a stage or top-level query. We key on these rather
+ than on `\"fields\"` (which a join shares with a stage) so a join is not mistaken for a
+ stage by [[stage-like-map?]]."
+ [m]
+ (and (map? m)
+ (or (contains? m "conditions")
+ (and (contains? m "alias")
+ (contains? m "stages")))))
+
+(defn- stage-like-map?
+ "A map that looks like an MBQL stage: has `\"source-table\"`, `\"source-card\"`, or any of the
+ stage-body keys (`filters`, `aggregation`, `breakout`, `order-by`, `fields`, `joins`,
+ `expressions`, `limit`). Not a top-level query and not an explicit join (a join can carry
+ `\"fields\"`, so we exclude it explicitly)."
+ [m]
+ (and (map? m)
+ (not (top-level-query-map? m))
+ (not (join-like-map? m))
+ (boolean
+ (some #(contains? m %)
+ ["source-table" "source-card" "filters" "aggregation"
+ "breakout" "order-by" "fields" "joins" "expressions" "limit"]))))
+
+;;; ============================================================
+;;; Pass 2.95 -- hoist a temporal bucket: move it off an `absolute-datetime` literal and onto the
+;;; ref it is compared to, aligning the literal to the unit it gives up. Sideways, within one
+;;; clause; nothing moves up a level.
+;;;
+;;; ["=", {}, ["field", {}, <col>], ["absolute-datetime", {}, "2025-01-01", "month"]]
+;;; => ["=", {}, ["field", {"temporal-unit": "month"}, <col>], "2025-01-01"]
+;;;
+;;; ["during", {}, ["field", {}, <col>], "2025-06-15", "month"]
+;;; => ["=", {}, ["field", {"temporal-unit": "month"}, <col>], "2025-06-01"]
+;;;
+;;; ["during", {}, ["field", {}, <col>], "2025-01-01T10:30:00", "hour"]
+;;; => ["=", {}, ["field", {"temporal-unit": "hour"}, <col>], "2025-01-01T10:00"]
+;;;
+;;; ["between", {}, ["field", {}, <col>], ["absolute-datetime", {}, "2025-01-01", "day"], ["now", {}]]
+;;; => ["between", {}, ["field", {}, <col>], "2025-01-01", ["now", {}]]
+;;;
+;;; Why: the frontend opens a Metabot question from a `/question#` hash holding *legacy* MBQL
+;;; (`metabase.metabot.agent.links/->legacy-mbql`). JSON turns the unit into a string, and the legacy
+;;; schema cannot decode it back for `absolute-datetime` / `during`, so the clause never survives the
+;;; hop -- the last paragraph below says what that costs, which depends on where the clause sits. A
+;;; `temporal-unit` on a ref decodes fine (`legacy-json-round-trip-temporal-filters-test`).
+;;;
+;;; Only inside a stage's `filters` and `expressions`, at every nesting level of the two -- a
+;;; comparison inside a `case` included -- and in every stage of the query, a join's inner stages
+;;; among them, where `optimize-temporal-filters` reads the unit from either side. Not in
+;;; `aggregation:` (`count-where`, `sum-where`, `share`), `order-by:` or a join's own `conditions:`:
+;;; that middleware walks `:filters` and `:expressions` and nothing else, so there the bucket really
+;;; would truncate the column --
+;;;
+;;; ["count-where", {}, [">", {}, <col>, ["absolute-datetime", {}, "2027-06-15", "month"]]]
+;;; as written: SUM(CASE WHEN created_at > date_trunc(month, ...) ...)
+;;; hoisted: SUM(CASE WHEN date_trunc(month, created_at) > date_trunc(month, ...) ...)
+;;;
+;;; -- a different answer. Also left as written: a unit the literal cannot carry (`hour` on a date,
+;;; `month-of-year`), `year` on an `expression` ref ([[unit-hoistable-onto-ref?]] says why), a ref
+;;; that already has a `temporal-unit`, `between` bounds with different buckets, and an unparseable
+;;; literal (resolve's `:invalid-temporal-literal` check still needs to see it). The `between`
+;;; example drops the wrapper anyway: `day` on a date adds nothing, and the bound ends up as the
+;;; query builder writes it.
+;;;
+;;; `::query` / `::stage` are `:closed false`, so a stray `filters:` on the query root or a join map
+;;; sits in no stage: E7 names the clause, never the stray key, and the model's fix is dropped again.
+;;;
+;;; Both forms then select the same rows -- or compute the same value, inside `expressions` --
+;;; wherever `optimize-temporal-filters` can read the unit off either side: a `field` ref, or an
+;;; `expression` ref that carries a type. Two exceptions.
+;;;
+;;; (1) `!=` / `not-in` on a `:type/Date` column bucketed by `day`. Only the hoisted form reaches
+;;; `date-field-with-day-bucketing?`, so the literal-side form becomes a negated range
+;;; (`d < x OR d >= x+1`, dropping NULL rows) while the hoisted one stays an `!=` and gets the
+;;; `IS NULL` disjunct `sql.qp` adds to every other `!=` (`correct-null-behaviour`). Hoisting is the
+;;; only option here, not just the better one: every `!=` shape that survives the JSON hop compiles
+;;; to `<> ... OR ... IS NULL`, and the one shape that does not is the broken one above. Pinned by
+;;; `optimize-date-not-equals-null-semantics-test`. The divergence is about *which rows*; inside a
+;;; `case` under `expressions` that becomes a difference in a computed value rather than in the
+;;; filter.
+;;;
+;;; (2) An `expression` ref. The middleware optimises neither form, for two unrelated reasons: as
+;;; written, resolve stamps no `base-type` on the ref and `temporal-ref?` reads the type from the
+;;; ref's own options, so the clause is not optimizable at all; hoisted, `wrap-value-literals` copies
+;;; a ref's unit onto the bare literal only for `field` refs (`type-info-from-col` sits behind a
+;;; `clause-of-type? :field` guard), so a `month`-bucketed `expression` ref meets a `day` literal and
+;;; the units disagree.
+;;;
+;;; [">", {}, ["expression", {}, "Ship"], ["absolute-datetime", {}, "2025-06-01", "month"]]
+;;; as written: WHERE <Ship> > date_trunc(month, '2025-06-01') -- 18744 rows
+;;; hoisted: WHERE date_trunc(month, <Ship>) > '2025-06-01' -- 18709 rows
+;;;
+;;; The hoisted count is what both forms give once the ref is typed, so the rewrite makes an untyped
+;;; *temporal* `expression` ref behave like a typed one -- a real change, and the right one. `=` and
+;;; `<=` diverge the same way; `<` and `>=` happen to agree. Both asymmetries are QP gaps
+;;; (`temporal-ref?` should consult `lib/type-of`; `wrap-value-literals` should read a unit off any
+;;; ref); worth their own issue. Pinned by `optimize-untyped-expression-ref-not-optimized-test`.
+;;;
+;;; That is also why the pass aligns the literal itself ([[align-literal-to-unit]]). The middleware
+;;; truncates *both* sides of a comparison, so on a `field` ref a literal sitting in the middle of
+;;; its bucket costs nothing. On a hoisted `expression` ref it fires only when the literal carries a
+;;; time component, which `wrap-value-literals` types for it; a date-only literal is left alone, so
+;;; only the column is truncated and the comparison asks a different question --
+;;;
+;;; ["=", {}, ["expression", {"temporal-unit": "month"}, "Ship"], "2025-06-15"]
+;;; date-only: WHERE date_trunc(month, <Ship>) = '2025-06-15' -- never true
+;;; "…T10:00:00" or "Z": WHERE <Ship> >= '2025-06-01' AND <Ship> < '2025-07-01'
+;;;
+;;; -- and the ordered heads land up to one bucket out. `during` makes that the mainline case, since
+;;; it names the unit *containing* its literal. Aligning reproduces what the middleware would have
+;;; done, and is a no-op on a `field` ref.
+;;;
+;;; The one unit alignment cannot rescue is `year`, which is both a truncation and an extraction
+;;; unit: `lib/type-of` calls a `year`-bucketed untyped ref `:type/Integer`, and the QP's
+;;; `auto-parse-filter-values` then fails on the date literal. [[unit-hoistable-onto-ref?]] declines
+;;; that hoist on an `expression` ref, leaving the clause to Pass 6; a model writing the bucketed ref
+;;; itself still reaches the QP, so the hole belongs with the two gaps above.
+;;;
+;;; Column types are unknown here, so a bucket can land on a text or numeric column;
+;;; [[assert-temporal-buckets-on-temporal-columns!]] rejects that after resolve, on an `expression`
+;;; ref as well as a `field` one. It matters more on the former: the QP drops the bucket silently on
+;;; a `field` ref, but keeps it on an `expression` ref, where it compiles to nonsense on a text
+;;; column and fails preprocessing outright on a numeric one. That is the same exposure `filters`
+;;; already has; walking `expressions` opens a second position onto it.
+;;;
+;;; Runs after Pass 2.9, which creates new `filters`. Idempotent: the output matches neither
+;;; predicate.
+;;;
+;;; None of the survivors above is merely unrewritten -- each is broken. The legacy unit enums have no
+;;; `:decode/normalize`, so after the JSON hop the clause never normalizes back. What that costs
+;;; depends on where it sits. In `filters`, `expressions` and join `conditions:` the dev and CI
+;;; question page 400s, while a production JAR -- where `normalize-or-throw`'s `mu/defn` output check
+;;; is compiled out -- lets `lib/query`'s `clean-stage-schema-errors` delete the clause, and the
+;;; question silently answers something else. In `aggregation:` the legacy schema is permissive
+;;; enough that even dev never throws: the `count-where` above is deleted in both, silently. Pass 6's
+;;; [[unencodable-temporal-clause-error!]] catches every one and raises a retryable `:agent-error?`,
+;;; except the unparseable literal, which resolve names more precisely.
+;;; ============================================================
+
+(def ^:private date-truncation-units
+ "Bucketing units that mean the same thing on a ref as on a date literal."
+ #{"day" "week" "month" "quarter" "year"})
+
+(def ^:private datetime-truncation-units
+ "The date units plus the sub-day ones a datetime literal supports."
+ (into date-truncation-units #{"second" "minute" "hour"}))
+
+(def ^:private bucket-hoistable-heads
+ "Comparison heads whose literal operands may carry a hoistable bucket."
+ #{"=" "!=" "<" "<=" ">" ">=" "in" "not-in"})
+
+(def ^:private date-only-pattern #"\d{4}-\d{2}-\d{2}")
+(def ^:private year-month-pattern #"\d{4}-\d{2}")
+(def ^:private year-pattern #"\d{4}")
+
+(defn- widen-partial-date
+ "`yyyy-MM` -> `yyyy-MM-01`, `yyyy` -> `yyyy-01-01`; anything else unchanged."
+ [s]
+ (cond
+ (re-matches year-month-pattern s) (str s "-01")
+ (re-matches year-pattern s) (str s "-01-01")
+ :else s))
+
+(defn- align-literal-to-unit
+ "Truncate `literal` to the start of the `unit` that contains it.
+
+ [\"2025-06-15\" \"month\"] => \"2025-06-01\"
+ [\"2025-06-15\" \"day\"] => \"2025-06-15\" ; already aligned, returned as written
+
+ The generalisation of [[widen-partial-date]], which already does this for `yyyy-MM` / `yyyy`. On a
+ `field` ref it changes no answer - the QP truncates both sides of such a comparison itself - and on
+ an `expression` ref it is what keeps the hoisted clause meaning what the literal-side one did; the
+ pass header says why the QP cannot do it there.
+
+ `literal` comes back as written unless truncation both moves the instant and leaves an ISO string.
+ `default` names no unit, so it is skipped."
+ [literal unit]
+ (if (= "default" unit)
+ literal
+ (let [parsed (u.date/parse literal)
+ t (u.date/truncate parsed (keyword unit))
+ ;; a `Z`-suffixed literal parses to a `ZonedDateTime`, which renders back as
+ ;; `2025-06-01T00:00Z[UTC]` - that would reach the `/question#` hash and the LLM-facing
+ ;; export verbatim, so a rewrite that is not ISO is dropped rather than emitted
+ aligned (str t)]
+ (if (and (not= parsed t) (iso-date-string? aligned))
+ aligned
+ literal))))
+
+(defn- parseable-temporal-literal? [s]
+ (try
+ (some? (u.date/parse s))
+ (catch Exception _
+ false)))
+
+(defn- truncation-units-for
+ "The bucketing units a widened literal of this shape can carry.
+
+ A date-only literal takes the date units; a datetime literal also takes the sub-day ones."
+ [literal]
+ (if (re-matches date-only-pattern literal)
+ date-truncation-units
+ datetime-truncation-units))
+
+(defn- hoistable-bucket
+ "Return `[literal unit]` for an `absolute-datetime` literal whose bucket the compared ref can take.
+
+ [\"absolute-datetime\" {} \"2025-03\" \"MONTH\"] => [\"2025-03-01\" \"month\"]
+ [\"absolute-datetime\" {} \"2025-06-15\" \"month\"] => [\"2025-06-01\" \"month\"]
+ [\"absolute-datetime\" {} \"2025-01-01\" \"hour\"] => nil ; a date cannot carry `hour`
+
+ Both values come back normalised: trimmed, lower-cased, `yyyy-MM` / `yyyy` widened to the first day
+ they name, and the literal aligned to its unit ([[align-literal-to-unit]]) so the hoisted clause
+ means what the literal-side one did. `nil` for anything that is not such a literal."
+ [v]
+ (when (and (vector? v)
+ (= 4 (count v))
+ (string? (nth v 0))
+ (= "absolute-datetime" (u/lower-case-en (nth v 0)))
+ (map? (nth v 1))
+ ;; a named literal is only ever an expression's top-level clause, never a comparison
+ ;; operand; a type hint is safe to drop
+ (empty? (dissoc (nth v 1) "base-type" "effective-type"))
+ (string? (nth v 2))
+ (string? (nth v 3)))
+ (let [literal (widen-partial-date (str/trim (nth v 2)))
+ unit (u/lower-case-en (nth v 3))
+ units (truncation-units-for literal)]
+ (when (and (iso-date-string? literal)
+ (or (= "default" unit) (contains? units unit))
+ (parseable-temporal-literal? literal))
+ [(align-literal-to-unit literal unit) unit]))))
+
+(defn- during-clause? [v]
+ (and (vector? v)
+ (= 5 (count v))
+ (= "during" (nth v 0))
+ (map? (nth v 1))))
+
+(defn- unbucketed-ref-clause?
+ "A `field` or `expression` ref with no `temporal-unit`. A `nil` unit counts as none: lib drops it."
+ [v]
+ (and (vector? v)
+ (= 3 (count v))
+ (contains? #{"field" "expression"} (nth v 0))
+ (map? (nth v 1))
+ (nil? (get (nth v 1) "temporal-unit"))))
+
+(defn- hoistable-comparison-node?
+ "A comparison clause shaped so that a bucket could move onto its ref operand. Says nothing about
+ whether the literal operands actually carry one."
+ [v]
+ (and (vector? v)
+ (>= (count v) 4)
+ (string? (nth v 0))
+ (map? (nth v 1))
+ (unbucketed-ref-clause? (nth v 2))
+ (or (between-clause? v)
+ (during-clause? v)
+ (contains? bucket-hoistable-heads (nth v 0)))))
+
+(defn- with-temporal-unit [ref-clause unit]
+ (assoc-in ref-clause [1 "temporal-unit"] unit))
+
+(defn- unit-hoistable-onto-ref?
+ "Whether `unit` may move onto `ref-clause`. False only for `year` on an `expression` ref.
+
+ `year` is both a truncation and an extraction unit, so `lib/type-of` answers `:type/Integer` for a
+ `year`-bucketed ref that carries no `effective-type` - which is exactly an `expression` ref, since
+ resolve stamps no type on one - and the QP's `auto-parse-filter-values` then fails outright trying
+ to read the date literal as an integer. A `field` ref carries a type, so the same bucket is fine
+ there. Declining leaves the literal standing for [[unencodable-temporal-clause-error!]] to turn
+ into a retryable error."
+ [ref-clause unit]
+ (not (and (= "expression" (nth ref-clause 0))
+ (= "year" unit))))
+
+(defn- hoist-bucket-in-comparison
+ "When every literal of a comparison carries the same hoistable bucket, put it on the ref.
+
+ [\"in\" {} [\"field\" {} <col>] [\"absolute-datetime\" {} \"2025-01-01\" \"month\"]
+ [\"absolute-datetime\" {} \"2025-03-01\" \"month\"]]
+ => [\"in\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-01-01\" \"2025-03-01\"]"
+ [[head opts ref & literals :as node]]
+ (let [buckets (map hoistable-bucket literals)
+ unit (second (first buckets))]
+ (if (and (every? some? buckets)
+ (apply = (map second buckets))
+ (unit-hoistable-onto-ref? ref unit))
+ (into [head opts (with-temporal-unit ref unit)] (map first buckets))
+ node)))
+
+(defn- redundant-bucket-literal
+ "The bare literal of a bucketed bound whose bucket adds nothing to it: `default`, or `day` on a
+ date-only literal. `nil` otherwise."
+ [v]
+ (when-let [[literal unit] (hoistable-bucket v)]
+ (when (or (= "default" unit)
+ (and (= "day" unit) (re-matches date-only-pattern literal)))
+ literal)))
+
+(defn- hoist-bucket-in-between
+ "Both bounds bucketed alike: hoist the bucket onto the ref. Otherwise drop only a wrapper that adds nothing.
+
+ [\"between\" {} <ref> [\"absolute-datetime\" {} \"2025-01-01\" \"month\"]
+ [\"absolute-datetime\" {} \"2025-06-01\" \"month\"]]
+ => [\"between\" {} <ref bucketed by month> \"2025-01-01\" \"2025-06-01\"]
+
+ [\"between\" {} <ref> [\"absolute-datetime\" {} \"2025-01-01\" \"day\"] [\"now\" {}]]
+ => [\"between\" {} <ref> \"2025-01-01\" [\"now\" {}]]"
+ [[head opts ref lo hi]]
+ (let [[lo' lo-unit] (hoistable-bucket lo)
+ [hi' hi-unit] (hoistable-bucket hi)]
+ (if (and lo' hi' (= lo-unit hi-unit) (unit-hoistable-onto-ref? ref lo-unit))
+ [head opts (with-temporal-unit ref lo-unit) lo' hi']
+ [head opts ref (or (redundant-bucket-literal lo) lo) (or (redundant-bucket-literal hi) hi)])))
+
+(defn- hoist-bucket-in-during
+ "`during` is `=` against the ref bucketed by the same unit; say it that way.
+
+ [\"during\" {} [\"field\" {} <col>] \"2025-01-01\" \"month\"]
+ => [\"=\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-01-01\"]
+
+ [\"during\" {} [\"field\" {} <col>] \"2025-06-15\" \"month\"]
+ => [\"=\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-06-01\"]
+
+ [\"during\" {} [\"field\" {} <col>] \"2025-01-01\" \"hour\"] => unchanged
+
+ The unit set follows the literal's shape, as it does for a comparison ([[hoistable-bucket]]):
+ rewriting `hour` on a date would narrow the filter to one hour. `during` names the unit
+ *containing* the literal, so a literal in the middle of its bucket is its normal input and is
+ [[align-literal-to-unit]]'s mainline case - the second example above.
+ [[unencodable-temporal-clause-error!]] reports what is left standing."
+ [[_ opts ref literal unit :as node]]
+ (let [literal (when (string? literal) (widen-partial-date (str/trim literal)))
+ unit (when (string? unit) (u/lower-case-en unit))]
+ (if (and literal
+ (iso-date-string? literal)
+ (contains? (truncation-units-for literal) unit)
+ (unit-hoistable-onto-ref? ref unit)
+ (parseable-temporal-literal? literal))
+ ["=" opts (with-temporal-unit ref unit) (align-literal-to-unit literal unit)]
+ node)))
+
+(defn- hoist-bucket-in-clause [node]
+ (cond
+ (during-clause? node) (hoist-bucket-in-during node)
+ (between-clause? node) (hoist-bucket-in-between node)
+ :else (hoist-bucket-in-comparison node)))
+
+(def ^:private bucket-hoist-stage-keys
+ "The stage keys the QP's `optimize-temporal-filters` rewrites, and so the only ones a bucket may
+ move inside."
+ ["filters" "expressions"])
+
+(defn- hoist-buckets-in-clauses
+ "Rewrite every hoistable comparison at any depth of one stage key's clause vector."
+ [clauses]
+ (walk/postwalk (fn [n] (if (hoistable-comparison-node? n) (hoist-bucket-in-clause n) n)) clauses))
+
+(defn- hoist-temporal-buckets*
+ "Hoist each `absolute-datetime` bucket in a stage's `filters` / `expressions` onto the compared ref.
+
+ {\"filters\" [[\"=\" {} [\"field\" {} <col>] [\"absolute-datetime\" {} \"2025-01-01\" \"month\"]]]}
+ => {\"filters\" [[\"=\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-01-01\"]]}
+
+ Only those two keys, and only on a map [[stage-like-map?]] recognises -- the same predicate Pass 2
+ uses to decide what a stage is, so the hoist's notion of a stage is the one that stamps the marker.
+ The pass header says why the scope is those two keys."
+ [form]
+ (walk/postwalk
+ (fn [node]
+ (if (stage-like-map? node)
+ (reduce (fn [stage k]
+ (cond-> stage
+ ;; belt and braces: Pass 1.5 `normalize-expressions-shape*` has already turned a
+ ;; map-form `expressions:` into the sequential form, and a stage carrying neither
+ ;; key falls through untouched
+ (vector? (get stage k)) (update k hoist-buckets-in-clauses)))
+ node
+ bucket-hoist-stage-keys)
+ node))
+ form))
+
+;;; ============================================================
+;;; Pass 1.867 -- rewrite `temporal-extract` to the `get-*` clause that means the same thing.
+;;;
+;;; ["temporal-extract", {}, <col>, "month-of-year"] => ["get-month", {}, <col>]
+;;; ["temporal-extract", {}, <col>, "week-of-year-iso"] => ["get-week", {}, <col>, "iso"]
+;;; ["temporal-extract", {}, <col>, "day-of-week", "us"] => ["get-day-of-week", {}, <col>]
+;;;
+;;; Same hazard as Pass 2.95: the unit lands in the legacy `::TemporalExtractUnit` enum, which has
+;;; no decoder, so the question page 400s. A `get-*` clause carries no unit. The week-mode units keep
+;;; their meaning through the trailing argument `get-week` / `get-day-of-week` already take. The
+;;; clause's own optional mode slot (third example) is dropped: SQL compilation reads only the unit
+;;; (`:temporal-extract` in `metabase.driver.sql.query-processor`), so it never mattered.
+;;;
+;;; An exact synonym wherever the clause appears, so not scoped to `filters`. Idempotent: the new
+;;; head is not `temporal-extract`.
+;;; ============================================================
+
+(def ^:private temporal-extract-unit->getter
+ "Extraction unit -> the `get-*` head meaning exactly the same thing, followed by any trailing
+ week-mode argument that head needs to preserve the unit's meaning."
+ {"year-of-era" ["get-year"]
+ "quarter-of-year" ["get-quarter"]
+ "month-of-year" ["get-month"]
+ "day-of-month" ["get-day"]
+ "day-of-week" ["get-day-of-week"]
+ "day-of-week-iso" ["get-day-of-week" "iso"]
+ "hour-of-day" ["get-hour"]
+ "minute-of-hour" ["get-minute"]
+ "second-of-minute" ["get-second"]
+ "week-of-year-iso" ["get-week" "iso"]
+ "week-of-year-us" ["get-week" "us"]
+ "week-of-year-instance" ["get-week" "instance"]})
+
+(defn- rewrite-temporal-extract*
+ "Rewrite each `temporal-extract` clause in `form` to the `get-*` clause that means the same thing.
+
+ [\"temporal-extract\" {} <col> \"week-of-year-iso\"] => [\"get-week\" {} <col> \"iso\"]
+ [\"temporal-extract\" {} <col> \"day-of-week\" \"us\"] => [\"get-day-of-week\" {} <col>]
+
+ A unit with no `get-*` equivalent is left as written for validation to report."
+ [form]
+ (walk/postwalk
+ (fn [node]
+ (if-let [[getter & mode] (and (vector? node)
+ (<= 4 (count node) 5)
+ (string? (nth node 0))
+ (= "temporal-extract" (u/lower-case-en (nth node 0)))
+ (map? (nth node 1))
+ (string? (nth node 3))
+ (temporal-extract-unit->getter (u/lower-case-en (nth node 3))))]
+ (into [getter (nth node 1) (nth node 2)] mode)
+ node))
+ form))
+
;;; ============================================================
;;; Pass 1.55 -- normalise `fields:` on a stage / join when the LLM wrote a single clause
;;; instead of a sequential of clauses.
@@ -1091,37 +1506,11 @@
;;; ============================================================
;;; Pass 2 -- fill in missing `lib/type` markers
+;;;
+;;; The structural predicates `needs-lib-type-marker?` keys on live in the shared section above Pass
+;;; 2.95, which gates on the same notion of a stage.
;;; ============================================================
-(defn- top-level-query-map?
- [m]
- (match/matches? m {"database" _, "stages" _}))
-
-(defn- join-like-map?
- "A map that looks like an explicit join: it carries join-only keys (`conditions`, or
- `alias`+`stages`) that never appear on a stage or top-level query. We key on these rather
- than on `\"fields\"` (which a join shares with a stage) so a join is not mistaken for a
- stage by [[stage-like-map?]]."
- [m]
- (and (map? m)
- (or (contains? m "conditions")
- (and (contains? m "alias")
- (contains? m "stages")))))
-
-(defn- stage-like-map?
- "A map that looks like an MBQL stage: has `\"source-table\"`, `\"source-card\"`, or any of the
- stage-body keys (`filters`, `aggregation`, `breakout`, `order-by`, `fields`, `joins`,
- `expressions`, `limit`). Not a top-level query and not an explicit join (a join can carry
- `\"fields\"`, so we exclude it explicitly)."
- [m]
- (and (map? m)
- (not (top-level-query-map? m))
- (not (join-like-map? m))
- (boolean
- (some #(contains? m %)
- ["source-table" "source-card" "filters" "aggregation"
- "breakout" "order-by" "fields" "joins" "expressions" "limit"]))))
-
(defn- needs-lib-type-marker? [m]
(and (map? m)
(not (contains? m "lib/type"))
@@ -2426,6 +2815,82 @@
:error :numeric-field-id
:clause &match})))))
+;;; ----- E7: a temporal clause the `/question#` hash cannot carry ---------------------
+
+(defn- deferred-to-invalid-temporal-literal?
+ "True for an `absolute-datetime` whose literal cannot parse.
+
+ Resolve's own check names the literal (`:invalid-temporal-literal`,
+ [[metabase.agent-lib.representations.resolve/validate-temporal-literals]]), which is the more
+ useful complaint, so E7 stays quiet and lets that one fire. It does not inspect `during` or
+ `value`, which is why only this head gets the carve-out.
+
+ The carve-out is per *clause*, not per query: a query holding an unparseable literal alongside any
+ other offender still reports that other offender, because E7 throws on the first match anywhere.
+ Deferring the whole query instead would be a worse trade -- `validate-temporal-literals` does not
+ inspect every position, since a bare literal on a comparison head is checked only when an operand
+ is *known* temporal, so an unparseable literal in a position it does not inspect would silence E7
+ with nothing to replace it, reopening the silent-deletion hole
+ Pass 2.95 and this detector exist to close. The model fixes what E7 named and sees the literal
+ complaint on the next turn."
+ [node]
+ (let [literal (nth node 2)]
+ (and (string? literal)
+ (not (parseable-temporal-literal? literal)))))
+
+(defn- unencodable-temporal-clause-error!
+ "Detect an `absolute-datetime`, `during` or unit-carrying `value` clause that Pass 2.95 could not
+ rewrite.
+
+ [\"count-where\", {}, [\">\", {}, <col>, [\"absolute-datetime\", {}, \"2025-01-01\", \"month\"]]]
+
+ All three hold a temporal unit in a slot the legacy schema types with `::DateUnit` /
+ `::DateTimeUnit` (`metabase.legacy-mbql.schema`, via `::ValueTypeInfo` for `value`), and neither
+ enum carries a `:decode/normalize`. Once the `/question#` hash has JSON-encoded the query the unit
+ is a string and the clause never normalizes back. In `filters`, `expressions` and join
+ `conditions:` that is a 400 on the dev and CI question page, and in a production JAR -- where
+ `normalize-or-throw`'s `mu/defn` output check is compiled out -- `lib/query`'s
+ `clean-stage-schema-errors` deletes the clause outright and the question silently answers something
+ else. In `aggregation:` the legacy schema is permissive enough that nothing throws even in dev: the
+ clause is deleted silently in both. Pass 2.95 rewrites every shape it can rewrite without changing
+ the answer, so anything still standing here is broken either way.
+
+ `relative-datetime` and `datetime-diff` units do carry `:decode/normalize`, so they survive the hop
+ and are not listed. `temporal-extract`'s `::TemporalExtractUnit` has no decoder either, but Pass
+ 1.867 has already rewritten every one to a `get-*` clause by the time this runs -- narrow that pass
+ and this detector needs the head. `time` is excluded on purpose: it 400s in dev, but `->mbql5`
+ salvages the raw clause in prod (unit correctly keywordized, filter intact), so flagging it would
+ reject a query that works for customers. This detector exists only because `::DateUnit`,
+ `::TimeUnit`, `::DateTimeUnit` and `::TemporalExtractUnit` lack decoders -- if they ever gain them,
+ delete it rather than inherit a false-positive machine.
+
+ Throws `:agent-error?` ex-info on the first offender."
+ [form]
+ (walk/postwalk
+ (fn [node]
+ ;; `map-entry?`: a `{\"value\" …}` entry postwalks as a 2-element vector. The arity checks below
+ ;; already exclude it; the guard matches E2 / E6 and survives a head being added at that arity.
+ (when (and (vector? node)
+ (not (map-entry? node))
+ (string? (nth node 0 nil))
+ (map? (nth node 1 nil))
+ (case (u/lower-case-en (nth node 0))
+ "absolute-datetime" (and (= 4 (count node))
+ (not (deferred-to-invalid-temporal-literal? node)))
+ "during" (= 5 (count node))
+ ;; a `nil` unit counts as none, as it does on a ref -- that shape round-trips fine
+ "value" (and (= 3 (count node))
+ (some? (get (nth node 1) "unit")))
+ false))
+ (throw (ex-info
+ (tru "`{0}` does not survive the round trip through the question link the frontend opens: the question either fails to load, or silently loses the clause and answers something else. Put the unit on the column instead - a `temporal-unit` option on the `field` / `expression` reference, compared to the first day of the period as a plain date string like \"2025-01-01\" - or write the range out as `>=` its first day and `<` the first day after it, both plain date strings (not `between`, which drops everything after midnight on its last day). Use the range for `year` on an `expression` reference, which the query engine reads as a number, and inside `aggregation:` (`count-where`, `sum-where`, `share`) and join `conditions:`, where moving the unit onto the column would change the answer."
+ (pr-str node))
+ {:agent-error? true
+ :error :unencodable-temporal-clause
+ :clause node})))
+ node)
+ form))
+
;;; ----- friendly-errors pipeline driver -----------------------------------------------
(defn- friendly-errors*
@@ -2439,7 +2904,8 @@
(case-default-in-opts-error! query)
(sexp-legacy-op-as-clause-error! query)
(blank-expression-ref-error! query)
- (numeric-field-id-error! query))
+ (numeric-field-id-error! query)
+ (unencodable-temporal-clause-error! query))
query)
;;; ============================================================
@@ -2464,8 +2930,8 @@
rewrite-lib-type-aliases*
merge-trailing-options*
merge-string-filter-trailing-options*
- wrap-iso-date-bounds*
wrap-now-literals*
+ rewrite-temporal-extract*
swap-between-bounds*
normalise-case-clauses*
normalise-fields-shape*
@@ -2489,11 +2955,16 @@
1.75. strip stray surrounding double-quotes from the string segments of `field` clauses'
portable-FK vector targets, e.g. `\"col\"` → `col` (cross-stage string targets are left
to the resolution-aware cross-stage matching in pass 5);
+ 1.867. rewrite `temporal-extract` to the equivalent `get-*` clause, so its unit no longer sits
+ where legacy normalization cannot decode it (examples in the pass header);
1.87. rewrite a known-misspelled `\"lib/type\"` marker to its canonical value (e.g. the
join slip `\"mbql.join/join\"` → `\"mbql/join\"`);
1.88. merge a trailing extra options-map back into position-1 options on fixed-arity
tuple clauses (e.g. `[\"time-interval\" {} <expr> -1 \"month\" {\"include-current\" true}]`);
2. fill in missing `\"lib/type\"` markers on the query, joins, and stages;
+ 2.95. inside a stage's `filters` and `expressions` only, hoist a temporal bucket off an
+ `absolute-datetime` literal and onto the ref it is compared to, for the same reason
+ (examples in the pass header; runs after Pass 2.9, which creates new `filters`);
3. rewrite inline aggregation expressions in `order-by` to aggregation references when
they match an aggregation in the same stage's `aggregation:` list (synthesising the
referenced aggregation's `lib/uuid` if needed);
@@ -2560,6 +3031,7 @@
rewrite-order-by-inline-aggs*
resolve-aggregation-ref-indexes*
split-post-agg-filters*
+ hoist-temporal-buckets*
(resolve-source-field-join-alias* mp content-store)
(resolve-implicit-joins* mp content-store)
(infer-source-card-field-types* mp content-store)
@@ -2607,3 +3079,109 @@
:mode mode
:clause clause})))))
pmbql-query)
+
+(defn- unbucketed-ref
+ "`clause` with its `temporal-unit`, and the type the bucket stamped on it, taken off.
+
+ A *coercion's* `:effective-type` is the column's own type rather than a bucket artifact, and stays.
+
+ [:expression {:temporal-unit :day-of-week, :effective-type :type/Integer,
+ :base-type :type/DateTime} \"Ship\"]
+ => [:expression {:base-type :type/DateTime} \"Ship\"]
+
+ ;; field 105 is ISO-8601 text an admin coerced to a timestamp
+ [:field {:temporal-unit :year, :effective-type :type/DateTime,
+ :base-type :type/Text} 105]
+ => [:field {:effective-type :type/DateTime, :base-type :type/Text} 105]
+
+ `with-temporal-bucket` alone restores `:effective-type` from `:lib/original-effective-type`, which
+ is only there on a ref lib or [[metabase.agent-lib.representations.resolve/export-query]] produced.
+ On a ref resolve typed, an *extraction* unit leaves `:effective-type` describing the bucket rather
+ than the column - `:type/Integer` for `day-of-week` on a datetime column - and
+ [[metabase.lib.core/type-of]] reads that key first, so it has to go too. What is left resolves to
+ `:base-type`, or to the expression's own type when the ref carries none."
+ [clause]
+ (let [{:keys [effective-type temporal-unit] :lib/keys [original-effective-type]} (nth clause 1)]
+ (cond-> (lib/with-temporal-bucket clause nil)
+ (and (nil? original-effective-type)
+ (contains? lib.schema.temporal-bucketing/datetime-extraction-units temporal-unit)
+ ;; A bucket's own type is never temporal, so a temporal stamp has to be the column's:
+ ;; resolve copies `:effective-type` straight off a coerced column, where it is the
+ ;; post-coercion type and `:base-type` is the storage type. Dropping it there would leave
+ ;; `:type/Text` on an ISO-8601 column and make the gate reject a bucket lib itself offers.
+ (not (isa? effective-type :type/Temporal)))
+ (update 1 dissoc :effective-type))))
+
+(defn- assert-bucketed-ref-is-temporal!
+ "Throw when the bucketed ref `clause` names a column that is not temporal.
+
+ Call only on a `field` / `expression` ref that carries a `temporal-unit`: the walk in
+ [[assert-temporal-buckets-on-temporal-columns!]] tests that itself, before resolving a stage."
+ [query stage-number clause]
+ (let [temporal-unit (:temporal-unit (nth clause 1))
+ bare-ref (unbucketed-ref clause)
+ column-type (lib/type-of query stage-number bare-ref)]
+ (when (and column-type
+ (not= column-type :type/*)
+ (not (isa? column-type :type/Temporal)))
+ (let [column-name (lib/display-name query stage-number bare-ref)
+ type-name (name column-type)
+ unit-name (name temporal-unit)]
+ (throw (ex-info (if (= :expression (nth clause 0))
+ (tru "`{0}` is a {1} custom column, so it cannot be bucketed by {2}. Compare it to a plain value, or bucket a date / datetime column instead."
+ column-name type-name unit-name)
+ (tru "`{0}` is a {1} column, so it cannot be bucketed by {2}. Compare it to a plain value, or bucket a date / datetime column instead."
+ column-name type-name unit-name))
+ {:agent-error? true
+ :error :temporal-unit-on-non-temporal-column
+ :status-code 400
+ :column-type column-type
+ :temporal-unit temporal-unit
+ :clause clause}))))))
+
+(defn assert-temporal-buckets-on-temporal-columns!
+ "Throw a retryable `:agent-error?` when a resolved `field` or `expression` ref carries a
+ `temporal-unit` but its column is not temporal. Returns `pmbql-query`.
+
+ lib accepts that shape - its unit check only knows Date / Time / DateTime columns. The QP then
+ drops the bucket silently on a `field` ref; on an `expression` ref it keeps it and applies it to a
+ non-temporal value, which compiles to nonsense on a text column and fails preprocessing outright
+ on a numeric one. Pass 2.95 can produce either, in `filters` and in `expressions`, since it moves a
+ bucket onto a ref without knowing the column's type; a model writing the bucket itself lands here
+ too, and Pass 6's [[unencodable-temporal-clause-error!]] sees neither, because the hoist has
+ already consumed the literal.
+
+ Walks with [[metabase.lib.walk/walk-clauses]] rather than `clojure.walk`: an `expression` ref
+ carries no type in its options, so the column type has to come from [[metabase.lib.core/type-of]],
+ which needs the ref's stage. A join condition's refs do not all live in the same stage - one
+ carrying `:join-alias` belongs to the join, one without it to the parent stage - so the path is
+ chosen per ref. A ref whose type cannot be resolved is skipped rather than reported: this gate must
+ never reject a query it merely failed to understand.
+
+ Must run after the `::lib.schema/query` gate ([[metabase.metabot.tools.construct]]'s `_runnable`):
+ a dangling expression ref makes `type-of` throw rather than resolve, and while `walk-clauses`
+ rejects the query first under `mu/defn` instrumentation, that is compiled out of a production JAR."
+ [pmbql-query]
+ (lib.walk/walk-clauses
+ pmbql-query
+ (fn [query path-type path clause]
+ ;; `walk-clauses` calls this on non-clause arguments too - the bare `2` in `[:= {} <ref> 2]`.
+ ;; Everything a bucketed ref is not falls out here, before a stage is resolved for it: that is
+ ;; the expensive half, and all but a handful of nodes skip it.
+ (let [opts (when (vector? clause) (nth clause 1 nil))]
+ (when (and (map? opts)
+ (:temporal-unit opts)
+ (contains? #{:field :expression} (nth clause 0 nil)))
+ (let [stage-path (if (and (= path-type :lib.walk/join) (not (:join-alias opts)))
+ (lib.walk/join-parent-stage-path path)
+ path)]
+ (try
+ (lib.walk/apply-f-for-stage-at-path assert-bucketed-ref-is-temporal! query stage-path clause)
+ ;; `Throwable`, not `ExceptionInfo`: the contract is that an unresolvable ref is skipped,
+ ;; and anything narrower leaks past `execute-representations-query`'s relay as a 500
+ (catch Throwable e
+ ;; re-throw our own complaint; swallow lib's failure to resolve the ref
+ (when (:agent-error? (ex-data e))
+ (throw e)))))))
+ nil))
+ pmbql-query)
diff --git a/src/metabase/agent_lib/representations/resolve.clj b/src/metabase/agent_lib/representations/resolve.clj
index 096d1e37c01..7db8a37f60e 100644
--- a/src/metabase/agent_lib/representations/resolve.clj
+++ b/src/metabase/agent_lib/representations/resolve.clj
@@ -140,7 +140,7 @@
;; `s` is untrusted agent input — bound it before it lands in an exception message or ex-data so a
;; pathological literal can't flood logs. A real temporal literal is well under this length.
(let [s' (u/truncate s 64)]
- (throw (ex-info (tru "Invalid temporal literal {0} in :absolute-datetime — use an ISO-8601 date, datetime, year, or year-month."
+ (throw (ex-info (tru "Invalid temporal literal {0} — use an ISO-8601 date, datetime, year, or year-month."
(pr-str s'))
{:agent-error? true
:status-code 400
@@ -148,24 +148,113 @@
:literal s'}
e))))))
-(defn- validate-absolute-datetime-literals
- "Validate (without coercing) the string literals in `:absolute-datetime` clauses, so malformed
- temporal input fails fast here — where the agent sees it and can correct — instead of surviving to
- query execution.
+(def ^:private iso-date-shaped-pattern
+ "A string that looks like an ISO-8601 date, with or without a time portion."
+ #"\d{4}-\d{2}-\d{2}.*")
+
+(defn- operand-types
+ "The types describing MBQL clause `x`, as a set — empty or `nil` when nothing here describes it.
+
+ [:field {:base-type :type/Text} 43] => #{:type/DateTime} (43 is a datetime)
+ [:expression {:base-type :type/DateTime,
+ :effective-type :type/Integer} \"Ship\"] => #{:type/DateTime :type/Integer}
+
+ For a resolved `[:field opts <id>]` the metadata provider is authoritative and the answer is one
+ type. A type in `opts` is not evidence about the column: [[annotate-field-types]] stamps only a ref
+ that has no `:base-type`, so a wrong model-authored stamp survives while the column stays whatever
+ the provider says it is. Excusing a literal on that stamp buys one of two bad endings, both
+ measured: a non-retryable `:type :qp` from `wrap-value-literals` wherever something upstream buckets
+ the ref off the column (every `between`; `=` on a clean `yyyy-MM-dd`), and otherwise a filter that
+ compiles, runs, and matches nothing for ever — `= <a timestamp column> \"2024-01-01 batch A\"`
+ returns zero rows.
+
+ Every other clause has only its own options, where `:base-type` and `:effective-type` can disagree:
+ a bucket stamps an extraction type over a temporal column, a coercion stamps a temporal type over a
+ text one. Returning both leaves that choice to the callers, which read the set with opposite
+ quantifiers."
+ [metadata-provider x]
+ (when (and (vector? x) (map? (nth x 1 nil)))
+ (let [id (nth x 2 nil)]
+ (if (and (= :field (nth x 0)) (pos-int? id))
+ ;; `pos-int?` is load-bearing, not decoration: `field` is a `mu/defn` whose input check throws
+ ;; on `0` and on a negative id rather than answering `nil` the way an unknown id does.
+ (when-let [field (lib.metadata.protocols/field metadata-provider id)]
+ (when-let [t (or (:effective-type field) (:base-type field))]
+ #{t}))
+ (into #{} (keep (nth x 1)) [:effective-type :base-type])))))
+
+(defn- operand-temporality
+ "Whether MBQL clause `x` is `:temporal`, `:non-temporal`, or `:unknown` when nothing here types it.
+
+ Three-valued on purpose: the two comparison walks below start from opposite defaults, so \"nothing
+ types it\" has to stay distinguishable from \"typed, and not temporal\"."
+ [metadata-provider x]
+ (let [ts (operand-types metadata-provider x)]
+ (cond
+ ;; `some`, not `every?`: a coerced column is described by *both* its storage type and its
+ ;; coercion result — `#{:type/Text :type/DateTime}` for an ISO-8601 string column — and the
+ ;; temporal one is the type the QP compares a literal against.
+ (some #(isa? % :type/Temporal) ts) :temporal
+ (seq ts) :non-temporal
+ :else :unknown)))
+
+(def ^:private temporal-comparison-heads
+ "Comparison heads whose operands can be a temporal ref next to bare temporal string literals.
+ `:between` is absent: it has its own walk, with the opposite default. `during` is absent because it
+ never arrives — Pass 2.95 rewrites a filter-position `during` into the `=`-on-a-bucketed-ref form
+ this set does cover, and Pass 6's
+ [[metabase.agent-lib.representations.repair/unencodable-temporal-clause-error!]] rejects the rest."
+ #{:= :!= :< :<= :> :>= :in :not-in})
+
+(defn- validate-temporal-literals
+ "Validate (without coercing) the temporal string literals the model wrote, so malformed input fails
+ fast here — where the agent sees it and can correct — instead of surviving to query execution: the
+ literal of every `:absolute-datetime` clause, any bare `between` bound that looks like an ISO date,
+ and any such string compared against a temporal operand by one of [[temporal-comparison-heads]] —
+ the shape Pass 2.95's bucket hoist makes canonical and Pass 6's message tells the model to write.
+
+ The two bare-literal walks start from opposite defaults, which is the point. A `between` bound is
+ checked unless the compared column is *known* non-temporal, because it is checked today and a
+ column we cannot type is not a reason to stop: `[\"between\" {} <STATUS> \"2024-01-01 batch A\"
+ \"2024-06-01 batch Z\"]` is a legal text range and must pass, but a bound next to an untyped custom
+ column must still be checked. The other heads are checked only when some operand is *known*
+ temporal and none is known non-temporal, because they are not checked today and
+ `[\"=\" {} <STATUS> \"2024-01-01 batch A\"]` is a perfectly good text comparison.
The actual string → `java.time` coercion is intentionally NOT done here: it happens later in the
QP's `wrap-value-literals`, where the comparison field's type and the report timezone are
available. This pass only rejects literals that can't possibly parse. Returns the query unchanged."
- [pmbql-query]
+ [pmbql-query metadata-provider]
;; `match/match-many`, not `lib.walk/walk-clauses`: the latter is a `mu/defn` that validates its
;; whole-query argument against `::lib.schema/query`, so under test instrumentation it would throw a
;; raw "Invalid input" on an otherwise-malformed query (e.g. an `:offset` in `:expressions`) here,
;; pre-empting the friendlier not-runnable gate downstream. We only need to inspect literals, and
;; matching a bare `[:absolute-datetime _ s _]` vector suffices — the same shape check the sibling
- ;; `annotate-field-types` pass uses.
+ ;; `annotate-field-types` pass uses. Separate walks rather than one with several patterns:
+ ;; `match-many` does not descend into a form it matched, so a wrapped literal inside a comparison
+ ;; has to be reached by a walk that did not match the comparison.
(match/match-many pmbql-query
[:absolute-datetime _ (s :guard string?) _]
(assert-parseable-temporal-literal! s))
+ (match/match-many pmbql-query
+ [:between _ ref lo hi]
+ (when-not (= :non-temporal (operand-temporality metadata-provider ref))
+ (doseq [s [lo hi]
+ :when (and (string? s) (re-matches iso-date-shaped-pattern s))]
+ (assert-parseable-temporal-literal! s))))
+ ;; The operand test lives in the *pattern*, not in the body: a head-only pattern matches an inert
+ ;; comparison too, and `match-many` would then stop and never see a checkable comparison nested
+ ;; inside it. A failing guard leaves the form unmatched, so the walk descends. `match-many` rejects
+ ;; a lambda guard, and this one must close over `metadata-provider`, so it is bound to a name first.
+ (let [checkable-operands? (fn [args]
+ (let [ts (map #(operand-temporality metadata-provider %) args)]
+ (and (some #{:temporal} ts)
+ (not-any? #{:non-temporal} ts))))]
+ (match/match-many pmbql-query
+ [(_ :guard temporal-comparison-heads) _ & (args :guard checkable-operands?)]
+ (doseq [s args
+ :when (and (string? s) (re-matches iso-date-shaped-pattern s))]
+ (assert-parseable-temporal-literal! s))))
pmbql-query)
(defn resolve-query
@@ -188,7 +277,7 @@
(-> (lib.normalize/normalize ::lib.schema/query with-mp)
(annotate-field-types metadata-provider)
annotate-metric-and-measure-ref-types
- validate-absolute-datetime-literals))))
+ (validate-temporal-literals metadata-provider)))))
;;; ============================================================
;;; Export final MBQL 5 back to portable representations
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index 2c80b90cf49..388e76eaea5 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -683,10 +683,13 @@
6. Resolve portable FKs to numeric IDs and normalize through `lib.schema/query` against the
metadata-provider.
6.5. Backstop-gate the resolved query: mirror the FE `canRun` schema validation
- ([[query-not-runnable-explanation]]), then run the FE expression editor's own
+ ([[query-not-runnable-explanation]]), run the FE expression editor's own
diagnostics over every custom column / aggregation / filter
- ([[repr.repair/assert-editor-accepts-expressions!]]). Either failure is a retryable
- `:agent-error?` - success on a query the editor rejects is BOT-1442.
+ ([[repr.repair/assert-editor-accepts-expressions!]]), then reject a temporal bucket
+ sitting on a column that is not temporal
+ ([[repr.repair/assert-temporal-buckets-on-temporal-columns!]]). Any of the three
+ failing is a retryable `:agent-error?` - success on a query the editor rejects is
+ BOT-1442.
7. Export that final numeric MBQL 5 back to the portable form for the LLM-facing
`:query-json` / `query-content` output.
@@ -746,6 +749,7 @@
;; after the `_runnable` gate: `diagnose-expression` itself validates its query
;; argument against the same schema.
_editor-ok (repr.repair/assert-editor-accepts-expressions! pmbql-query)
+ _buckets-ok (repr.repair/assert-temporal-buckets-on-temporal-columns! pmbql-query)
exported-repr (repr.resolve/export-query mp pmbql-query permission-aware-content-store)
_validated' (repr/validate-query exported-repr)
query-id (u/generate-nano-id)]
diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.clj
index 738098cb4bd..ffab922fe0e 100644
--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj
+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj
@@ -233,13 +233,15 @@
(def ^:private raw-value? (complement lib/clause?))
-;; Some queries carry temporal literals in their portable / wire form — e.g. Metabot's representations
-;; repair wraps `between` bounds as `[:absolute-datetime {} "2024-01-01" :day]`, and on CLJS a string
-;; is the only representation. The comparison arms below parse temporal strings that arrive as *raw*
-;; values, but would otherwise skip ones already wrapped in `:absolute-datetime`, leaving a bare string
-;; that later middleware (e.g. `optimize-temporal-filters`) chokes on. This predicate lets those arms
-;; detect such clauses; the unit travels with the string so a `:year`/`:month` literal keeps its bucket
-;; instead of collapsing to the field's default when re-parsed.
+;; Some queries carry temporal literals in their portable / wire form — e.g. a Metabot-built query
+;; whose repair layer leaves a bucketed literal as written wherever moving the bucket onto the column
+;; would change the predicate, such as `[:absolute-datetime {} "2025-01-01" :month]` inside a
+;; `count-where`; and on CLJS a string is the only representation. The comparison arms below parse
+;; temporal strings that arrive as *raw* values, but would otherwise skip ones already wrapped in
+;; `:absolute-datetime`, leaving a bare string that later middleware (e.g. `optimize-temporal-filters`)
+;; chokes on. This predicate lets those arms detect such clauses; the unit travels with the string so
+;; a `:year`/`:month` literal keeps its bucket instead of collapsing to the field's default when
+;; re-parsed.
(defn- string-valued-absolute-datetime
"If `x` is an `:absolute-datetime` clause whose literal is still an (unparsed) string, return a
`[string unit]` pair of that inner string and the clause's temporal unit; otherwise `nil`."
diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.clj
index 5cdf571b5d2..eec516be3f8 100644
--- a/test/metabase/agent_lib/representations/repair_test.clj
+++ b/test/metabase/agent_lib/representations/repair_test.clj
@@ -7,7 +7,8 @@
[clojure.test.check.properties :as prop]
[metabase.agent-lib.representations :as repr]
[metabase.agent-lib.representations.repair :as repair]
- [metabase.lib.test-util :as lib.tu]))
+ [metabase.lib.test-util :as lib.tu]
+ [metabase.util.date-2 :as u.date]))
(set! *warn-on-reflection* true)
@@ -22,6 +23,15 @@
(lib.tu/mock-metadata-provider
{:database {:id 1 :name "Sample"}}))
+(def ^:private mp-temporal
+ "ORDERS with a datetime column, for the hoist test whose prefix stage has to resolve."
+ (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "Sample"}
+ :tables [{:id 10 :name "ORDERS" :schema "PUBLIC" :db-id 1}]
+ :fields [{:id 100 :name "ID" :table-id 10 :base-type :type/Integer}
+ {:id 102 :name "USER_ID" :table-id 10 :base-type :type/Integer}
+ {:id 103 :name "CREATED_AT" :table-id 10 :base-type :type/DateTime}]}))
+
(def ^:private mp-fks
"3-table MP: ORDERS → PRODUCTS, ORDERS → USERS."
(lib.tu/mock-metadata-provider
@@ -662,78 +672,501 @@
(is (= ["not" {} inner] (repair/repair trivial-mp input))))))
;;; ============================================================
-;;; Pass 1.86 - wrap bare ISO-date strings as absolute-datetime in between
+;;; Bare ISO-date `between` bounds
;;; ============================================================
-(deftest ^:parallel wrap-iso-date-bounds-both-bare-test
- (testing "both bounds are bare yyyy-mm-dd strings: both get wrapped"
+(deftest ^:parallel iso-date-bounds-left-bare-test
+ (testing "both bounds bare yyyy-mm-dd: left exactly as the model wrote them"
(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
input ["between" {} field "2024-01-01" "2024-12-31"]]
- (is (= ["between" {}
- field
- ["absolute-datetime" {} "2024-01-01" "day"]
- ["absolute-datetime" {} "2024-12-31" "day"]]
- (repair/repair trivial-mp input))))))
+ (is (= input (repair/repair trivial-mp input))))))
-(deftest ^:parallel wrap-iso-date-bounds-mixed-test
- (testing "one bound already absolute-datetime, the other a bare string: bare side
- gets wrapped"
- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
- wrapped ["absolute-datetime" {} "2024-01-01" "day"]
- input ["between" {} field wrapped "2024-12-31"]]
- (is (= ["between" {}
- field
- wrapped
- ["absolute-datetime" {} "2024-12-31" "day"]]
- (repair/repair trivial-mp input)))))
- (testing "one bound a relative-datetime, the other a bare ISO string: bare side gets
- wrapped (presence of temporal sibling triggers the rule)"
+(deftest ^:parallel iso-datetime-bounds-keep-their-time-test
+ (testing "bounds carrying a time portion keep it"
+ (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ input ["between" {} field "2024-06-15T09:00:00" "2024-06-15T18:00:00"]]
+ (is (= input (repair/repair trivial-mp input))))))
+
+(deftest ^:parallel iso-date-bound-beside-temporal-clause-test
+ (testing "a temporal sibling does not pull a bare bound into a wrapper"
(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
rel ["relative-datetime" {} -7 "day"]
input ["between" {} field rel "2024-12-31"]]
- (is (= ["between" {} field rel
- ["absolute-datetime" {} "2024-12-31" "day"]]
- (repair/repair trivial-mp input))))))
+ (is (= input (repair/repair trivial-mp input))))))
-(deftest ^:parallel wrap-iso-date-bounds-numeric-untouched-test
- (testing "both bounds numeric: not wrapped (between also works for numbers)"
- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "TOTAL"]]
- input ["between" {} field 10 100]]
+(deftest ^:parallel non-temporal-between-bounds-untouched-test
+ (testing "numeric and non-ISO string bounds are untouched"
+ (doseq [input [["between" {} ["field" {} ["Sample" "PUBLIC" "ORDERS" "TOTAL"]] 10 100]
+ ["between" {} ["field" {} ["Sample" "PUBLIC" "X"]] "hello" "world"]]]
(is (= input (repair/repair trivial-mp input))))))
-(deftest ^:parallel wrap-iso-date-bounds-with-time-portion-test
- (testing "yyyy-mm-ddThh:mm:ss bounds also get wrapped"
- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
- input ["between" {} field "2024-06-15T09:00:00" "2024-06-15T18:00:00"]]
- (is (= ["between" {}
- field
- ["absolute-datetime" {} "2024-06-15T09:00:00" "day"]
- ["absolute-datetime" {} "2024-06-15T18:00:00" "day"]]
- (repair/repair trivial-mp input))))))
+(deftest ^:parallel iso-date-bound-is-a-temporal-sibling-test
+ (testing "a bare ISO date counts as a temporal sibling, so bare 'now' is canonicalised"
+ (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]]
+ (is (= ["between" {} field "2024-01-01" ["now" {}]]
+ (repair/repair trivial-mp ["between" {} field "2024-01-01" "now"])))
+ (is (= ["between" {} field ["now" {}] "2024-12-31"]
+ (repair/repair trivial-mp ["between" {} field "now" "2024-12-31"]))))))
-(deftest ^:parallel wrap-iso-date-bounds-idempotent-test
- (testing "wrapping is idempotent: two wraps == one wrap"
- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
- input ["between" {} field "2024-01-01" "2024-12-31"]
- once (repair/repair trivial-mp input)
- twice (repair/repair trivial-mp once)]
- (is (= once twice)))))
+;;; ============================================================
+;;; Pass 2.95 - hoist a temporal bucket from an absolute-datetime literal onto the ref
+;;; ============================================================
-(deftest ^:parallel wrap-iso-date-bounds-then-swap-test
- (testing "wrap + swap compose: out-of-order bare ISO strings wrap then swap"
- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
- input ["between" {} field "2024-12-31" "2024-01-01"]]
- (is (= ["between" {}
- field
- ["absolute-datetime" {} "2024-01-01" "day"]
- ["absolute-datetime" {} "2024-12-31" "day"]]
- (repair/repair trivial-mp input))))))
+(def ^:private created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]])
+
+(defn- created-at-bucketed [unit]
+ ["field" {"temporal-unit" unit} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]])
+
+(defn- abs-dt [literal unit]
+ ["absolute-datetime" {} literal unit])
+
+(defn- week-start
+ "`literal` truncated to the start of its week - what the hoist aligns a `week`-bucketed literal to.
+
+ Computed rather than written out: `u.date/truncate` reads the site `start-of-week` setting, other
+ namespaces mutate it, and the tests below are `^:parallel`. Under the default Sunday `2025-01-01`
+ gives `2024-12-29`."
+ [literal]
+ (str (u.date/truncate (u.date/parse literal) :week)))
+
+(defn- repair-in-stage
+ "Run `repair` on a query carrying `stage-kvs`, returning the repaired first stage. The hoist pass
+ is scoped to a stage's `filters` and `expressions`, so its inputs have to be given in one of those
+ positions."
+ [stage-kvs]
+ (-> (repair/repair trivial-mp
+ {"lib/type" "mbql/query"
+ "stages" [(merge {"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}
+ stage-kvs)]})
+ (get "stages")
+ first))
+
+(defn- repair-filter [filter-clause]
+ (first (get (repair-in-stage {"filters" [filter-clause]}) "filters")))
+
+(defn- hoist-filter
+ "Run only Pass 2.95 over a stage carrying `filter-clause`, returning the rewritten clause.
+
+ Unlike [[repair-filter]] this skips Pass 6, so it can show what the pass itself declines - Pass 6's
+ `unencodable-temporal-clause-error!` throws on most of those shapes."
+ [filter-clause]
+ (-> (#'repair/hoist-temporal-buckets*
+ {"lib/type" "mbql/query"
+ "stages" [{"lib/type" "mbql.stage/mbql" "filters" [filter-clause]}]})
+ (get-in ["stages" 0 "filters"])
+ first))
+
+(deftest ^:parallel hoist-temporal-bucket-comparison-test
+ (testing "a bucketed absolute-datetime literal moves its unit onto the field ref"
+ (doseq [op ["=" "!=" "<" "<=" ">" ">="]]
+ (is (= [op {} (created-at-bucketed "month") "2025-01-01"]
+ (repair-filter [op {} created-at (abs-dt "2025-01-01" "month")]))
+ op))))
+
+(deftest ^:parallel hoist-temporal-bucket-units-test
+ (testing "every date unit moves onto the field ref, and the literal aligns to the unit it gave up"
+ ;; `week` is the only date unit `2025-01-01` is not already aligned to - it is a Wednesday
+ (doseq [[unit literal] {"default" "2025-01-01"
+ "day" "2025-01-01"
+ "week" (week-start "2025-01-01")
+ "month" "2025-01-01"
+ "quarter" "2025-01-01"
+ "year" "2025-01-01"}]
+ (is (= ["=" {} (created-at-bucketed unit) literal]
+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01" unit)]))
+ unit)))
+ (testing "a datetime literal also takes the sub-day units"
+ (doseq [[unit literal] {"default" "2025-01-01T10:30:00"
+ "second" "2025-01-01T10:30:00"
+ "minute" "2025-01-01T10:30:00"
+ "hour" "2025-01-01T10:00"
+ "day" "2025-01-01T00:00"
+ "month" "2025-01-01T00:00"}]
+ (is (= ["=" {} (created-at-bucketed unit) literal]
+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01T10:30:00" unit)]))
+ unit))))
+
+(deftest ^:parallel hoist-temporal-bucket-partial-date-test
+ (testing "`yyyy-MM` and `yyyy` literals are widened to the first day they name"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-03-01"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-03" "month")])))
+ (is (= ["=" {} (created-at-bucketed "year") "2025-01-01"]
+ (repair-filter ["=" {} created-at (abs-dt "2025" "year")])))))
+
+(deftest ^:parallel hoist-temporal-bucket-multi-value-test
+ (testing "variadic `=` / `!=` and `in` / `not-in`: every literal carries the same bucket"
+ (doseq [op ["=" "!=" "in" "not-in"]]
+ (is (= [op {} (created-at-bucketed "month") "2025-01-01" "2025-03-01"]
+ (repair-filter [op {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-03-01" "month")]))
+ op)))
+ (testing "literals bucketed by different units are left alone"
+ (let [input ["in" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-03-01" "year")]]
+ (is (= input (hoist-filter input)))))
+ (testing "a mix of bucketed and bare literals is left alone"
+ (let [input ["in" {} created-at (abs-dt "2025-01-01" "month") "2025-03-01"]]
+ (is (= input (hoist-filter input))))))
+
+(deftest ^:parallel hoist-temporal-bucket-expression-ref-test
+ (testing "a custom-column ref takes the bucket the same way a field ref does"
+ (is (= ["=" {} ["expression" {"temporal-unit" "month"} "Ship Date"] "2025-01-01"]
+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-01-01" "month")])))))
+
+;; The hoisted clause truncates the column, so the literal has to be truncated too or the clause
+;; asks a different question - `date_trunc(month, col) = "2025-06-15"` is never true, and the
+;; ordered heads land up to one bucket out. `optimize-temporal-filters` does this for a `field` ref
+;; and never for an `expression` ref, so the pass does it itself. These are shape assertions; the
+;; row-level proof that alignment reproduces the field-ref answer lives next to that middleware, in
+;; `expression-ref-hoisted-bucket-needs-an-aligned-literal-test`.
+(deftest ^:parallel hoist-temporal-bucket-aligns-literal-test
+ (testing "the hoisted literal is truncated to the unit it gave up"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-01"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15" "month")])))
+ (is (= ["=" {} (created-at-bucketed "quarter") "2025-04-01"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15" "quarter")])))
+ (is (= ["=" {} ["expression" {"temporal-unit" "month"} "Ship Date"] "2025-06-01"]
+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-15" "month")]))))
+ (testing "`during` names the unit containing its literal, so this is its mainline case"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-01"]
+ (repair-filter ["during" {} created-at "2025-06-15" "month"]))))
+ (testing "both `between` bounds align"
+ (is (= ["between" {} (created-at-bucketed "month") "2025-06-01" "2025-09-01"]
+ (repair-filter ["between" {} created-at (abs-dt "2025-06-15" "month") (abs-dt "2025-09-20" "month")]))))
+ (testing "so does every literal of a variadic comparison"
+ (is (= ["in" {} (created-at-bucketed "month") "2025-06-01" "2025-09-01"]
+ (repair-filter ["in" {} created-at (abs-dt "2025-06-15" "month") (abs-dt "2025-09-20" "month")]))))
+ (testing "a literal already aligned comes back as written, not reformatted"
+ (doseq [[literal unit] [["2025-06-01" "month"] ["2025-06-15" "day"]
+ ["2025-01-01T10:30:00" "minute"] ["2025-01-01T10:30:00" "second"]]]
+ (is (= ["=" {} (created-at-bucketed unit) literal]
+ (repair-filter ["=" {} created-at (abs-dt literal unit)]))
+ (str literal " " unit))))
+ (testing "`default` names no unit, so its literal is never truncated"
+ (is (= ["=" {} (created-at-bucketed "default") "2025-06-15"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15" "default")]))))
+ (testing "a `Z`-suffixed literal truncates to `2025-06-01T00:00Z[UTC]`, which is not ISO, so it is kept as written"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-15T10:00:00Z"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15T10:00:00Z" "month")]))))
+ (testing "an offset literal does truncate to an ISO string, so it aligns"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-01T00:00+02:00"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15T10:00:00+02:00" "month")])))))
+
+(deftest ^:parallel hoist-temporal-bucket-year-on-expression-ref-test
+ (testing "`year` is left on the literal when the ref is an `expression`"
+ ;; `year` is both a truncation and an extraction unit, so `lib/type-of` calls a `year`-bucketed
+ ;; ref with no type `:type/Integer` - which is exactly an `expression` ref - and the QP fails
+ ;; trying to read the date literal as an integer. Declining leaves it to Pass 6.
+ (doseq [input [["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-01" "year")]
+ ["between" {} ["expression" {} "Ship Date"] (abs-dt "2025-01-01" "year") (abs-dt "2026-01-01" "year")]
+ ["during" {} ["expression" {} "Ship Date"] "2025-06-01" "year"]]]
+ (is (= input (hoist-filter input)) (pr-str input))))
+ (testing "a `field` ref carries a type, so the same bucket still moves onto it"
+ (is (= ["=" {} (created-at-bucketed "year") "2025-01-01"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-06-01" "year")])))
+ (is (= ["=" {} (created-at-bucketed "year") "2025-01-01"]
+ (repair-filter ["during" {} created-at "2025-06-01" "year"]))))
+ (testing "every other unit still moves onto an `expression` ref"
+ (doseq [[unit literal] {"day" "2025-06-15"
+ "week" (week-start "2025-06-15")
+ "month" "2025-06-01"
+ "quarter" "2025-04-01"}]
+ (is (= ["=" {} ["expression" {"temporal-unit" unit} "Ship Date"] literal]
+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-15" unit)]))
+ unit))))
+
+(deftest ^:parallel hoist-temporal-bucket-between-test
+ (testing "both bounds bucketed by the same unit: hoisted, bounds become plain strings"
+ (is (= ["between" {} (created-at-bucketed "month") "2025-01-01" "2025-06-01"]
+ (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "month")]))))
+ (testing "hoisting composes with the swap pass for out-of-order bounds"
+ (is (= ["between" {} (created-at-bucketed "month") "2025-01-01" "2025-06-01"]
+ (repair-filter ["between" {} created-at (abs-dt "2025-06-01" "month") (abs-dt "2025-01-01" "month")]))))
+ (testing "bounds bucketed by different units are left alone"
+ (let [input ["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "year")]]
+ (is (= input (hoist-filter input)))))
+ (testing "one bound bucketed by a unit that changes it (`month`): left alone"
+ (let [input ["between" {} created-at (abs-dt "2025-01-01" "month") "2025-06-01"]]
+ (is (= input (hoist-filter input)))))
+ (testing "one bound bucketed by `day` or `default`: the wrapper goes and the bound stays a plain date"
+ (is (= ["between" {} created-at "2025-01-01" ["now" {}]]
+ (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "day") "now"])))
+ (is (= ["between" {} created-at "2025-01-01" "2025-06-30"]
+ (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "default") "2025-06-30"])))))
+
+(deftest ^:parallel hoist-temporal-bucket-during-test
+ (testing "`during` becomes `=` against the bucketed field - what the clause already means"
+ (doseq [[unit literal] {"day" "2025-01-01"
+ "week" (week-start "2025-01-01")
+ "month" "2025-01-01"
+ "quarter" "2025-01-01"
+ "year" "2025-01-01"}]
+ (is (= ["=" {} (created-at-bucketed unit) literal]
+ (repair-filter ["during" {} created-at "2025-01-01" unit]))
+ unit)))
+ (testing "a datetime literal with a sub-day unit is rewritten the same way"
+ (is (= ["=" {} (created-at-bucketed "hour") "2025-01-01T10:00"]
+ (repair-filter ["during" {} created-at "2025-01-01T10:30:00" "hour"]))))
+ (testing "a sub-day unit on a date-only literal is left alone, like the comparison arm"
+ (doseq [[literal unit] [["2025-01-01" "hour"] ["2025-01-01" "minute"] ["2025-01-01" "second"]
+ ["2025-03" "hour"] ["2025" "hour"]]]
+ (let [input ["during" {} created-at literal unit]]
+ (is (= input (hoist-filter input)) (str literal " " unit)))))
+ (testing "the `default` unit is left alone: `during` does not accept it, so Pass 6 reports it"
+ (let [input ["during" {} created-at "2025-01-01" "default"]]
+ (is (= input (hoist-filter input)))))
+ (testing "a field that already carries a temporal-unit is never clobbered"
+ (let [input ["during" {} (created-at-bucketed "day") "2025-01-01" "month"]]
+ (is (= input (hoist-filter input))))))
+
+(deftest ^:parallel hoist-temporal-bucket-only-comparison-heads-test
+ (testing "a non-comparison clause of the same arity is not a comparison and is left alone"
+ (doseq [head ["coalesce" "datetime-add" "concat"]]
+ (let [input [head {} created-at (abs-dt "2025-01-01" "month")]]
+ (is (= input (hoist-filter input)) head)))))
+
+(deftest ^:parallel hoist-temporal-bucket-scope-test
+ (testing "a stage's filters and expressions are rewritten; the same comparison elsewhere is not"
+ ;; Pass 2.95 alone, not `repair`: Pass 6 rejects the shapes this pass declines.
+ (let [cmp [">" {} created-at (abs-dt "2025-01-01" "month")]
+ cmp' [">" {} (created-at-bucketed "month") "2025-01-01"]
+ agg ["count-where" {} cmp]
+ expr ["case" {"lib/expression-name" "E"} [[cmp 1]] 0]
+ ob ["asc" {} ["sum-where" {} ["field" {} ["Sample" "PUBLIC" "ORDERS" "TOTAL"]] cmp]]
+ brk ["case" {} [[cmp 1]] 0]
+ fld ["case" {} [[cmp 2]] 0]
+ stage (-> (#'repair/hoist-temporal-buckets*
+ {"lib/type" "mbql/query"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "filters" [cmp]
+ "aggregation" [agg]
+ "expressions" [expr]
+ "order-by" [ob]
+ "breakout" [brk]
+ "fields" [fld]}]})
+ (get "stages")
+ first)]
+ (is (= [cmp'] (get stage "filters")))
+ (is (= [agg] (get stage "aggregation")))
+ (is (= [["case" {"lib/expression-name" "E"} [[cmp' 1]] 0]] (get stage "expressions")))
+ (is (= [ob] (get stage "order-by")))
+ (is (= [brk] (get stage "breakout")))
+ (is (= [fld] (get stage "fields"))))))
+
+(deftest ^:parallel hoist-temporal-bucket-join-scope-test
+ ;; Pass 2.95 alone, not `repair`: Pass 6 rejects the conditions shape.
+ (let [cmp [">" {} created-at (abs-dt "2025-01-01" "month")]
+ cmp' [">" {} (created-at-bucketed "month") "2025-01-01"]
+ stage (-> (#'repair/hoist-temporal-buckets*
+ {"lib/type" "mbql/query"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "joins" [{"lib/type" "mbql/join"
+ "alias" "J"
+ "conditions" [cmp]
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "filters" [cmp]}]}]}]})
+ (get "stages")
+ first)]
+ (testing "a join's own `conditions:` are not a stage key the optimizer rewrites: left as written"
+ (is (= [cmp] (get-in stage ["joins" 0 "conditions"]))))
+ (testing "a join's inner stage is a stage, and the optimizer covers it: hoisted"
+ (is (= [cmp'] (get-in stage ["joins" 0 "stages" 0 "filters"]))))))
+
+(deftest ^:parallel hoist-temporal-bucket-stage-marker-drift-test
+ ;; The gate is `stage-like-map?` and not a `lib/type` match on purpose: Pass 2 only fills a
+ ;; *missing* marker, so a stage the model mis-marked would drop out of a marker gate entirely.
+ (testing "a stage the model mis-marked is still recognised structurally, so the marker complaint
+ reaches the model instead of a misleading temporal one"
+ (doseq [marker ["mbql/stage" "mbql.stage/mbql5"]]
+ (let [stage (-> (#'repair/hoist-temporal-buckets*
+ {"lib/type" "mbql/query"
+ "stages" [{"lib/type" marker
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "filters" [[">" {} created-at (abs-dt "2025-01-01" "month")]]}]})
+ (get "stages")
+ first)]
+ (is (= [[">" {} (created-at-bucketed "month") "2025-01-01"]] (get stage "filters")) marker)))))
+
+(deftest ^:parallel hoist-temporal-bucket-nested-in-filter-test
+ (testing "hoisting reaches comparisons nested under boolean combinators"
+ (is (= ["and" {} ["=" {} (created-at-bucketed "month") "2025-01-01"]
+ ["not" {} ["=" {} (created-at-bucketed "year") "2024-01-01"]]]
+ (repair-filter ["and" {}
+ ["=" {} created-at (abs-dt "2025-01-01" "month")]
+ ["not" {} ["=" {} created-at (abs-dt "2024-01-01" "year")]]])))))
+
+(deftest ^:parallel hoist-temporal-bucket-inside-case-test
+ (testing "a comparison nested in a `case` inside a filter is hoisted - the QP optimizer walks
+ filters to the same depth, so the two forms compile alike"
+ (is (= [">" {} ["case" {} [[[">" {} (created-at-bucketed "month") "2025-01-01"] 1]] 0] 0]
+ (repair-filter [">" {} ["case" {} [[[">" {} created-at (abs-dt "2025-01-01" "month")] 1]] 0] 0])))))
+
+(deftest ^:parallel hoist-temporal-bucket-cross-stage-ref-test
+ (testing "a cross-stage string-named ref is bucketed the same way as a portable-FK ref"
+ (is (= ["=" {} ["field" {"temporal-unit" "month"} "CREATED_AT"] "2025-01-01"]
+ (repair-filter ["=" {} ["field" {} "CREATED_AT"] (abs-dt "2025-01-01" "month")])))))
+
+(deftest ^:parallel hoist-temporal-bucket-after-post-agg-split-test
+ (testing "a post-aggregation filter that Pass 2.9 moves into a new stage is hoisted there"
+ (let [stages (-> (repair/repair mp-temporal
+ {"lib/type" "mbql/query"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "aggregation" [["max" {} created-at]]
+ "breakout" [["field" {} ["Sample" "PUBLIC" "ORDERS" "USER_ID"]]]
+ "filters" [["<" {} ["aggregation" {} 0]
+ (abs-dt "2025-01-01" "month")]]}]})
+ (get "stages"))
+ [_ _ ref literal] (first (get (second stages) "filters"))]
+ (is (= 2 (count stages)))
+ (is (= "month" (get-in ref [1 "temporal-unit"])))
+ (is (= "2025-01-01" literal)))))
+
+(deftest ^:parallel hoist-temporal-bucket-guards-test
+ (testing "a sub-day unit on a date-only literal is not valid MBQL 5: the pass declines it, Pass 6 reports it"
+ (doseq [unit ["hour" "minute" "second" "millisecond"]]
+ (let [input ["=" {} created-at (abs-dt "2025-01-01" unit)]]
+ (is (= input (hoist-filter input)) unit))))
+ (testing "extraction units are a different operation on a field ref: the pass declines them, Pass 6 reports them"
+ (let [input ["=" {} created-at (abs-dt "2025-01-01" "month-of-year")]]
+ (is (= input (hoist-filter input)))))
+ ;; the one shape Pass 6 deliberately stays quiet about, so `repair-filter` (the full pipeline) is
+ ;; what this block has to exercise
+ (testing "a literal that does not parse is left alone, so the resolve-time check still reports it"
+ (let [input ["=" {} created-at (abs-dt "2024-13-45" "month")]]
+ (is (= input (repair-filter input)))))
+ (testing "a field that already carries a temporal-unit is never clobbered"
+ (let [input ["=" {} (created-at-bucketed "day") (abs-dt "2025-01-01" "month")]]
+ (is (= input (hoist-filter input)))))
+ (testing "a non-empty options map is never rewritten (it would annihilate an expression name; Pass 6 reports the clause)"
+ (let [input ["=" {} created-at ["absolute-datetime" {"lib/expression-name" "Cutoff"} "2025-01-01" "day"]]]
+ (is (= input (hoist-filter input))))))
+
+(deftest ^:parallel hoist-temporal-bucket-normalisation-test
+ (testing "the unit is matched case-insensitively, as lib normalises it"
+ (is (= ["=" {} (created-at-bucketed "default") "2025-01-01"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01" "DEFAULT")]))))
+ (testing "a `nil` temporal-unit counts as none - lib drops it"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-01-01"]
+ (repair-filter ["=" {} ["field" {"temporal-unit" nil} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ (abs-dt "2025-01-01" "month")]))))
+ (testing "a type hint on the literal does not block the rewrite"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-01-01"]
+ (repair-filter ["=" {} created-at ["absolute-datetime" {"base-type" "type/Date"} "2025-01-01" "month"]])))))
+
+(deftest ^:parallel hoist-temporal-bucket-idempotent-test
+ (testing "hoisting is idempotent"
+ (let [once (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "month")])]
+ (is (= once (repair-filter once))))))
+
+(deftest ^:parallel unencodable-temporal-clause-error-test
+ (testing "a temporal clause the hoist pass could not move becomes a retryable agent error"
+ (doseq [[label stage-kvs]
+ {"in `aggregation:`"
+ {"aggregation" [["count-where" {} [">" {} created-at (abs-dt "2025-01-01" "month")]]]}
+ "with the literal on the left"
+ {"filters" [["=" {} (abs-dt "2025-01-01" "month") created-at]]}
+ "compared to an already-bucketed ref"
+ {"filters" [["=" {} (created-at-bucketed "day") (abs-dt "2025-01-01" "month")]]}
+ "with an extraction unit"
+ {"filters" [["=" {} created-at (abs-dt "2025-01-01" "month-of-year")]]}
+ "`between` bounds bucketed by different units"
+ {"filters" [["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "year")]]}
+ "`during` against an already-bucketed ref"
+ {"filters" [["during" {} (created-at-bucketed "day") "2025-01-01" "month"]]}
+ "`during` with a sub-day unit on a date-only literal"
+ {"filters" [["during" {} created-at "2025-01-01" "hour"]]}
+ ;; `millisecond` is a legal datetime bucket, but not one the QP turns into a range, so Pass
+ ;; 2.95 declines it even on a datetime literal
+ "`millisecond` on a datetime literal"
+ {"filters" [["=" {} created-at (abs-dt "2025-01-01T10:00:00.123" "millisecond")]]}
+ "`during` with `millisecond` on a datetime literal"
+ {"filters" [["during" {} created-at "2025-01-01T10:00:00.123" "millisecond"]]}
+ ;; a `value` clause keeps its raw string keys and still passes validation (BOT-2095)
+ "a `value` clause carrying a unit"
+ {"filters" [["=" {} created-at ["value" {"base-type" "type/DateTime" "unit" "day"} "2025-01-01"]]]}
+ ;; E7 still reaches `expressions:` - Pass 2.95 declines an extraction unit everywhere
+ "with an extraction unit in a `case` under `expressions:`"
+ {"expressions" [["case" {"lib/expression-name" "E"}
+ [[[">" {} created-at (abs-dt "2025-01-01" "month-of-year")] 1]] 0]]}}]
+ (testing label
+ (try
+ (repair-in-stage stage-kvs)
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (let [d (ex-data e)]
+ (is (true? (:agent-error? d)))
+ (is (= :unencodable-temporal-clause (:error d)))))))))
+ (testing "shapes that round-trip intact are never flagged"
+ (is (= ["=" {} (created-at-bucketed "month") "2025-01-01"]
+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01" "month")])))
+ (testing "a `value` clause with no unit, or a nil unit, survives the hop"
+ (doseq [opts [{"base-type" "type/DateTime"}
+ {"base-type" "type/DateTime" "unit" nil}]]
+ (let [input ["=" {} created-at ["value" opts "2025-01-01"]]]
+ (is (= input (repair-filter input)) (pr-str opts)))))
+ (testing "a hoistable bucket in a `case` under `expressions:` is rewritten, not flagged"
+ (is (= [["case" {"lib/expression-name" "E"}
+ [[[">" {} (created-at-bucketed "month") "2025-01-01"] 1]] 0]]
+ (get (repair-in-stage
+ {"expressions" [["case" {"lib/expression-name" "E"}
+ [[[">" {} created-at (abs-dt "2025-01-01" "month")] 1]] 0]]})
+ "expressions")))))
+ (testing "an unparseable literal is left for resolve's `:invalid-temporal-literal` to name"
+ (let [input ["=" {} created-at (abs-dt "2024-13-45" "month")]]
+ (is (= input (repair-filter input))))))
+
+(deftest ^:parallel unencodable-temporal-clause-message-test
+ (testing "the message steers to a half-open range, and names `year` on an `expression` ref as needing it"
+ ;; `year` is also an extraction unit, so on an untyped `expression` ref the QP reads the bucketed
+ ;; column as an integer and fails on the date literal; and `between` compiles to SQL BETWEEN wherever
+ ;; `optimize-temporal-filters` does not run - every `expression` ref, all of `aggregation:` - so a
+ ;; plain upper date keeps only midnight of the last day
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"`>=` its first day and `<` the first day after it.*not `between`.*`year` on an `expression` reference"
+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-01" "year")])))))
-(deftest ^:parallel wrap-iso-date-bounds-non-iso-untouched-test
- (testing "random strings (not yyyy-mm-dd shaped) are left alone"
- (let [field ["field" {} ["Sample" "PUBLIC" "X"]]
- input ["between" {} field "hello" "world"]]
- (is (= input (repair/repair trivial-mp input))))))
+;;; ============================================================
+;;; Pass 1.867 - rewrite temporal-extract to its get-* equivalent
+;;; ============================================================
+
+(deftest ^:parallel rewrite-temporal-extract-test
+ (testing "each extraction unit becomes its get-* clause, week modes passed on"
+ (doseq [[unit expected] {"year-of-era" ["get-year" {} created-at]
+ "quarter-of-year" ["get-quarter" {} created-at]
+ "month-of-year" ["get-month" {} created-at]
+ "day-of-month" ["get-day" {} created-at]
+ "day-of-week" ["get-day-of-week" {} created-at]
+ "day-of-week-iso" ["get-day-of-week" {} created-at "iso"]
+ "hour-of-day" ["get-hour" {} created-at]
+ "minute-of-hour" ["get-minute" {} created-at]
+ "second-of-minute" ["get-second" {} created-at]
+ "week-of-year-iso" ["get-week" {} created-at "iso"]
+ "week-of-year-us" ["get-week" {} created-at "us"]
+ "week-of-year-instance" ["get-week" {} created-at "instance"]}]
+ (is (= ["=" {} expected 2]
+ (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at unit] 2]))
+ unit))))
+
+(deftest ^:parallel rewrite-temporal-extract-preserves-options-test
+ (testing "the clause's options survive the rewrite"
+ (is (= ["get-month" {"lib/expression-name" "Mo"} created-at]
+ (repair/repair trivial-mp ["temporal-extract" {"lib/expression-name" "Mo"} created-at "month-of-year"])))))
+
+(deftest ^:parallel rewrite-temporal-extract-idempotent-test
+ (testing "rewriting is idempotent"
+ (let [once (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at "day-of-week"] 2])]
+ (is (= once (repair/repair trivial-mp once))))))
+
+(deftest ^:parallel rewrite-temporal-extract-week-mode-slot-test
+ (testing "the optional week-mode slot is dropped: the unit alone decides, as in SQL compilation"
+ (is (= ["=" {} ["get-day-of-week" {} created-at "iso"] 2]
+ (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at "day-of-week-iso" "us"] 2])))
+ (is (= ["=" {} ["get-month" {} created-at] 2]
+ (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at "month-of-year" "iso"] 2])))))
;;; ============================================================
;;; Pass 1.865 - wrap bare "now" literals in temporal-comparison contexts
@@ -771,11 +1204,8 @@
(let [field-with-unit ["field" {"temporal-unit" "day"}
["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
input ["between" {} field-with-unit "2024-01-01" "now"]]
- ;; ISO date wrapped (Pass 1.86), bare 'now' wrapped (Pass 1.865).
- (is (= ["between" {}
- field-with-unit
- ["absolute-datetime" {} "2024-01-01" "day"]
- ["now" {}]]
+ ;; the ISO bound stays exactly as written; only the bare 'now' is canonicalised
+ (is (= ["between" {} field-with-unit "2024-01-01" ["now" {}]]
(repair/repair trivial-mp input)))))
(testing "between(<numeric-field>, 'now', 'now'): no temporal sibling, leave alone"
;; The expression's column is non-temporal and bare 'now' is the only thing in sight;
@@ -810,47 +1240,37 @@
(is (= once twice)))))
(deftest ^:parallel swap-between-bounds-iso-string-test
- ;; Note: bare ISO strings are wrapped by Pass 1.86 ("wrap-iso-date-bounds*") *before*
- ;; this swap pass runs, so the canonical post-repair form has \"[absolute-datetime ...]\"
- ;; bounds. The swap still proceeds because \"swap-between-bounds*\" knows how to extract
- ;; the inner ISO string for comparison.
- (testing "ISO-8601 date strings ordered chronologically: wrapped + swapped"
+ ;; Bare ISO strings now reach this pass unchanged; `swap-between-bounds*` compares them
+ ;; directly via its `string?` arm (ISO-8601 lex order is chronological order).
+ (testing "ISO-8601 date strings out of order: swapped, still bare"
(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
input ["between" {} field "2024-12-31" "2024-01-01"]]
- (is (= ["between" {}
- field
- ["absolute-datetime" {} "2024-01-01" "day"]
- ["absolute-datetime" {} "2024-12-31" "day"]]
+ (is (= ["between" {} field "2024-01-01" "2024-12-31"]
(repair/repair trivial-mp input)))))
- (testing "ISO-8601 datetime strings: wrapped + swapped"
+ (testing "ISO-8601 datetime strings out of order: swapped, time portion preserved"
(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
input ["between" {} field "2024-06-15T18:00:00" "2024-06-15T09:00:00"]]
- (is (= ["between" {}
- field
- ["absolute-datetime" {} "2024-06-15T09:00:00" "day"]
- ["absolute-datetime" {} "2024-06-15T18:00:00" "day"]]
+ (is (= ["between" {} field "2024-06-15T09:00:00" "2024-06-15T18:00:00"]
(repair/repair trivial-mp input)))))
- (testing "in-order ISO strings: wrapped, no swap"
+ (testing "in-order ISO strings: untouched"
(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
input ["between" {} field "2024-01-01" "2024-12-31"]]
- (is (= ["between" {}
- field
- ["absolute-datetime" {} "2024-01-01" "day"]
- ["absolute-datetime" {} "2024-12-31" "day"]]
- (repair/repair trivial-mp input))))))
+ (is (= input (repair/repair trivial-mp input))))))
(deftest ^:parallel swap-between-bounds-absolute-datetime-test
+ ;; A bare clause outside a stage, so Pass 2.95 does not apply and the bounds stay
+ ;; wrapped - the subject here is `between-bound-comparable`'s absolute-datetime arm, not hoisting.
(testing "out-of-order absolute-datetime clauses: swap by inner ISO string"
(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
- hi ["absolute-datetime" {} "2024-12-31" "day"]
- lo ["absolute-datetime" {} "2024-01-01" "day"]
+ hi ["absolute-datetime" {} "2024-12-31T10:00:00" "hour"]
+ lo ["absolute-datetime" {} "2024-01-01T10:00:00" "hour"]
input ["between" {} field hi lo]]
(is (= ["between" {} field lo hi]
(repair/repair trivial-mp input)))))
(testing "in-order absolute-datetime clauses are left alone"
(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
- lo ["absolute-datetime" {} "2024-01-01" "day"]
- hi ["absolute-datetime" {} "2024-12-31" "day"]
+ lo ["absolute-datetime" {} "2024-01-01T10:00:00" "hour"]
+ hi ["absolute-datetime" {} "2024-12-31T10:00:00" "hour"]
input ["between" {} field lo hi]]
(is (= input (repair/repair trivial-mp input))))))
@@ -1411,11 +1831,33 @@
;; deliberately wrap values in a vector to trigger Pass 1.82
(gen/fmap (fn [[f vs]] ["in" {} f vs])
(gen/tuple gen-field-clause (gen/vector gen/string-alphanumeric 1 4)))
- ;; between with maybe-out-of-order ISO date strings (Pass 1.86 + 1.87)
+ ;; between with maybe-out-of-order ISO date strings (Pass 1.87 swap)
(gen/fmap (fn [[f a b]] ["between" {} f a b])
(gen/tuple gen-field-clause-with-temporal
(gen/elements ["2024-01-01" "2024-06-30" "2024-12-31"])
(gen/elements ["2023-01-01" "2024-12-31" "now"])))
+ ;; comparison against a bucketed absolute-datetime literal, hoistable and not (Pass 2.95)
+ (gen/fmap (fn [[op f d unit]] [op {} f ["absolute-datetime" {} d unit]])
+ (gen/tuple (gen/elements ["=" "!=" "<" "<=" ">" ">="])
+ gen-field-clause
+ (gen/elements ["2024-01-01" "2024-06-30" "2024-01-01T10:30:00"])
+ (gen/elements ["default" "day" "month" "year" "hour" "month-of-year"])))
+ ;; between with absolute-datetime bounds whose units may disagree (Pass 2.95)
+ (gen/fmap (fn [[f a b ua ub]]
+ ["between" {} f ["absolute-datetime" {} a ua] ["absolute-datetime" {} b ub]])
+ (gen/tuple gen-field-clause
+ (gen/elements ["2024-01-01" "2024-06-30"])
+ (gen/elements ["2024-12-31" "2023-01-01"])
+ (gen/elements ["day" "month" "year"])
+ (gen/elements ["day" "month" "year"])))
+ ;; temporal-extract, including the week-mode and unmapped units (Pass 1.867)
+ (gen/fmap (fn [[f unit mode n]]
+ ["=" {} (cond-> ["temporal-extract" {} f unit] mode (conj mode)) n])
+ (gen/tuple gen-field-clause
+ (gen/elements ["day-of-week" "month-of-year" "week-of-year-iso"
+ "day-of-week-iso" "day-of-year"])
+ (gen/elements [nil "iso" "us"])
+ gen/small-integer))
;; post-aggregation filter that triggers Pass 2.9 split
(gen/fmap (fn [n] [">" {} ["aggregation" {} 0] n])
gen/small-integer)]))
@@ -2555,6 +2997,9 @@
;;; ============================================================
;;; Pass 6 - friendly error messages (E1..E6)
+;;;
+;;; E7 (`unencodable-temporal-clause-error!`) is tested with Pass 2.95 above, in
+;;; `unencodable-temporal-clause-error-test` - it only fires on what that pass declines.
;;; ============================================================
;;; ----- E1: [field, ...] in aggregation block ---------------------------------------
diff --git a/test/metabase/agent_lib/representations/resolve_test.clj b/test/metabase/agent_lib/representations/resolve_test.clj
index 29b9398bc7b..e93c37dc8cc 100644
--- a/test/metabase/agent_lib/representations/resolve_test.clj
+++ b/test/metabase/agent_lib/representations/resolve_test.clj
@@ -185,6 +185,116 @@
(is (re-find #"Invalid temporal literal" (ex-message ex)))
(is (true? (:agent-error? (ex-data ex)))))))
+;;; ============================================================
+;;; bare temporal literals: validation follows the compared column's type
+;;; ============================================================
+
+(def ^:private created-at ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"])
+(def ^:private total ["Sample" "PUBLIC" "ORDERS" "TOTAL"])
+(def ^:private category ["Sample" "PUBLIC" "PRODUCTS" "CATEGORY"])
+
+(def ^:private coerced-mp
+ "[[mock-metadata]] plus `TS_TEXT`, an ISO-8601 text column an admin coerced to a datetime — the
+ shape in which `:base-type` and `:effective-type` legitimately disagree."
+ (lib.tu/mock-metadata-provider
+ (update mock-metadata :fields conj
+ {:id 112 :name "TS_TEXT" :table-id 10 :base-type :type/Text
+ :effective-type :type/DateTime :coercion-strategy :Coercion/ISO8601->DateTime})))
+
+(defn- resolve-stage
+ "Resolve a single-stage query over `table` carrying `stage-keys`, returning the resolved stage."
+ [table stage-keys]
+ (-> (repr.resolve/resolve-query mp {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [(assoc stage-keys
+ "lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" table])]})
+ (get-in [:stages 0])))
+
+(defn- resolve-error
+ "The `:error` [[resolve-stage]] throws for `stage-keys`, or `nil` when the stage resolves cleanly."
+ [table stage-keys]
+ (try (resolve-stage table stage-keys) nil
+ (catch clojure.lang.ExceptionInfo e (:error (ex-data e)))))
+
+(deftest validate-temporal-literals-leaves-a-text-between-alone-test
+ (testing "date-shaped bounds on a text column are a legal range, not malformed dates"
+ (let [clause (-> (resolve-stage "PRODUCTS"
+ {"filters" [["between" {} ["field" {} category]
+ "2024-01-01 batch A" "2024-06-01 batch Z"]]})
+ (get-in [:filters 0]))]
+ (is (= :between (nth clause 0)))
+ (is (= ["2024-01-01 batch A" "2024-06-01 batch Z"] (subvec clause 3))))))
+
+(deftest validate-temporal-literals-rejects-a-bad-between-bound-test
+ (testing "a bound compared to a temporal column is still checked"
+ (is (= :invalid-temporal-literal
+ (resolve-error "ORDERS" {"filters" [["between" {} ["field" {} created-at]
+ "2024-13-45" "2024-12-31"]]})))))
+
+(deftest validate-temporal-literals-checks-a-between-on-an-untyped-ref-test
+ (testing "an `expression` ref carries no type here, which is no reason to stop checking its bounds"
+ (is (= :invalid-temporal-literal
+ (resolve-error "ORDERS"
+ {"expressions" [["datetime-add" {"lib/expression-name" "Shifted"}
+ ["field" {} created-at] 3 "day"]]
+ "filters" [["between" {} ["expression" {} "Shifted"]
+ "2024-13-45" "2024-12-31"]]})))))
+
+(deftest validate-temporal-literals-ignores-a-model-authored-base-type-test
+ (testing "the metadata provider is authoritative — a wrong stamp does not excuse a bad literal"
+ ;; CREATED_AT is :type/DateTime in the provider, and the QP resolves it from there too;
+ ;; `annotate-field-types` leaves a ref that already carries `:base-type` alone.
+ (is (= :invalid-temporal-literal
+ (resolve-error "ORDERS" {"filters" [["between" {} ["field" {"base-type" "type/Text"} created-at]
+ "2024-13-45" "2024-12-31"]]})))))
+
+(deftest validate-temporal-literals-rejects-a-bad-literal-on-a-temporal-comparison-test
+ (testing "a bare literal compared to a temporal ref is checked, whatever the comparison head"
+ (doseq [clause [["=" {} ["field" {"temporal-unit" "month"} created-at] "2024-13-45"]
+ [">=" {} ["field" {} created-at] "2024-13-45"]
+ ["in" {} ["field" {} created-at] "2024-13-45" "2024-02-01"]]]
+ (testing (pr-str clause)
+ (is (= :invalid-temporal-literal (resolve-error "ORDERS" {"filters" [clause]})))))))
+
+(deftest validate-temporal-literals-skips-a-known-non-temporal-comparison-test
+ (testing "a date-shaped string compared to a text column is a text comparison"
+ (is (nil? (resolve-error "PRODUCTS" {"filters" [["=" {} ["field" {} category]
+ "2024-01-01 batch A"]]}))))
+ (testing "a temporal operand elsewhere in the clause does not make this literal a date"
+ ;; `=` takes any number of operands with no pairwise type check, so a temporal one can sit
+ ;; beside a text one. Rejecting here would be the very defect this pass was fixed for.
+ (is (nil? (resolve-error "PRODUCTS" {"filters" [["=" {} ["field" {} category]
+ ["field" {} created-at]
+ "2024-01-01 batch A"]]})))))
+
+(deftest validate-temporal-literals-descends-past-an-inert-comparison-test
+ (testing "a comparison that is not itself checkable must not swallow a nested one that is"
+ ;; the operand test has to live in the *pattern*: with a head-only pattern this outer `=` matches,
+ ;; `match-many` stops descending, and the nested comparison's literal is never seen.
+ (is (= :invalid-temporal-literal
+ (resolve-error "ORDERS"
+ {"filters" [["=" {} ["field" {} total]
+ ["case" {} [[[">=" {} ["field" {} created-at] "2024-13-45"] 1]]
+ 0]]]})))))
+
+(deftest operand-temporality-types-a-coerced-column-test
+ (testing "a `year` bucket does not make a coerced column look like text — the provider is read"
+ (is (= :temporal
+ (#'repr.resolve/operand-temporality
+ coerced-mp
+ [:field {:temporal-unit :year :base-type :type/Text :effective-type :type/DateTime} 112]))))
+ (testing "a cross-stage ref carries the storage type and the coercion result, and is temporal"
+ ;; the pair a coerced column resolves to is not a contradiction: the coercion result is the
+ ;; type the QP compares a literal against, so one temporal member is enough.
+ (is (= :temporal
+ (#'repr.resolve/operand-temporality
+ coerced-mp
+ [:field {:base-type :type/Text :effective-type :type/DateTime} "TS_TEXT"]))))
+ (testing "an operand nothing here can type stays distinguishable from one typed as not temporal"
+ (is (= :unknown (#'repr.resolve/operand-temporality coerced-mp [:expression {} "Shifted"])))
+ (is (= :non-temporal (#'repr.resolve/operand-temporality coerced-mp [:field {} 111])))))
+
;;; ============================================================
;;; fields (projection)
;;; ============================================================
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index a36379f7bcf..fd16929d1a0 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -19,6 +19,7 @@
[metabase.lib.test-util :as lib.tu]
[metabase.mcp.v2.message :as message]
[metabase.mcp.v2.recovery-hints :as v2-hints]
+ [metabase.metabot.agent.links :as links]
[metabase.metabot.db :as metabot.db]
[metabase.metabot.tools.construct :as construct]
[metabase.metabot.tools.entity-details :as entity-details]
@@ -26,7 +27,8 @@
[metabase.models.interface :as mi]
[metabase.models.serialization.resolve :as serdes.resolve]
[metabase.models.serialization.resolve.mp :as resolve.mp]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [metabase.util.json :as json]))
(set! *warn-on-reflection* true)
@@ -96,16 +98,21 @@
(f)))
(def ^:private mp-coerced
- "ORDERS with a UNIX-seconds column coerced to a timestamp: `base-type :type/Integer`,
- `effective-type :type/DateTime`. Exercises the editor gate's unsuppressed expression
- type check against coerced columns (the #41122 false-positive class that
- `*suppress-expression-type-check?*` exists for)."
+ "ORDERS with two columns an admin coerced to a timestamp - UNIX seconds and ISO-8601 text - each
+ paired with a plain column of the same `base-type`, so a test can vary the coercion alone.
+
+ Exercises the editor gate's unsuppressed expression type check against coerced columns (the
+ #41122 false-positive class that `*suppress-expression-type-check?*` exists for), and the
+ temporal-bucket gate, whose whole job is telling a coerced column from a non-temporal one."
(lib.tu/mock-metadata-provider
{:database {:id 1 :name "Sample"}
:tables [{:id 10 :name "ORDERS" :schema "PUBLIC" :db-id 1}]
:fields [{:id 100 :name "ID" :table-id 10 :base-type :type/Integer}
{:id 104 :name "CREATED_AT_UNIX" :table-id 10 :base-type :type/Integer
- :effective-type :type/DateTime :coercion-strategy :Coercion/UNIXSeconds->DateTime}]}))
+ :effective-type :type/DateTime :coercion-strategy :Coercion/UNIXSeconds->DateTime}
+ {:id 105 :name "STATUS" :table-id 10 :base-type :type/Text}
+ {:id 106 :name "CREATED_AT_ISO" :table-id 10 :base-type :type/Text
+ :effective-type :type/DateTime :coercion-strategy :Coercion/ISO8601->DateTime}]}))
(defn- with-coerced-mp-and-stubs! [f]
(with-redefs [lib-be/application-database-metadata-provider (fn [_db-id] mp-coerced)
@@ -114,6 +121,23 @@
api/query-check allow-read-check]
(f)))
+(def ^:private mp-temporal
+ "ORDERS with a plain `:type/DateTime` column, for the temporal-clause round-trip tests."
+ (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "Sample"}
+ :tables [{:id 10 :name "ORDERS" :schema "PUBLIC" :db-id 1}]
+ :fields [{:id 100 :name "ID" :table-id 10 :base-type :type/Integer}
+ {:id 102 :name "USER_ID" :table-id 10 :base-type :type/Integer}
+ {:id 103 :name "CREATED_AT" :table-id 10 :base-type :type/DateTime}
+ {:id 104 :name "STATUS" :table-id 10 :base-type :type/Text}]}))
+
+(defn- with-temporal-mp-and-stubs! [f]
+ (with-redefs [lib-be/application-database-metadata-provider (fn [_db-id] mp-temporal)
+ construct/resolve-database-id-from-first-stage (fn [_] 1)
+ api/read-check allow-read-check
+ api/query-check allow-read-check]
+ (f)))
+
(defn- query-data
"Build the keyword-keyed external-query map the tool accepts. Tests author the query as a
string-keyed map (the form documented in earlier YAML-era tests); we walk-keywordize so the
@@ -1694,3 +1718,332 @@
"source-table" ["Sample" "PUBLIC" "ORDERS"]
"aggregation" [["count" {}]]}]})))))
(is (false? @seen)))))
+
+(defn- legacy-json-wire
+ "Run the tool on a single-stage ORDERS query carrying `stage-kvs`, returning `[before wire]`.
+
+ `before` is the MBQL 5 query the tool built; `wire` is that query converted to legacy the way
+ `links` does for the `/question#` hash and JSON round-tripped."
+ [stage-kvs]
+ (let [result (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [(merge {"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}
+ stage-kvs)]}))
+ before (get-in result [:structured-output :query])]
+ [before (-> before links/->legacy-mbql json/encode (json/decode true))]))
+
+(defn- stage-clause-counts
+ "Per-stage counts of the clause vectors on a query, for comparing a query with its round-trip."
+ [query]
+ (mapv (fn [stage]
+ (into (sorted-map)
+ (for [k [:filters :aggregation :expressions :breakout :order-by :fields :joins]
+ :when (contains? stage k)]
+ [k (count (get stage k))])))
+ (:stages query)))
+
+(defn- assert-survives-json-hop! [[before wire]]
+ ;; `->legacy-mbql` returns the MBQL 5 query unchanged if conversion throws, and a
+ ;; JSON-round-tripped MBQL 5 query passes `lib/query` - so without this the
+ ;; assertion below could hold while proving nothing.
+ (is (= "query" (:type wire))
+ "->legacy-mbql fell back to MBQL 5 instead of converting")
+ ;; ... and `lib/query` does not throw on a wire whose clause `clean-stage-schema-errors` deleted,
+ ;; so "no throw" is not survival. Compare the clauses - by count, since uuids and idents are
+ ;; regenerated across the hop and deep equality is not available. A clause swapped for a different
+ ;; clause of the same arity would slip through; deletion, the failure mode here, would not.
+ (let [after (lib/query mp-temporal wire)]
+ (is (map? after) "did not survive the JSON round-trip")
+ (is (= (stage-clause-counts before) (stage-clause-counts after))
+ "a clause was silently dropped by the round-trip")))
+
+;; A Metabot-built question reaches the frontend as legacy MBQL inside a base64 `/question#` hash,
+;; so it makes a JSON hop that turns every keyword into a string. Legacy normalization has no
+;; `:decode/normalize` on the temporal-unit enums used by `:absolute-datetime`, `:during` or
+;; `:temporal-extract`, so a query carrying one came back un-normalized and the question page 400'd
+;; on `/api/dataset/query_metadata`. The JSON hop is what makes this reproduce - converting to
+;; legacy alone does not, which is why the plain legacy round-trip gates above never caught it
+;; (BOT-2095).
+(deftest legacy-json-round-trip-temporal-filters-test
+ (with-temporal-mp-and-stubs!
+ (fn []
+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ abs-dt (fn [literal unit] ["absolute-datetime" {} literal unit])]
+ (doseq [[label filter-clause]
+ {"bare date range"
+ ["between" {} created-at "2025-01-01" "2025-12-31"]
+ "absolute-datetime literal carrying a bucket"
+ ["=" {} created-at (abs-dt "2025-01-01" "month")]
+ "absolute-datetime literal with the default unit"
+ ["=" {} created-at (abs-dt "2025-01-01" "default")]
+ "absolute-datetime datetime literal with the default unit"
+ [">" {} created-at (abs-dt "2025-01-01T10:30:00" "default")]
+ "absolute-datetime datetime literal bucketed by hour"
+ ["=" {} created-at (abs-dt "2025-01-01T10:00:00" "hour")]
+ "absolute-datetime year-month literal"
+ ["=" {} created-at (abs-dt "2025-01" "month")]
+ "several bucketed literals in one `in`"
+ ["in" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-03-01" "month")]
+ "between with one redundant bucket and `now`"
+ ["between" {} created-at (abs-dt "2025-01-01" "day") "now"]
+ "during"
+ ["during" {} created-at "2025-01-01" "month"]
+ "during with a sub-day unit"
+ ["during" {} created-at "2025-01-01T10:00:00" "hour"]
+ "temporal-extract"
+ ["=" {} ["temporal-extract" {} created-at "day-of-week"] 2]
+ "temporal-extract carrying a week mode"
+ ["=" {} ["temporal-extract" {} created-at "week-of-year-iso"] 2]
+ "temporal-extract with the optional week-mode slot"
+ ["=" {} ["temporal-extract" {} created-at "day-of-week" "iso"] 2]}]
+ (testing label
+ (assert-survives-json-hop! (legacy-json-wire {"filters" [filter-clause]
+ "aggregation" [["count" {}]]}))))
+ (testing "a bucketed literal compared to a custom column"
+ (assert-survives-json-hop!
+ (legacy-json-wire {"expressions" [["datetime-add" {"lib/expression-name" "Ship Date"} created-at 3 "day"]]
+ "filters" [["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-01-01" "month")]]
+ "aggregation" [["count" {}]]})))
+ (testing "a bucketed literal inside a `case` under `expressions`"
+ (assert-survives-json-hop!
+ (legacy-json-wire {"expressions" [["case" {"lib/expression-name" "Jan"}
+ [[["=" {} created-at (abs-dt "2025-01-01" "month")] 1]] 0]]
+ "aggregation" [["count" {}]]})))
+ (testing "a post-aggregation filter, which repair moves into a second stage"
+ (assert-survives-json-hop!
+ (legacy-json-wire {"aggregation" [["max" {} created-at]]
+ "breakout" [["field" {} ["Sample" "PUBLIC" "ORDERS" "USER_ID"]]]
+ "filters" [["<" {} ["aggregation" {} 0] (abs-dt "2025-01-01" "month")]]})))))))
+
+;; A bucketed temporal clause that Pass 2.95 cannot hoist does not survive the `/question#` hash the
+;; frontend opens: 400 in dev, and in production `lib/query`'s cleaner deletes the clause and the
+;; question silently answers something else. Pass 6 turns each into a retryable agent error (BOT-2095).
+(deftest unencodable-temporal-clause-reaches-the-agent-test
+ (with-temporal-mp-and-stubs!
+ (fn []
+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ abs-dt (fn [literal unit] ["absolute-datetime" {} literal unit])]
+ (doseq [[label stage-kvs]
+ {"in `count-where`"
+ {"aggregation" [["count-where" {} [">" {} created-at (abs-dt "2025-01-01" "month")]]]}
+ "in a join condition"
+ {"joins" [{"lib/type" "mbql/join"
+ "alias" "O2"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}]
+ "conditions" [[">" {} created-at (abs-dt "2025-01-01" "month")]]}]
+ "aggregation" [["count" {}]]}
+ "a `value` clause carrying a unit"
+ {"filters" [["=" {} created-at
+ ["value" {"base-type" "type/DateTime" "unit" "day"} "2025-01-01"]]]
+ "aggregation" [["count" {}]]}}]
+ (testing label
+ (try
+ (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [(merge {"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}
+ stage-kvs)]}))
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (let [d (ex-data e)]
+ (is (true? (:agent-error? d)))
+ (is (= :unencodable-temporal-clause (:error d))))))))))))
+
+;; Repair moves a bucket onto the ref without knowing the column's type; lib accepts a bucket on a
+;; text column, and the QP would then drop it silently on a `field` ref and keep it on an
+;; `expression` ref, where it compiles to nonsense. The gate after resolve turns either into an
+;; agent-facing error (the pre-hoist shape used to be rejected by the expression editor gate).
+(deftest temporal-bucket-on-non-temporal-column-test
+ (with-temporal-mp-and-stubs!
+ (fn []
+ (let [status ["field" {} ["Sample" "PUBLIC" "ORDERS" "STATUS"]]
+ abs-dt (fn [literal unit] ["absolute-datetime" {} literal unit])
+ label-expr ["concat" {"lib/expression-name" "Label"} status "x"]]
+ (doseq [[label stage-kvs]
+ {"hoisted from a bucketed literal"
+ {"filters" [["<" {} status (abs-dt "2025-01-01" "month")]]
+ "aggregation" [["count" {}]]}
+ "written by the model"
+ {"filters" [["=" {} ["field" {"temporal-unit" "month"} ["Sample" "PUBLIC" "ORDERS" "STATUS"]] "2025-01-01"]]
+ "aggregation" [["count" {}]]}
+ "hoisted onto a text custom column"
+ {"expressions" [label-expr]
+ "filters" [["<" {} ["expression" {} "Label"] (abs-dt "2025-01-01" "month")]]
+ "aggregation" [["count" {}]]}
+ "written by the model onto a text custom column"
+ {"expressions" [label-expr]
+ "filters" [["=" {} ["expression" {"temporal-unit" "month"} "Label"] "2025-01-01"]]
+ "aggregation" [["count" {}]]}
+ "hoisted onto a text custom column inside an `expressions` definition"
+ {"expressions" [label-expr
+ ["case" {"lib/expression-name" "Cohort"}
+ [[["<" {} ["expression" {} "Label"] (abs-dt "2025-01-01" "month")] "a"]] "b"]]
+ "aggregation" [["count" {}]]}
+ ;; an extraction unit stamps the *bucket's* type on `effective-type`, so the gate
+ ;; has to look past it to the column's own type - here still Text
+ "written by the model with the bucket's own type stamped on the ref"
+ {"expressions" [label-expr]
+ "breakout" [["expression" {"temporal-unit" "month-of-year"
+ "effective-type" "type/Integer"} "Label"]]
+ "aggregation" [["count" {}]]}
+ ;; the join condition's left-hand ref belongs to the *parent* stage, so the gate has
+ ;; to type it there - the join's own stage has no `Label`
+ "written by the model onto a text custom column in a join condition"
+ {"expressions" [label-expr]
+ "joins" [{"lib/type" "mbql/join"
+ "alias" "O2"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}]
+ "conditions" [["=" {} ["expression" {"temporal-unit" "month"} "Label"]
+ ["field" {"join-alias" "O2"} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]]]}]
+ "aggregation" [["count" {}]]}}]
+ (testing label
+ (try
+ (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [(merge {"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}
+ stage-kvs)]}))
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (let [d (ex-data e)]
+ (is (true? (:agent-error? d)))
+ (is (= :temporal-unit-on-non-temporal-column (:error d))))))))
+ (testing "the message names a custom column as such"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"`Label` is a Text custom column"
+ (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "expressions" [label-expr]
+ "filters" [["<" {} ["expression" {} "Label"] (abs-dt "2025-01-01" "month")]]
+ "aggregation" [["count" {}]]}]}))))
+ (is (thrown-with-msg?
+ ;; Text, not the `Integer` the extraction bucket stamped on `effective-type`
+ clojure.lang.ExceptionInfo #"`Label` is a Text custom column"
+ (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "expressions" [label-expr]
+ "breakout" [["expression" {"temporal-unit" "month-of-year"
+ "effective-type" "type/Integer"} "Label"]]
+ "aggregation" [["count" {}]]}]}))))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"`Status` is a Text column"
+ (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "filters" [["<" {} status (abs-dt "2025-01-01" "month")]]
+ "aggregation" [["count" {}]]}]}))))))
+ ;; The gate is only useful if it leaves working queries alone. Typing an `expression` ref needs
+ ;; the ref's stage, and a join condition mixes two of them: this query runs today, and a gate
+ ;; that resolved every join ref in the join's own stage would reject it with `No expression
+ ;; named "Ship"`.
+ (testing "a join condition bucketing a temporal custom column from the parent stage is accepted"
+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ result (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "expressions" [["datetime-add" {"lib/expression-name" "Ship"} created-at 3 "day"]]
+ "joins" [{"lib/type" "mbql/join"
+ "alias" "O2"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}]
+ "conditions" [["=" {}
+ ["expression" {"temporal-unit" "month"} "Ship"]
+ ["field" {"join-alias" "O2" "temporal-unit" "month"}
+ ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]]]}]
+ "aggregation" [["count" {}]]}]}))]
+ (is (some? (get-in result [:structured-output :query])))))
+ ;; The mirror of the Text case above: an extraction unit stamps `:type/Integer` on
+ ;; `effective-type` while `base-type` keeps the column's own type. This query runs and answers
+ ;; correctly, so reading the stamped type would reject a working query.
+ (testing "a `day-of-week` bucket on a temporal custom column carrying the bucket's type is accepted"
+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ result (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "expressions" [["datetime-add" {"lib/expression-name" "Ship"} created-at 3 "day"]]
+ "breakout" [["expression" {"temporal-unit" "day-of-week"
+ "effective-type" "type/Integer"
+ "base-type" "type/DateTime"} "Ship"]]
+ "aggregation" [["count" {}]]}]}))]
+ (is (some? (get-in result [:structured-output :query])))))
+ ;; `with-temporal-bucket` records the pre-bucket type in `:lib/original-effective-type` and
+ ;; stamps the bucketed one on `:effective-type`, so an extraction unit makes a datetime column
+ ;; look like an integer. The gate strips the bucket before asking for the type; reading the
+ ;; ref's own options instead would reject this legitimate query builder output.
+ (testing "a `month-of-year` bucket on a field ref the query builder produced is accepted"
+ (let [q0 (lib/query mp-temporal (lib.metadata/table mp-temporal 10))
+ col (first (filter #(= 103 (:id %)) (lib/filterable-columns q0)))
+ ref (lib/with-temporal-bucket (lib/ref col) :month-of-year)
+ q (lib/filter q0 (lib/= ref 6))]
+ (is (= [:type/Integer :type/DateTime]
+ ((juxt :effective-type :lib/original-effective-type) (second ref))))
+ (is (= q (repr.repair/assert-temporal-buckets-on-temporal-columns! q))))))))
+
+;; A coerced column's `:effective-type` is its own post-coercion type, not the artifact of an
+;; extraction bucket, so the gate has to keep it: strip it and an ISO-8601 timestamp column reads
+;; back as `:type/Text`. Pass 2.95 funnels every way of asking "filter this column by year" into the
+;; bucketed-ref shape below, so rejecting it left the model with no phrasing that worked - and advice
+;; ("bucket a date / datetime column instead") that pointed back at the same column.
+(deftest temporal-bucket-on-coerced-column-accepted-test
+ (with-coerced-mp-and-stubs!
+ (fn []
+ (doseq [column ["CREATED_AT_UNIX" "CREATED_AT_ISO"]
+ :let [bare ["field" {} ["Sample" "PUBLIC" "ORDERS" column]]
+ bucketed ["field" {"temporal-unit" "year"} ["Sample" "PUBLIC" "ORDERS" column]]]
+ [shape filters]
+ {"a bucketed literal" [["=" {} bare ["absolute-datetime" {} "2024-01-01" "year"]]]
+ "a `during`" [["during" {} bare "2024-01-01" "year"]]
+ "the bucket on the ref" [["=" {} bucketed "2024-01-01"]]
+ "a bucketed `between`" [["between" {} bucketed "2024-01-01" "2024-12-31"]]}]
+ (testing (str column " bucketed by year, written as " shape)
+ (let [result (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "filters" filters
+ "aggregation" [["count" {}]]}]}))]
+ (is (some? (get-in result [:structured-output :query])))))))))
+
+;; The pin for the test above: `year` is an extraction unit as well as a truncation one, and what
+;; buys a coerced column its bucket is the coercion, not the unit. `ID` and `STATUS` are the
+;; uncoerced twins of the two columns accepted above - same `base-type`, no `effective-type`.
+(deftest temporal-bucket-by-year-on-non-temporal-column-rejected-test
+ (with-coerced-mp-and-stubs!
+ (fn []
+ (doseq [[column type-name] {"ID" "Integer", "STATUS" "Text"}]
+ (testing (str column " bucketed by year")
+ (try
+ (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "filters" [["=" {} ["field" {"temporal-unit" "year"}
+ ["Sample" "PUBLIC" "ORDERS" column]]
+ "2024-01-01"]]
+ "aggregation" [["count" {}]]}]}))
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (let [d (ex-data e)]
+ (is (true? (:agent-error? d)))
+ (is (= :temporal-unit-on-non-temporal-column (:error d)))
+ (is (str/includes? (ex-message e) (str "is a " type-name " column")))))))))))
diff --git a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
index 1b6710c7382..c7e4075b9c7 100644
--- a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
+++ b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
@@ -571,6 +571,118 @@
[:field (meta/id :checkins :date) {:base-type :type/Date, :temporal-unit unit}]
[:absolute-datetime #t "2014-05-08" unit]]}))))))))
+(deftest ^:parallel optimize-date-not-equals-null-semantics-test
+ (testing (str "`!=` against a DATE column bucketed by `day` is NOT interchangeable between "
+ "bucket-on-the-ref and bucket-on-the-literal: only the former keeps the `!=`, and "
+ "so the `IS NULL` disjunct `sql.qp` adds to every `!=`. Pass 2.95 in agent-lib "
+ "repair hoists the bucket onto the ref for exactly this reason (BOT-2095).")
+ (testing "bucket on the ref: stays `!=`, so NULL rows are included"
+ (is (=? {:query {:filter [:!=
+ [:field (meta/id :checkins :date) {:base-type :type/Date, :temporal-unit :default}]
+ [:absolute-datetime #t "2014-05-08" :default]]}}
+ (optimize-query
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:!=
+ [:field (meta/id :checkins :date) {:base-type :type/Date, :temporal-unit :day}]
+ [:absolute-datetime #t "2014-05-08" :day]]})))))
+ (testing "bucket on the literal: becomes a negated range, which excludes NULL rows"
+ (is (=? {:query {:filter [:or
+ [:< [:field (meta/id :checkins :date) {:base-type :type/Date}]
+ [:absolute-datetime #t "2014-05-08" :default]]
+ [:>= [:field (meta/id :checkins :date) {:base-type :type/Date}]
+ [:absolute-datetime #t "2014-05-09" :default]]]}}
+ (optimize-query
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:!=
+ [:field (meta/id :checkins :date) {:base-type :type/Date}]
+ [:absolute-datetime #t "2014-05-08" :day]]})))))))
+
+(deftest ^:parallel optimize-untyped-expression-ref-not-optimized-test
+ (testing (str "`temporal-ref?` reads the column type off the ref's own options, and resolve stamps "
+ "none on an `expression` ref, so the literal-side form is left unoptimized on an "
+ "untyped ref and rewritten on a typed one. Pass 2.95 in agent-lib repair hoists the "
+ "bucket onto the ref, which is what makes the two agree again (BOT-2095).")
+ (testing "untyped ref, bucket on the literal: left alone"
+ (let [clause [:> [:expression "date"] [:absolute-datetime #t "2025-06-01" :month]]]
+ (is (= clause (optimize-filters clause)))))
+ (testing "typed ref, same clause: rewritten into a range"
+ (is (= [:>= [:expression "date" {:base-type :type/DateTime}] [:absolute-datetime #t "2025-07-01" :default]]
+ (optimize-filters [:> [:expression "date" {:base-type :type/DateTime}]
+ [:absolute-datetime #t "2025-06-01" :month]]))))
+ ;; The hoisted form Pass 2.95 emits is `<ref bucketed by month> > "2025-06-01"`, and by the time
+ ;; this middleware runs `wrap-value-literals` has turned that bare string into a `:day` literal
+ ;; WITHOUT copying the ref's unit onto it - so the units disagree and nothing is optimized. Write
+ ;; the literal as `:default` here instead and the clause *is* optimized: a different fact, pinned
+ ;; under this test's name. Nor is this half about typing - the typed ref behaves identically.
+ (testing "bucket on the ref: left alone whether or not the ref carries a type"
+ (doseq [opts [{:temporal-unit :month}
+ {:base-type :type/DateTime, :temporal-unit :month}]]
+ (let [clause [:> [:expression "date" opts] [:absolute-datetime #t "2025-06-01" :day]]]
+ (is (= clause (optimize-filters clause)) (pr-str opts)))))))
+
+(deftest ^:parallel expression-ref-hoisted-bucket-needs-an-aligned-literal-test
+ (testing (str "This middleware truncates BOTH sides of a comparison, so a literal sitting in the "
+ "middle of its bucket costs a `field` ref nothing. It never fires on an `expression` "
+ "ref, where only the column is truncated - so agent-lib repair's Pass 2.95 truncates "
+ "the literal itself when it hoists a bucket onto a ref. Without that the hoisted "
+ "clause asks a different question and answers it silently (BOT-2095).")
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ created-at (lib.metadata/field mp (mt/id :people :created_at))
+ ;; `Ship` is `created_at` shifted by zero days, so the field-ref answer is the right one
+ base (-> (lib/query mp (lib.metadata/table mp (mt/id :people)))
+ (lib/expression "Ship" (lib/datetime-add created-at 0 :day)))
+ cnt (fn [filter-clause]
+ (-> base
+ (lib/filter filter-clause)
+ (lib/aggregate (lib/count))
+ qp/process-query
+ mt/rows
+ ffirst))
+ fld (fn [unit] (lib/with-temporal-bucket (lib/ref created-at) unit))
+ expr (fn [unit] (lib/with-temporal-bucket (lib/expression-ref base "Ship") unit))]
+ (doseq [[label ground-clause unaligned-clause aligned-clause]
+ [["="
+ (lib/= (fld :month) "2018-06-15")
+ (lib/= (expr :month) "2018-06-15")
+ (lib/= (expr :month) "2018-06-01")]
+ ["<"
+ (lib/< (fld :month) "2018-06-15")
+ (lib/< (expr :month) "2018-06-15")
+ (lib/< (expr :month) "2018-06-01")]
+ [">="
+ (lib/>= (fld :month) "2018-06-15")
+ (lib/>= (expr :month) "2018-06-15")
+ (lib/>= (expr :month) "2018-06-01")]
+ ["between"
+ (lib/between (fld :month) "2018-06-15" "2018-09-20")
+ (lib/between (expr :month) "2018-06-15" "2018-09-20")
+ (lib/between (expr :month) "2018-06-01" "2018-09-01")]]]
+ (testing label
+ (let [ground (cnt ground-clause)]
+ (is (pos? ground)
+ "the field-ref clause has to match some rows, or the comparison below is vacuous")
+ (is (= ground (cnt aligned-clause))
+ "the aligned literal reproduces the field-ref answer")
+ (is (not= ground (cnt unaligned-clause))
+ "the unaligned literal does not - this is the defect Pass 2.95 aligns away"))))
+ ;; This is what licenses Pass 2.95 applying the alignment unconditionally instead of only
+ ;; when the ref is an `expression` - the middleware truncates the literal for a `field` ref
+ ;; anyway, so pre-truncating it cannot move the answer.
+ (testing "aligning the literal is a no-op for a field ref"
+ (doseq [[label aligned unaligned]
+ [["=" (lib/= (fld :month) "2018-06-01") (lib/= (fld :month) "2018-06-15")]
+ ["!=" (lib/!= (fld :month) "2018-06-01") (lib/!= (fld :month) "2018-06-15")]
+ ["<" (lib/< (fld :month) "2018-06-01") (lib/< (fld :month) "2018-06-15")]
+ ["<=" (lib/<= (fld :month) "2018-06-01") (lib/<= (fld :month) "2018-06-15")]
+ [">" (lib/> (fld :month) "2018-06-01") (lib/> (fld :month) "2018-06-15")]
+ [">=" (lib/>= (fld :month) "2018-06-01") (lib/>= (fld :month) "2018-06-15")]
+ ["quarter" (lib/= (fld :quarter) "2018-04-01") (lib/= (fld :quarter) "2018-06-15")]
+ ["between" (lib/between (fld :month) "2018-06-01" "2018-09-01")
+ (lib/between (fld :month) "2018-06-15" "2018-09-20")]]]
+ (testing label
+ (is (= (cnt unaligned) (cnt aligned))))))))))
+
(deftest ^:parallel do-not-change-unit-of-relative-datetime-to-default-test
(testing "Never change the unit of a relative datetime to :default. That would not make any sense."
(is (= {:database (meta/id)