The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Stop Metabot emitting filters legacy MBQL can't normalize (#81947)
Paolo Rascunàe85197580628
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.cljindex f4bd9025b93..b24b3cde08c 100644--- a/src/metabase/agent_lib/representations/repair.clj+++ b/src/metabase/agent_lib/representations/repair.clj@@ -34,9 +34,12 @@[metabase.lib.expression :as lib.expression][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.schema.mbql-clause :as mbql-clause]+ [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]+ [metabase.lib.walk :as lib.walk][metabase.models.serialization.resolve :as resolve][metabase.models.serialization.resolve.mp :as resolve.mp][metabase.util :as u]+ [metabase.util.date-2 :as u.date][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli.registry :as mr]@@ -311,7 +314,7 @@;;; ============================================================;;; Pass 1.87 -- rewrite known misspelled `lib/type` markers to their canonical value.;;;-;;; Pass 2's `infer-*` helpers only FILL a missing marker; they never rewrite a present one.+;;; Pass 2's `ensure-lib-types*` only FILLS a missing marker; it never rewrites a present one.;;; This pass handles the present-but-wrong case via a small alias table, exactly like the;;; `rewrite-operator-name-aliases*` / `rewrite-temporal-bucket-aliases*` passes.;;; ============================================================@@ -802,29 +805,16 @@(&recur (-> &match (assoc 3 (nth &match 4)) (assoc 4 (nth &match 3))))));;; ============================================================-;;; Pass 1.86 -- wrap bare ISO-date string bounds in `between` clauses as-;;; `[absolute-datetime, {}, <iso-str>, "day"]`.-;;;-;;; LLMs frequently write `[between, {}, <date-field>, "2024-01-01", "2024-12-31"]`,-;;; using bare strings as the bounds. lib's `:between` schema demands a temporal-;;; expression on each side once any side is temporal; bare strings won't satisfy-;;; `:type/Date`. We detect the case where at least one of the two bounds matches the-;;; ISO-8601 `yyyy-mm-dd` pattern and wrap each matching string as an-;;; `["absolute-datetime" {} <iso-str> "day"]` clause. Carried over from the sexp-;;; pipeline's `wrap-iso-date-as-absolute-datetime` (see-;;; `repr-deletion-followups.md` § 1.6). High-frequency LLM pattern.-;;;-;;; Idempotency: after wrap, bounds are vectors, so the predicate (string + ISO regex)-;;; no longer matches.+;;; Temporal-literal helpers, shared by the passes below.;;; ============================================================-(def ^:private iso-date-pattern+(def ^:private iso-date-or-datetime-pattern"Recognises an ISO-8601 calendar-date string (yyyy-mm-dd, optionally with a time portion)."#"\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+\-]\d{2}:?\d{2})?)?")(defn- iso-date-string? [v](and (string? v)- (re-matches iso-date-pattern (str/trim v))))+ (re-matches iso-date-or-datetime-pattern (str/trim v))))(defn- temporal-clause-head?"Heads that are unambiguously temporal-shaped clauses."@@ -840,37 +830,6 @@(string? (nth v 0))(temporal-clause-head? (nth v 0))))-(defn- wrap-iso-date [v]- (if (iso-date-string? v)- ["absolute-datetime" {} (str/trim v) "day"]- v))--(defn- between-needs-iso-wrap?- "Trigger the ISO-wrap when at least one bound is already a temporal-shaped clause OR- at least one bound is an ISO-date string. The first case mirrors sexp's behaviour- (`temporal-expression?` on either side), the second case is a small extension - if- both bounds are bare ISO date strings the structure is unambiguous and bare strings- would fail validation anyway."- [lo hi]- (or (temporal-shaped-clause? lo)- (temporal-shaped-clause? hi)- (iso-date-string? lo)- (iso-date-string? hi)))--(defn- between-needs-iso-wrap-clause? [node]- (and (between-clause? node)- (between-needs-iso-wrap? (nth node 3) (nth node 4))))--(defn- wrap-iso-date-bounds*- [form]- ;; No `&recur`: a wrapped ISO bound becomes an `absolute-datetime` clause, which is itself- ;; temporal-shaped, so `between-needs-iso-wrap-clause?` would fire on the result forever.- (match/replace form- (_ :guard between-needs-iso-wrap-clause?)- (-> &match- (assoc 3 (wrap-iso-date (nth &match 3)))- (assoc 4 (wrap-iso-date (nth &match 4))))))-;;; ============================================================;;; Pass 1.865 -- wrap bare `"now"` string literals in temporal contexts as the canonical;;; `["now" {}]` clause.@@ -900,7 +859,11 @@(defn- temporal-context-operand? [v](or (temporal-shaped-clause? v)- (field-with-temporal-unit? v)))+ (field-with-temporal-unit? v)+ ;; A bare ISO-date bound counts as a temporal sibling: in `["between" {} <field>+ ;; "2024-01-01" "now"]` neither operand is a clause, and without this the `"now"`+ ;; would survive as a bare string and blow up in `wrap-value-literals`.+ (iso-date-string? v)))(def ^:private temporal-comparison-heads"Comparison-style heads for which we'll wrap a bare `\"now\"` literal in another operand@@ -939,6 +902,458 @@node))form))+;;; ============================================================+;;; Structural predicates -- what a bare map looks like, with no `lib/type` marker to go on.+;;;+;;; Shared by Pass 2.95, which hoists only inside a stage, and Pass 2, which stamps a missing+;;; marker on one. One definition, so the hoist's notion of a stage is the one that stamps the+;;; marker -- a marker the model wrote can be wrong, a derivation cannot disagree with itself.+;;; ============================================================++(defn- top-level-query-map?+ [m]+ (match/matches? m {"database" _, "stages" _}))++(defn- join-like-map?+ "A map that looks like an explicit join: it carries join-only keys (`conditions`, or+ `alias`+`stages`) that never appear on a stage or top-level query. We key on these rather+ than on `\"fields\"` (which a join shares with a stage) so a join is not mistaken for a+ stage by [[stage-like-map?]]."+ [m]+ (and (map? m)+ (or (contains? m "conditions")+ (and (contains? m "alias")+ (contains? m "stages")))))++(defn- stage-like-map?+ "A map that looks like an MBQL stage: has `\"source-table\"`, `\"source-card\"`, or any of the+ stage-body keys (`filters`, `aggregation`, `breakout`, `order-by`, `fields`, `joins`,+ `expressions`, `limit`). Not a top-level query and not an explicit join (a join can carry+ `\"fields\"`, so we exclude it explicitly)."+ [m]+ (and (map? m)+ (not (top-level-query-map? m))+ (not (join-like-map? m))+ (boolean+ (some #(contains? m %)+ ["source-table" "source-card" "filters" "aggregation"+ "breakout" "order-by" "fields" "joins" "expressions" "limit"]))))++;;; ============================================================+;;; Pass 2.95 -- hoist a temporal bucket: move it off an `absolute-datetime` literal and onto the+;;; ref it is compared to, aligning the literal to the unit it gives up. Sideways, within one+;;; clause; nothing moves up a level.+;;;+;;; ["=", {}, ["field", {}, <col>], ["absolute-datetime", {}, "2025-01-01", "month"]]+;;; => ["=", {}, ["field", {"temporal-unit": "month"}, <col>], "2025-01-01"]+;;;+;;; ["during", {}, ["field", {}, <col>], "2025-06-15", "month"]+;;; => ["=", {}, ["field", {"temporal-unit": "month"}, <col>], "2025-06-01"]+;;;+;;; ["during", {}, ["field", {}, <col>], "2025-01-01T10:30:00", "hour"]+;;; => ["=", {}, ["field", {"temporal-unit": "hour"}, <col>], "2025-01-01T10:00"]+;;;+;;; ["between", {}, ["field", {}, <col>], ["absolute-datetime", {}, "2025-01-01", "day"], ["now", {}]]+;;; => ["between", {}, ["field", {}, <col>], "2025-01-01", ["now", {}]]+;;;+;;; Why: the frontend opens a Metabot question from a `/question#` hash holding *legacy* MBQL+;;; (`metabase.metabot.agent.links/->legacy-mbql`). JSON turns the unit into a string, and the legacy+;;; schema cannot decode it back for `absolute-datetime` / `during`, so the clause never survives the+;;; hop -- the last paragraph below says what that costs, which depends on where the clause sits. A+;;; `temporal-unit` on a ref decodes fine (`legacy-json-round-trip-temporal-filters-test`).+;;;+;;; Only inside a stage's `filters` and `expressions`, at every nesting level of the two -- a+;;; comparison inside a `case` included -- and in every stage of the query, a join's inner stages+;;; among them, where `optimize-temporal-filters` reads the unit from either side. Not in+;;; `aggregation:` (`count-where`, `sum-where`, `share`), `order-by:` or a join's own `conditions:`:+;;; that middleware walks `:filters` and `:expressions` and nothing else, so there the bucket really+;;; would truncate the column --+;;;+;;; ["count-where", {}, [">", {}, <col>, ["absolute-datetime", {}, "2027-06-15", "month"]]]+;;; as written: SUM(CASE WHEN created_at > date_trunc(month, ...) ...)+;;; hoisted: SUM(CASE WHEN date_trunc(month, created_at) > date_trunc(month, ...) ...)+;;;+;;; -- a different answer. Also left as written: a unit the literal cannot carry (`hour` on a date,+;;; `month-of-year`), `year` on an `expression` ref ([[unit-hoistable-onto-ref?]] says why), a ref+;;; that already has a `temporal-unit`, `between` bounds with different buckets, and an unparseable+;;; literal (resolve's `:invalid-temporal-literal` check still needs to see it). The `between`+;;; example drops the wrapper anyway: `day` on a date adds nothing, and the bound ends up as the+;;; query builder writes it.+;;;+;;; `::query` / `::stage` are `:closed false`, so a stray `filters:` on the query root or a join map+;;; sits in no stage: E7 names the clause, never the stray key, and the model's fix is dropped again.+;;;+;;; Both forms then select the same rows -- or compute the same value, inside `expressions` --+;;; wherever `optimize-temporal-filters` can read the unit off either side: a `field` ref, or an+;;; `expression` ref that carries a type. Two exceptions.+;;;+;;; (1) `!=` / `not-in` on a `:type/Date` column bucketed by `day`. Only the hoisted form reaches+;;; `date-field-with-day-bucketing?`, so the literal-side form becomes a negated range+;;; (`d < x OR d >= x+1`, dropping NULL rows) while the hoisted one stays an `!=` and gets the+;;; `IS NULL` disjunct `sql.qp` adds to every other `!=` (`correct-null-behaviour`). Hoisting is the+;;; only option here, not just the better one: every `!=` shape that survives the JSON hop compiles+;;; to `<> ... OR ... IS NULL`, and the one shape that does not is the broken one above. Pinned by+;;; `optimize-date-not-equals-null-semantics-test`. The divergence is about *which rows*; inside a+;;; `case` under `expressions` that becomes a difference in a computed value rather than in the+;;; filter.+;;;+;;; (2) An `expression` ref. The middleware optimises neither form, for two unrelated reasons: as+;;; written, resolve stamps no `base-type` on the ref and `temporal-ref?` reads the type from the+;;; ref's own options, so the clause is not optimizable at all; hoisted, `wrap-value-literals` copies+;;; a ref's unit onto the bare literal only for `field` refs (`type-info-from-col` sits behind a+;;; `clause-of-type? :field` guard), so a `month`-bucketed `expression` ref meets a `day` literal and+;;; the units disagree.+;;;+;;; [">", {}, ["expression", {}, "Ship"], ["absolute-datetime", {}, "2025-06-01", "month"]]+;;; as written: WHERE <Ship> > date_trunc(month, '2025-06-01') -- 18744 rows+;;; hoisted: WHERE date_trunc(month, <Ship>) > '2025-06-01' -- 18709 rows+;;;+;;; The hoisted count is what both forms give once the ref is typed, so the rewrite makes an untyped+;;; *temporal* `expression` ref behave like a typed one -- a real change, and the right one. `=` and+;;; `<=` diverge the same way; `<` and `>=` happen to agree. Both asymmetries are QP gaps+;;; (`temporal-ref?` should consult `lib/type-of`; `wrap-value-literals` should read a unit off any+;;; ref); worth their own issue. Pinned by `optimize-untyped-expression-ref-not-optimized-test`.+;;;+;;; That is also why the pass aligns the literal itself ([[align-literal-to-unit]]). The middleware+;;; truncates *both* sides of a comparison, so on a `field` ref a literal sitting in the middle of+;;; its bucket costs nothing. On a hoisted `expression` ref it fires only when the literal carries a+;;; time component, which `wrap-value-literals` types for it; a date-only literal is left alone, so+;;; only the column is truncated and the comparison asks a different question --+;;;+;;; ["=", {}, ["expression", {"temporal-unit": "month"}, "Ship"], "2025-06-15"]+;;; date-only: WHERE date_trunc(month, <Ship>) = '2025-06-15' -- never true+;;; "…T10:00:00" or "Z": WHERE <Ship> >= '2025-06-01' AND <Ship> < '2025-07-01'+;;;+;;; -- and the ordered heads land up to one bucket out. `during` makes that the mainline case, since+;;; it names the unit *containing* its literal. Aligning reproduces what the middleware would have+;;; done, and is a no-op on a `field` ref.+;;;+;;; The one unit alignment cannot rescue is `year`, which is both a truncation and an extraction+;;; unit: `lib/type-of` calls a `year`-bucketed untyped ref `:type/Integer`, and the QP's+;;; `auto-parse-filter-values` then fails on the date literal. [[unit-hoistable-onto-ref?]] declines+;;; that hoist on an `expression` ref, leaving the clause to Pass 6; a model writing the bucketed ref+;;; itself still reaches the QP, so the hole belongs with the two gaps above.+;;;+;;; Column types are unknown here, so a bucket can land on a text or numeric column;+;;; [[assert-temporal-buckets-on-temporal-columns!]] rejects that after resolve, on an `expression`+;;; ref as well as a `field` one. It matters more on the former: the QP drops the bucket silently on+;;; a `field` ref, but keeps it on an `expression` ref, where it compiles to nonsense on a text+;;; column and fails preprocessing outright on a numeric one. That is the same exposure `filters`+;;; already has; walking `expressions` opens a second position onto it.+;;;+;;; Runs after Pass 2.9, which creates new `filters`. Idempotent: the output matches neither+;;; predicate.+;;;+;;; None of the survivors above is merely unrewritten -- each is broken. The legacy unit enums have no+;;; `:decode/normalize`, so after the JSON hop the clause never normalizes back. What that costs+;;; depends on where it sits. In `filters`, `expressions` and join `conditions:` the dev and CI+;;; question page 400s, while a production JAR -- where `normalize-or-throw`'s `mu/defn` output check+;;; is compiled out -- lets `lib/query`'s `clean-stage-schema-errors` delete the clause, and the+;;; question silently answers something else. In `aggregation:` the legacy schema is permissive+;;; enough that even dev never throws: the `count-where` above is deleted in both, silently. Pass 6's+;;; [[unencodable-temporal-clause-error!]] catches every one and raises a retryable `:agent-error?`,+;;; except the unparseable literal, which resolve names more precisely.+;;; ============================================================++(def ^:private date-truncation-units+ "Bucketing units that mean the same thing on a ref as on a date literal."+ #{"day" "week" "month" "quarter" "year"})++(def ^:private datetime-truncation-units+ "The date units plus the sub-day ones a datetime literal supports."+ (into date-truncation-units #{"second" "minute" "hour"}))++(def ^:private bucket-hoistable-heads+ "Comparison heads whose literal operands may carry a hoistable bucket."+ #{"=" "!=" "<" "<=" ">" ">=" "in" "not-in"})++(def ^:private date-only-pattern #"\d{4}-\d{2}-\d{2}")+(def ^:private year-month-pattern #"\d{4}-\d{2}")+(def ^:private year-pattern #"\d{4}")++(defn- widen-partial-date+ "`yyyy-MM` -> `yyyy-MM-01`, `yyyy` -> `yyyy-01-01`; anything else unchanged."+ [s]+ (cond+ (re-matches year-month-pattern s) (str s "-01")+ (re-matches year-pattern s) (str s "-01-01")+ :else s))++(defn- align-literal-to-unit+ "Truncate `literal` to the start of the `unit` that contains it.++ [\"2025-06-15\" \"month\"] => \"2025-06-01\"+ [\"2025-06-15\" \"day\"] => \"2025-06-15\" ; already aligned, returned as written++ The generalisation of [[widen-partial-date]], which already does this for `yyyy-MM` / `yyyy`. On a+ `field` ref it changes no answer - the QP truncates both sides of such a comparison itself - and on+ an `expression` ref it is what keeps the hoisted clause meaning what the literal-side one did; the+ pass header says why the QP cannot do it there.++ `literal` comes back as written unless truncation both moves the instant and leaves an ISO string.+ `default` names no unit, so it is skipped."+ [literal unit]+ (if (= "default" unit)+ literal+ (let [parsed (u.date/parse literal)+ t (u.date/truncate parsed (keyword unit))+ ;; a `Z`-suffixed literal parses to a `ZonedDateTime`, which renders back as+ ;; `2025-06-01T00:00Z[UTC]` - that would reach the `/question#` hash and the LLM-facing+ ;; export verbatim, so a rewrite that is not ISO is dropped rather than emitted+ aligned (str t)]+ (if (and (not= parsed t) (iso-date-string? aligned))+ aligned+ literal))))++(defn- parseable-temporal-literal? [s]+ (try+ (some? (u.date/parse s))+ (catch Exception _+ false)))++(defn- truncation-units-for+ "The bucketing units a widened literal of this shape can carry.++ A date-only literal takes the date units; a datetime literal also takes the sub-day ones."+ [literal]+ (if (re-matches date-only-pattern literal)+ date-truncation-units+ datetime-truncation-units))++(defn- hoistable-bucket+ "Return `[literal unit]` for an `absolute-datetime` literal whose bucket the compared ref can take.++ [\"absolute-datetime\" {} \"2025-03\" \"MONTH\"] => [\"2025-03-01\" \"month\"]+ [\"absolute-datetime\" {} \"2025-06-15\" \"month\"] => [\"2025-06-01\" \"month\"]+ [\"absolute-datetime\" {} \"2025-01-01\" \"hour\"] => nil ; a date cannot carry `hour`++ Both values come back normalised: trimmed, lower-cased, `yyyy-MM` / `yyyy` widened to the first day+ they name, and the literal aligned to its unit ([[align-literal-to-unit]]) so the hoisted clause+ means what the literal-side one did. `nil` for anything that is not such a literal."+ [v]+ (when (and (vector? v)+ (= 4 (count v))+ (string? (nth v 0))+ (= "absolute-datetime" (u/lower-case-en (nth v 0)))+ (map? (nth v 1))+ ;; a named literal is only ever an expression's top-level clause, never a comparison+ ;; operand; a type hint is safe to drop+ (empty? (dissoc (nth v 1) "base-type" "effective-type"))+ (string? (nth v 2))+ (string? (nth v 3)))+ (let [literal (widen-partial-date (str/trim (nth v 2)))+ unit (u/lower-case-en (nth v 3))+ units (truncation-units-for literal)]+ (when (and (iso-date-string? literal)+ (or (= "default" unit) (contains? units unit))+ (parseable-temporal-literal? literal))+ [(align-literal-to-unit literal unit) unit]))))++(defn- during-clause? [v]+ (and (vector? v)+ (= 5 (count v))+ (= "during" (nth v 0))+ (map? (nth v 1))))++(defn- unbucketed-ref-clause?+ "A `field` or `expression` ref with no `temporal-unit`. A `nil` unit counts as none: lib drops it."+ [v]+ (and (vector? v)+ (= 3 (count v))+ (contains? #{"field" "expression"} (nth v 0))+ (map? (nth v 1))+ (nil? (get (nth v 1) "temporal-unit"))))++(defn- hoistable-comparison-node?+ "A comparison clause shaped so that a bucket could move onto its ref operand. Says nothing about+ whether the literal operands actually carry one."+ [v]+ (and (vector? v)+ (>= (count v) 4)+ (string? (nth v 0))+ (map? (nth v 1))+ (unbucketed-ref-clause? (nth v 2))+ (or (between-clause? v)+ (during-clause? v)+ (contains? bucket-hoistable-heads (nth v 0)))))++(defn- with-temporal-unit [ref-clause unit]+ (assoc-in ref-clause [1 "temporal-unit"] unit))++(defn- unit-hoistable-onto-ref?+ "Whether `unit` may move onto `ref-clause`. False only for `year` on an `expression` ref.++ `year` is both a truncation and an extraction unit, so `lib/type-of` answers `:type/Integer` for a+ `year`-bucketed ref that carries no `effective-type` - which is exactly an `expression` ref, since+ resolve stamps no type on one - and the QP's `auto-parse-filter-values` then fails outright trying+ to read the date literal as an integer. A `field` ref carries a type, so the same bucket is fine+ there. Declining leaves the literal standing for [[unencodable-temporal-clause-error!]] to turn+ into a retryable error."+ [ref-clause unit]+ (not (and (= "expression" (nth ref-clause 0))+ (= "year" unit))))++(defn- hoist-bucket-in-comparison+ "When every literal of a comparison carries the same hoistable bucket, put it on the ref.++ [\"in\" {} [\"field\" {} <col>] [\"absolute-datetime\" {} \"2025-01-01\" \"month\"]+ [\"absolute-datetime\" {} \"2025-03-01\" \"month\"]]+ => [\"in\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-01-01\" \"2025-03-01\"]"+ [[head opts ref & literals :as node]]+ (let [buckets (map hoistable-bucket literals)+ unit (second (first buckets))]+ (if (and (every? some? buckets)+ (apply = (map second buckets))+ (unit-hoistable-onto-ref? ref unit))+ (into [head opts (with-temporal-unit ref unit)] (map first buckets))+ node)))++(defn- redundant-bucket-literal+ "The bare literal of a bucketed bound whose bucket adds nothing to it: `default`, or `day` on a+ date-only literal. `nil` otherwise."+ [v]+ (when-let [[literal unit] (hoistable-bucket v)]+ (when (or (= "default" unit)+ (and (= "day" unit) (re-matches date-only-pattern literal)))+ literal)))++(defn- hoist-bucket-in-between+ "Both bounds bucketed alike: hoist the bucket onto the ref. Otherwise drop only a wrapper that adds nothing.++ [\"between\" {} <ref> [\"absolute-datetime\" {} \"2025-01-01\" \"month\"]+ [\"absolute-datetime\" {} \"2025-06-01\" \"month\"]]+ => [\"between\" {} <ref bucketed by month> \"2025-01-01\" \"2025-06-01\"]++ [\"between\" {} <ref> [\"absolute-datetime\" {} \"2025-01-01\" \"day\"] [\"now\" {}]]+ => [\"between\" {} <ref> \"2025-01-01\" [\"now\" {}]]"+ [[head opts ref lo hi]]+ (let [[lo' lo-unit] (hoistable-bucket lo)+ [hi' hi-unit] (hoistable-bucket hi)]+ (if (and lo' hi' (= lo-unit hi-unit) (unit-hoistable-onto-ref? ref lo-unit))+ [head opts (with-temporal-unit ref lo-unit) lo' hi']+ [head opts ref (or (redundant-bucket-literal lo) lo) (or (redundant-bucket-literal hi) hi)])))++(defn- hoist-bucket-in-during+ "`during` is `=` against the ref bucketed by the same unit; say it that way.++ [\"during\" {} [\"field\" {} <col>] \"2025-01-01\" \"month\"]+ => [\"=\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-01-01\"]++ [\"during\" {} [\"field\" {} <col>] \"2025-06-15\" \"month\"]+ => [\"=\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-06-01\"]++ [\"during\" {} [\"field\" {} <col>] \"2025-01-01\" \"hour\"] => unchanged++ The unit set follows the literal's shape, as it does for a comparison ([[hoistable-bucket]]):+ rewriting `hour` on a date would narrow the filter to one hour. `during` names the unit+ *containing* the literal, so a literal in the middle of its bucket is its normal input and is+ [[align-literal-to-unit]]'s mainline case - the second example above.+ [[unencodable-temporal-clause-error!]] reports what is left standing."+ [[_ opts ref literal unit :as node]]+ (let [literal (when (string? literal) (widen-partial-date (str/trim literal)))+ unit (when (string? unit) (u/lower-case-en unit))]+ (if (and literal+ (iso-date-string? literal)+ (contains? (truncation-units-for literal) unit)+ (unit-hoistable-onto-ref? ref unit)+ (parseable-temporal-literal? literal))+ ["=" opts (with-temporal-unit ref unit) (align-literal-to-unit literal unit)]+ node)))++(defn- hoist-bucket-in-clause [node]+ (cond+ (during-clause? node) (hoist-bucket-in-during node)+ (between-clause? node) (hoist-bucket-in-between node)+ :else (hoist-bucket-in-comparison node)))++(def ^:private bucket-hoist-stage-keys+ "The stage keys the QP's `optimize-temporal-filters` rewrites, and so the only ones a bucket may+ move inside."+ ["filters" "expressions"])++(defn- hoist-buckets-in-clauses+ "Rewrite every hoistable comparison at any depth of one stage key's clause vector."+ [clauses]+ (walk/postwalk (fn [n] (if (hoistable-comparison-node? n) (hoist-bucket-in-clause n) n)) clauses))++(defn- hoist-temporal-buckets*+ "Hoist each `absolute-datetime` bucket in a stage's `filters` / `expressions` onto the compared ref.++ {\"filters\" [[\"=\" {} [\"field\" {} <col>] [\"absolute-datetime\" {} \"2025-01-01\" \"month\"]]]}+ => {\"filters\" [[\"=\" {} [\"field\" {\"temporal-unit\" \"month\"} <col>] \"2025-01-01\"]]}++ Only those two keys, and only on a map [[stage-like-map?]] recognises -- the same predicate Pass 2+ uses to decide what a stage is, so the hoist's notion of a stage is the one that stamps the marker.+ The pass header says why the scope is those two keys."+ [form]+ (walk/postwalk+ (fn [node]+ (if (stage-like-map? node)+ (reduce (fn [stage k]+ (cond-> stage+ ;; belt and braces: Pass 1.5 `normalize-expressions-shape*` has already turned a+ ;; map-form `expressions:` into the sequential form, and a stage carrying neither+ ;; key falls through untouched+ (vector? (get stage k)) (update k hoist-buckets-in-clauses)))+ node+ bucket-hoist-stage-keys)+ node))+ form))++;;; ============================================================+;;; Pass 1.867 -- rewrite `temporal-extract` to the `get-*` clause that means the same thing.+;;;+;;; ["temporal-extract", {}, <col>, "month-of-year"] => ["get-month", {}, <col>]+;;; ["temporal-extract", {}, <col>, "week-of-year-iso"] => ["get-week", {}, <col>, "iso"]+;;; ["temporal-extract", {}, <col>, "day-of-week", "us"] => ["get-day-of-week", {}, <col>]+;;;+;;; Same hazard as Pass 2.95: the unit lands in the legacy `::TemporalExtractUnit` enum, which has+;;; no decoder, so the question page 400s. A `get-*` clause carries no unit. The week-mode units keep+;;; their meaning through the trailing argument `get-week` / `get-day-of-week` already take. The+;;; clause's own optional mode slot (third example) is dropped: SQL compilation reads only the unit+;;; (`:temporal-extract` in `metabase.driver.sql.query-processor`), so it never mattered.+;;;+;;; An exact synonym wherever the clause appears, so not scoped to `filters`. Idempotent: the new+;;; head is not `temporal-extract`.+;;; ============================================================++(def ^:private temporal-extract-unit->getter+ "Extraction unit -> the `get-*` head meaning exactly the same thing, followed by any trailing+ week-mode argument that head needs to preserve the unit's meaning."+ {"year-of-era" ["get-year"]+ "quarter-of-year" ["get-quarter"]+ "month-of-year" ["get-month"]+ "day-of-month" ["get-day"]+ "day-of-week" ["get-day-of-week"]+ "day-of-week-iso" ["get-day-of-week" "iso"]+ "hour-of-day" ["get-hour"]+ "minute-of-hour" ["get-minute"]+ "second-of-minute" ["get-second"]+ "week-of-year-iso" ["get-week" "iso"]+ "week-of-year-us" ["get-week" "us"]+ "week-of-year-instance" ["get-week" "instance"]})++(defn- rewrite-temporal-extract*+ "Rewrite each `temporal-extract` clause in `form` to the `get-*` clause that means the same thing.++ [\"temporal-extract\" {} <col> \"week-of-year-iso\"] => [\"get-week\" {} <col> \"iso\"]+ [\"temporal-extract\" {} <col> \"day-of-week\" \"us\"] => [\"get-day-of-week\" {} <col>]++ A unit with no `get-*` equivalent is left as written for validation to report."+ [form]+ (walk/postwalk+ (fn [node]+ (if-let [[getter & mode] (and (vector? node)+ (<= 4 (count node) 5)+ (string? (nth node 0))+ (= "temporal-extract" (u/lower-case-en (nth node 0)))+ (map? (nth node 1))+ (string? (nth node 3))+ (temporal-extract-unit->getter (u/lower-case-en (nth node 3))))]+ (into [getter (nth node 1) (nth node 2)] mode)+ node))+ form))+;;; ============================================================;;; Pass 1.55 -- normalise `fields:` on a stage / join when the LLM wrote a single clause;;; instead of a sequential of clauses.@@ -1091,37 +1506,11 @@;;; ============================================================;;; Pass 2 -- fill in missing `lib/type` markers+;;;+;;; The structural predicates `needs-lib-type-marker?` keys on live in the shared section above Pass+;;; 2.95, which gates on the same notion of a stage.;;; ============================================================-(defn- top-level-query-map?- [m]- (match/matches? m {"database" _, "stages" _}))--(defn- join-like-map?- "A map that looks like an explicit join: it carries join-only keys (`conditions`, or- `alias`+`stages`) that never appear on a stage or top-level query. We key on these rather- than on `\"fields\"` (which a join shares with a stage) so a join is not mistaken for a- stage by [[stage-like-map?]]."- [m]- (and (map? m)- (or (contains? m "conditions")- (and (contains? m "alias")- (contains? m "stages")))))--(defn- stage-like-map?- "A map that looks like an MBQL stage: has `\"source-table\"`, `\"source-card\"`, or any of the- stage-body keys (`filters`, `aggregation`, `breakout`, `order-by`, `fields`, `joins`,- `expressions`, `limit`). Not a top-level query and not an explicit join (a join can carry- `\"fields\"`, so we exclude it explicitly)."- [m]- (and (map? m)- (not (top-level-query-map? m))- (not (join-like-map? m))- (boolean- (some #(contains? m %)- ["source-table" "source-card" "filters" "aggregation"- "breakout" "order-by" "fields" "joins" "expressions" "limit"]))))-(defn- needs-lib-type-marker? [m](and (map? m)(not (contains? m "lib/type"))@@ -2426,6 +2815,82 @@:error :numeric-field-id:clause &match})))))+;;; ----- E7: a temporal clause the `/question#` hash cannot carry ---------------------++(defn- deferred-to-invalid-temporal-literal?+ "True for an `absolute-datetime` whose literal cannot parse.++ Resolve's own check names the literal (`:invalid-temporal-literal`,+ [[metabase.agent-lib.representations.resolve/validate-temporal-literals]]), which is the more+ useful complaint, so E7 stays quiet and lets that one fire. It does not inspect `during` or+ `value`, which is why only this head gets the carve-out.++ The carve-out is per *clause*, not per query: a query holding an unparseable literal alongside any+ other offender still reports that other offender, because E7 throws on the first match anywhere.+ Deferring the whole query instead would be a worse trade -- `validate-temporal-literals` does not+ inspect every position, since a bare literal on a comparison head is checked only when an operand+ is *known* temporal, so an unparseable literal in a position it does not inspect would silence E7+ with nothing to replace it, reopening the silent-deletion hole+ Pass 2.95 and this detector exist to close. The model fixes what E7 named and sees the literal+ complaint on the next turn."+ [node]+ (let [literal (nth node 2)]+ (and (string? literal)+ (not (parseable-temporal-literal? literal)))))++(defn- unencodable-temporal-clause-error!+ "Detect an `absolute-datetime`, `during` or unit-carrying `value` clause that Pass 2.95 could not+ rewrite.++ [\"count-where\", {}, [\">\", {}, <col>, [\"absolute-datetime\", {}, \"2025-01-01\", \"month\"]]]++ All three hold a temporal unit in a slot the legacy schema types with `::DateUnit` /+ `::DateTimeUnit` (`metabase.legacy-mbql.schema`, via `::ValueTypeInfo` for `value`), and neither+ enum carries a `:decode/normalize`. Once the `/question#` hash has JSON-encoded the query the unit+ is a string and the clause never normalizes back. In `filters`, `expressions` and join+ `conditions:` that is a 400 on the dev and CI question page, and in a production JAR -- where+ `normalize-or-throw`'s `mu/defn` output check is compiled out -- `lib/query`'s+ `clean-stage-schema-errors` deletes the clause outright and the question silently answers something+ else. In `aggregation:` the legacy schema is permissive enough that nothing throws even in dev: the+ clause is deleted silently in both. Pass 2.95 rewrites every shape it can rewrite without changing+ the answer, so anything still standing here is broken either way.++ `relative-datetime` and `datetime-diff` units do carry `:decode/normalize`, so they survive the hop+ and are not listed. `temporal-extract`'s `::TemporalExtractUnit` has no decoder either, but Pass+ 1.867 has already rewritten every one to a `get-*` clause by the time this runs -- narrow that pass+ and this detector needs the head. `time` is excluded on purpose: it 400s in dev, but `->mbql5`+ salvages the raw clause in prod (unit correctly keywordized, filter intact), so flagging it would+ reject a query that works for customers. This detector exists only because `::DateUnit`,+ `::TimeUnit`, `::DateTimeUnit` and `::TemporalExtractUnit` lack decoders -- if they ever gain them,+ delete it rather than inherit a false-positive machine.++ Throws `:agent-error?` ex-info on the first offender."+ [form]+ (walk/postwalk+ (fn [node]+ ;; `map-entry?`: a `{\"value\" …}` entry postwalks as a 2-element vector. The arity checks below+ ;; already exclude it; the guard matches E2 / E6 and survives a head being added at that arity.+ (when (and (vector? node)+ (not (map-entry? node))+ (string? (nth node 0 nil))+ (map? (nth node 1 nil))+ (case (u/lower-case-en (nth node 0))+ "absolute-datetime" (and (= 4 (count node))+ (not (deferred-to-invalid-temporal-literal? node)))+ "during" (= 5 (count node))+ ;; a `nil` unit counts as none, as it does on a ref -- that shape round-trips fine+ "value" (and (= 3 (count node))+ (some? (get (nth node 1) "unit")))+ false))+ (throw (ex-info+ (tru "`{0}` does not survive the round trip through the question link the frontend opens: the question either fails to load, or silently loses the clause and answers something else. Put the unit on the column instead - a `temporal-unit` option on the `field` / `expression` reference, compared to the first day of the period as a plain date string like \"2025-01-01\" - or write the range out as `>=` its first day and `<` the first day after it, both plain date strings (not `between`, which drops everything after midnight on its last day). Use the range for `year` on an `expression` reference, which the query engine reads as a number, and inside `aggregation:` (`count-where`, `sum-where`, `share`) and join `conditions:`, where moving the unit onto the column would change the answer."+ (pr-str node))+ {:agent-error? true+ :error :unencodable-temporal-clause+ :clause node})))+ node)+ form))+;;; ----- friendly-errors pipeline driver -----------------------------------------------(defn- friendly-errors*@@ -2439,7 +2904,8 @@(case-default-in-opts-error! query)(sexp-legacy-op-as-clause-error! query)(blank-expression-ref-error! query)- (numeric-field-id-error! query))+ (numeric-field-id-error! query)+ (unencodable-temporal-clause-error! query))query);;; ============================================================@@ -2464,8 +2930,8 @@rewrite-lib-type-aliases*merge-trailing-options*merge-string-filter-trailing-options*- wrap-iso-date-bounds*wrap-now-literals*+ rewrite-temporal-extract*swap-between-bounds*normalise-case-clauses*normalise-fields-shape*@@ -2489,11 +2955,16 @@1.75. strip stray surrounding double-quotes from the string segments of `field` clauses'portable-FK vector targets, e.g. `\"col\"` → `col` (cross-stage string targets are leftto the resolution-aware cross-stage matching in pass 5);+ 1.867. rewrite `temporal-extract` to the equivalent `get-*` clause, so its unit no longer sits+ where legacy normalization cannot decode it (examples in the pass header);1.87. rewrite a known-misspelled `\"lib/type\"` marker to its canonical value (e.g. thejoin slip `\"mbql.join/join\"` → `\"mbql/join\"`);1.88. merge a trailing extra options-map back into position-1 options on fixed-aritytuple clauses (e.g. `[\"time-interval\" {} <expr> -1 \"month\" {\"include-current\" true}]`);2. fill in missing `\"lib/type\"` markers on the query, joins, and stages;+ 2.95. inside a stage's `filters` and `expressions` only, hoist a temporal bucket off an+ `absolute-datetime` literal and onto the ref it is compared to, for the same reason+ (examples in the pass header; runs after Pass 2.9, which creates new `filters`);3. rewrite inline aggregation expressions in `order-by` to aggregation references whenthey match an aggregation in the same stage's `aggregation:` list (synthesising thereferenced aggregation's `lib/uuid` if needed);@@ -2560,6 +3031,7 @@rewrite-order-by-inline-aggs*resolve-aggregation-ref-indexes*split-post-agg-filters*+ hoist-temporal-buckets*(resolve-source-field-join-alias* mp content-store)(resolve-implicit-joins* mp content-store)(infer-source-card-field-types* mp content-store)@@ -2607,3 +3079,109 @@:mode mode:clause clause})))))pmbql-query)++(defn- unbucketed-ref+ "`clause` with its `temporal-unit`, and the type the bucket stamped on it, taken off.++ A *coercion's* `:effective-type` is the column's own type rather than a bucket artifact, and stays.++ [:expression {:temporal-unit :day-of-week, :effective-type :type/Integer,+ :base-type :type/DateTime} \"Ship\"]+ => [:expression {:base-type :type/DateTime} \"Ship\"]++ ;; field 105 is ISO-8601 text an admin coerced to a timestamp+ [:field {:temporal-unit :year, :effective-type :type/DateTime,+ :base-type :type/Text} 105]+ => [:field {:effective-type :type/DateTime, :base-type :type/Text} 105]++ `with-temporal-bucket` alone restores `:effective-type` from `:lib/original-effective-type`, which+ is only there on a ref lib or [[metabase.agent-lib.representations.resolve/export-query]] produced.+ On a ref resolve typed, an *extraction* unit leaves `:effective-type` describing the bucket rather+ than the column - `:type/Integer` for `day-of-week` on a datetime column - and+ [[metabase.lib.core/type-of]] reads that key first, so it has to go too. What is left resolves to+ `:base-type`, or to the expression's own type when the ref carries none."+ [clause]+ (let [{:keys [effective-type temporal-unit] :lib/keys [original-effective-type]} (nth clause 1)]+ (cond-> (lib/with-temporal-bucket clause nil)+ (and (nil? original-effective-type)+ (contains? lib.schema.temporal-bucketing/datetime-extraction-units temporal-unit)+ ;; A bucket's own type is never temporal, so a temporal stamp has to be the column's:+ ;; resolve copies `:effective-type` straight off a coerced column, where it is the+ ;; post-coercion type and `:base-type` is the storage type. Dropping it there would leave+ ;; `:type/Text` on an ISO-8601 column and make the gate reject a bucket lib itself offers.+ (not (isa? effective-type :type/Temporal)))+ (update 1 dissoc :effective-type))))++(defn- assert-bucketed-ref-is-temporal!+ "Throw when the bucketed ref `clause` names a column that is not temporal.++ Call only on a `field` / `expression` ref that carries a `temporal-unit`: the walk in+ [[assert-temporal-buckets-on-temporal-columns!]] tests that itself, before resolving a stage."+ [query stage-number clause]+ (let [temporal-unit (:temporal-unit (nth clause 1))+ bare-ref (unbucketed-ref clause)+ column-type (lib/type-of query stage-number bare-ref)]+ (when (and column-type+ (not= column-type :type/*)+ (not (isa? column-type :type/Temporal)))+ (let [column-name (lib/display-name query stage-number bare-ref)+ type-name (name column-type)+ unit-name (name temporal-unit)]+ (throw (ex-info (if (= :expression (nth clause 0))+ (tru "`{0}` is a {1} custom column, so it cannot be bucketed by {2}. Compare it to a plain value, or bucket a date / datetime column instead."+ column-name type-name unit-name)+ (tru "`{0}` is a {1} column, so it cannot be bucketed by {2}. Compare it to a plain value, or bucket a date / datetime column instead."+ column-name type-name unit-name))+ {:agent-error? true+ :error :temporal-unit-on-non-temporal-column+ :status-code 400+ :column-type column-type+ :temporal-unit temporal-unit+ :clause clause}))))))++(defn assert-temporal-buckets-on-temporal-columns!+ "Throw a retryable `:agent-error?` when a resolved `field` or `expression` ref carries a+ `temporal-unit` but its column is not temporal. Returns `pmbql-query`.++ lib accepts that shape - its unit check only knows Date / Time / DateTime columns. The QP then+ drops the bucket silently on a `field` ref; on an `expression` ref it keeps it and applies it to a+ non-temporal value, which compiles to nonsense on a text column and fails preprocessing outright+ on a numeric one. Pass 2.95 can produce either, in `filters` and in `expressions`, since it moves a+ bucket onto a ref without knowing the column's type; a model writing the bucket itself lands here+ too, and Pass 6's [[unencodable-temporal-clause-error!]] sees neither, because the hoist has+ already consumed the literal.++ Walks with [[metabase.lib.walk/walk-clauses]] rather than `clojure.walk`: an `expression` ref+ carries no type in its options, so the column type has to come from [[metabase.lib.core/type-of]],+ which needs the ref's stage. A join condition's refs do not all live in the same stage - one+ carrying `:join-alias` belongs to the join, one without it to the parent stage - so the path is+ chosen per ref. A ref whose type cannot be resolved is skipped rather than reported: this gate must+ never reject a query it merely failed to understand.++ Must run after the `::lib.schema/query` gate ([[metabase.metabot.tools.construct]]'s `_runnable`):+ a dangling expression ref makes `type-of` throw rather than resolve, and while `walk-clauses`+ rejects the query first under `mu/defn` instrumentation, that is compiled out of a production JAR."+ [pmbql-query]+ (lib.walk/walk-clauses+ pmbql-query+ (fn [query path-type path clause]+ ;; `walk-clauses` calls this on non-clause arguments too - the bare `2` in `[:= {} <ref> 2]`.+ ;; Everything a bucketed ref is not falls out here, before a stage is resolved for it: that is+ ;; the expensive half, and all but a handful of nodes skip it.+ (let [opts (when (vector? clause) (nth clause 1 nil))]+ (when (and (map? opts)+ (:temporal-unit opts)+ (contains? #{:field :expression} (nth clause 0 nil)))+ (let [stage-path (if (and (= path-type :lib.walk/join) (not (:join-alias opts)))+ (lib.walk/join-parent-stage-path path)+ path)]+ (try+ (lib.walk/apply-f-for-stage-at-path assert-bucketed-ref-is-temporal! query stage-path clause)+ ;; `Throwable`, not `ExceptionInfo`: the contract is that an unresolvable ref is skipped,+ ;; and anything narrower leaks past `execute-representations-query`'s relay as a 500+ (catch Throwable e+ ;; re-throw our own complaint; swallow lib's failure to resolve the ref+ (when (:agent-error? (ex-data e))+ (throw e)))))))+ nil))+ pmbql-query)diff --git a/src/metabase/agent_lib/representations/resolve.clj b/src/metabase/agent_lib/representations/resolve.cljindex 096d1e37c01..7db8a37f60e 100644--- a/src/metabase/agent_lib/representations/resolve.clj+++ b/src/metabase/agent_lib/representations/resolve.clj@@ -140,7 +140,7 @@;; `s` is untrusted agent input — bound it before it lands in an exception message or ex-data so a;; pathological literal can't flood logs. A real temporal literal is well under this length.(let [s' (u/truncate s 64)]- (throw (ex-info (tru "Invalid temporal literal {0} in :absolute-datetime — use an ISO-8601 date, datetime, year, or year-month."+ (throw (ex-info (tru "Invalid temporal literal {0} — use an ISO-8601 date, datetime, year, or year-month."(pr-str s')){:agent-error? true:status-code 400@@ -148,24 +148,113 @@:literal s'}e))))))-(defn- validate-absolute-datetime-literals- "Validate (without coercing) the string literals in `:absolute-datetime` clauses, so malformed- temporal input fails fast here — where the agent sees it and can correct — instead of surviving to- query execution.+(def ^:private iso-date-shaped-pattern+ "A string that looks like an ISO-8601 date, with or without a time portion."+ #"\d{4}-\d{2}-\d{2}.*")++(defn- operand-types+ "The types describing MBQL clause `x`, as a set — empty or `nil` when nothing here describes it.++ [:field {:base-type :type/Text} 43] => #{:type/DateTime} (43 is a datetime)+ [:expression {:base-type :type/DateTime,+ :effective-type :type/Integer} \"Ship\"] => #{:type/DateTime :type/Integer}++ For a resolved `[:field opts <id>]` the metadata provider is authoritative and the answer is one+ type. A type in `opts` is not evidence about the column: [[annotate-field-types]] stamps only a ref+ that has no `:base-type`, so a wrong model-authored stamp survives while the column stays whatever+ the provider says it is. Excusing a literal on that stamp buys one of two bad endings, both+ measured: a non-retryable `:type :qp` from `wrap-value-literals` wherever something upstream buckets+ the ref off the column (every `between`; `=` on a clean `yyyy-MM-dd`), and otherwise a filter that+ compiles, runs, and matches nothing for ever — `= <a timestamp column> \"2024-01-01 batch A\"`+ returns zero rows.++ Every other clause has only its own options, where `:base-type` and `:effective-type` can disagree:+ a bucket stamps an extraction type over a temporal column, a coercion stamps a temporal type over a+ text one. Returning both leaves that choice to the callers, which read the set with opposite+ quantifiers."+ [metadata-provider x]+ (when (and (vector? x) (map? (nth x 1 nil)))+ (let [id (nth x 2 nil)]+ (if (and (= :field (nth x 0)) (pos-int? id))+ ;; `pos-int?` is load-bearing, not decoration: `field` is a `mu/defn` whose input check throws+ ;; on `0` and on a negative id rather than answering `nil` the way an unknown id does.+ (when-let [field (lib.metadata.protocols/field metadata-provider id)]+ (when-let [t (or (:effective-type field) (:base-type field))]+ #{t}))+ (into #{} (keep (nth x 1)) [:effective-type :base-type])))))++(defn- operand-temporality+ "Whether MBQL clause `x` is `:temporal`, `:non-temporal`, or `:unknown` when nothing here types it.++ Three-valued on purpose: the two comparison walks below start from opposite defaults, so \"nothing+ types it\" has to stay distinguishable from \"typed, and not temporal\"."+ [metadata-provider x]+ (let [ts (operand-types metadata-provider x)]+ (cond+ ;; `some`, not `every?`: a coerced column is described by *both* its storage type and its+ ;; coercion result — `#{:type/Text :type/DateTime}` for an ISO-8601 string column — and the+ ;; temporal one is the type the QP compares a literal against.+ (some #(isa? % :type/Temporal) ts) :temporal+ (seq ts) :non-temporal+ :else :unknown)))++(def ^:private temporal-comparison-heads+ "Comparison heads whose operands can be a temporal ref next to bare temporal string literals.+ `:between` is absent: it has its own walk, with the opposite default. `during` is absent because it+ never arrives — Pass 2.95 rewrites a filter-position `during` into the `=`-on-a-bucketed-ref form+ this set does cover, and Pass 6's+ [[metabase.agent-lib.representations.repair/unencodable-temporal-clause-error!]] rejects the rest."+ #{:= :!= :< :<= :> :>= :in :not-in})++(defn- validate-temporal-literals+ "Validate (without coercing) the temporal string literals the model wrote, so malformed input fails+ fast here — where the agent sees it and can correct — instead of surviving to query execution: the+ literal of every `:absolute-datetime` clause, any bare `between` bound that looks like an ISO date,+ and any such string compared against a temporal operand by one of [[temporal-comparison-heads]] —+ the shape Pass 2.95's bucket hoist makes canonical and Pass 6's message tells the model to write.++ The two bare-literal walks start from opposite defaults, which is the point. A `between` bound is+ checked unless the compared column is *known* non-temporal, because it is checked today and a+ column we cannot type is not a reason to stop: `[\"between\" {} <STATUS> \"2024-01-01 batch A\"+ \"2024-06-01 batch Z\"]` is a legal text range and must pass, but a bound next to an untyped custom+ column must still be checked. The other heads are checked only when some operand is *known*+ temporal and none is known non-temporal, because they are not checked today and+ `[\"=\" {} <STATUS> \"2024-01-01 batch A\"]` is a perfectly good text comparison.The actual string → `java.time` coercion is intentionally NOT done here: it happens later in theQP's `wrap-value-literals`, where the comparison field's type and the report timezone areavailable. This pass only rejects literals that can't possibly parse. Returns the query unchanged."- [pmbql-query]+ [pmbql-query metadata-provider];; `match/match-many`, not `lib.walk/walk-clauses`: the latter is a `mu/defn` that validates its;; whole-query argument against `::lib.schema/query`, so under test instrumentation it would throw a;; raw "Invalid input" on an otherwise-malformed query (e.g. an `:offset` in `:expressions`) here,;; pre-empting the friendlier not-runnable gate downstream. We only need to inspect literals, and;; matching a bare `[:absolute-datetime _ s _]` vector suffices — the same shape check the sibling- ;; `annotate-field-types` pass uses.+ ;; `annotate-field-types` pass uses. Separate walks rather than one with several patterns:+ ;; `match-many` does not descend into a form it matched, so a wrapped literal inside a comparison+ ;; has to be reached by a walk that did not match the comparison.(match/match-many pmbql-query[:absolute-datetime _ (s :guard string?) _](assert-parseable-temporal-literal! s))+ (match/match-many pmbql-query+ [:between _ ref lo hi]+ (when-not (= :non-temporal (operand-temporality metadata-provider ref))+ (doseq [s [lo hi]+ :when (and (string? s) (re-matches iso-date-shaped-pattern s))]+ (assert-parseable-temporal-literal! s))))+ ;; The operand test lives in the *pattern*, not in the body: a head-only pattern matches an inert+ ;; comparison too, and `match-many` would then stop and never see a checkable comparison nested+ ;; inside it. A failing guard leaves the form unmatched, so the walk descends. `match-many` rejects+ ;; a lambda guard, and this one must close over `metadata-provider`, so it is bound to a name first.+ (let [checkable-operands? (fn [args]+ (let [ts (map #(operand-temporality metadata-provider %) args)]+ (and (some #{:temporal} ts)+ (not-any? #{:non-temporal} ts))))]+ (match/match-many pmbql-query+ [(_ :guard temporal-comparison-heads) _ & (args :guard checkable-operands?)]+ (doseq [s args+ :when (and (string? s) (re-matches iso-date-shaped-pattern s))]+ (assert-parseable-temporal-literal! s))))pmbql-query)(defn resolve-query@@ -188,7 +277,7 @@(-> (lib.normalize/normalize ::lib.schema/query with-mp)(annotate-field-types metadata-provider)annotate-metric-and-measure-ref-types- validate-absolute-datetime-literals))))+ (validate-temporal-literals metadata-provider)))));;; ============================================================;;; Export final MBQL 5 back to portable representationsdiff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex 2c80b90cf49..388e76eaea5 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -683,10 +683,13 @@6. Resolve portable FKs to numeric IDs and normalize through `lib.schema/query` against themetadata-provider.6.5. Backstop-gate the resolved query: mirror the FE `canRun` schema validation- ([[query-not-runnable-explanation]]), then run the FE expression editor's own+ ([[query-not-runnable-explanation]]), run the FE expression editor's owndiagnostics over every custom column / aggregation / filter- ([[repr.repair/assert-editor-accepts-expressions!]]). Either failure is a retryable- `:agent-error?` - success on a query the editor rejects is BOT-1442.+ ([[repr.repair/assert-editor-accepts-expressions!]]), then reject a temporal bucket+ sitting on a column that is not temporal+ ([[repr.repair/assert-temporal-buckets-on-temporal-columns!]]). Any of the three+ failing is a retryable `:agent-error?` - success on a query the editor rejects is+ BOT-1442.7. Export that final numeric MBQL 5 back to the portable form for the LLM-facing`:query-json` / `query-content` output.@@ -746,6 +749,7 @@;; after the `_runnable` gate: `diagnose-expression` itself validates its query;; argument against the same schema._editor-ok (repr.repair/assert-editor-accepts-expressions! pmbql-query)+ _buckets-ok (repr.repair/assert-temporal-buckets-on-temporal-columns! pmbql-query)exported-repr (repr.resolve/export-query mp pmbql-query permission-aware-content-store)_validated' (repr/validate-query exported-repr)query-id (u/generate-nano-id)]diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.cljindex 738098cb4bd..ffab922fe0e 100644--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj@@ -233,13 +233,15 @@(def ^:private raw-value? (complement lib/clause?))-;; Some queries carry temporal literals in their portable / wire form — e.g. Metabot's representations-;; repair wraps `between` bounds as `[:absolute-datetime {} "2024-01-01" :day]`, and on CLJS a string-;; is the only representation. The comparison arms below parse temporal strings that arrive as *raw*-;; values, but would otherwise skip ones already wrapped in `:absolute-datetime`, leaving a bare string-;; that later middleware (e.g. `optimize-temporal-filters`) chokes on. This predicate lets those arms-;; detect such clauses; the unit travels with the string so a `:year`/`:month` literal keeps its bucket-;; instead of collapsing to the field's default when re-parsed.+;; Some queries carry temporal literals in their portable / wire form — e.g. a Metabot-built query+;; whose repair layer leaves a bucketed literal as written wherever moving the bucket onto the column+;; would change the predicate, such as `[:absolute-datetime {} "2025-01-01" :month]` inside a+;; `count-where`; and on CLJS a string is the only representation. The comparison arms below parse+;; temporal strings that arrive as *raw* values, but would otherwise skip ones already wrapped in+;; `:absolute-datetime`, leaving a bare string that later middleware (e.g. `optimize-temporal-filters`)+;; chokes on. This predicate lets those arms detect such clauses; the unit travels with the string so+;; a `:year`/`:month` literal keeps its bucket instead of collapsing to the field's default when+;; re-parsed.(defn- string-valued-absolute-datetime"If `x` is an `:absolute-datetime` clause whose literal is still an (unparsed) string, return a`[string unit]` pair of that inner string and the clause's temporal unit; otherwise `nil`."diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.cljindex 5cdf571b5d2..eec516be3f8 100644--- a/test/metabase/agent_lib/representations/repair_test.clj+++ b/test/metabase/agent_lib/representations/repair_test.clj@@ -7,7 +7,8 @@[clojure.test.check.properties :as prop][metabase.agent-lib.representations :as repr][metabase.agent-lib.representations.repair :as repair]- [metabase.lib.test-util :as lib.tu]))+ [metabase.lib.test-util :as lib.tu]+ [metabase.util.date-2 :as u.date]))(set! *warn-on-reflection* true)@@ -22,6 +23,15 @@(lib.tu/mock-metadata-provider{:database {:id 1 :name "Sample"}}))+(def ^:private mp-temporal+ "ORDERS with a datetime column, for the hoist test whose prefix stage has to resolve."+ (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "Sample"}+ :tables [{:id 10 :name "ORDERS" :schema "PUBLIC" :db-id 1}]+ :fields [{:id 100 :name "ID" :table-id 10 :base-type :type/Integer}+ {:id 102 :name "USER_ID" :table-id 10 :base-type :type/Integer}+ {:id 103 :name "CREATED_AT" :table-id 10 :base-type :type/DateTime}]}))+(def ^:private mp-fks"3-table MP: ORDERS → PRODUCTS, ORDERS → USERS."(lib.tu/mock-metadata-provider@@ -662,78 +672,501 @@(is (= ["not" {} inner] (repair/repair trivial-mp input))))));;; ============================================================-;;; Pass 1.86 - wrap bare ISO-date strings as absolute-datetime in between+;;; Bare ISO-date `between` bounds;;; ============================================================-(deftest ^:parallel wrap-iso-date-bounds-both-bare-test- (testing "both bounds are bare yyyy-mm-dd strings: both get wrapped"+(deftest ^:parallel iso-date-bounds-left-bare-test+ (testing "both bounds bare yyyy-mm-dd: left exactly as the model wrote them"(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]input ["between" {} field "2024-01-01" "2024-12-31"]]- (is (= ["between" {}- field- ["absolute-datetime" {} "2024-01-01" "day"]- ["absolute-datetime" {} "2024-12-31" "day"]]- (repair/repair trivial-mp input))))))+ (is (= input (repair/repair trivial-mp input))))))-(deftest ^:parallel wrap-iso-date-bounds-mixed-test- (testing "one bound already absolute-datetime, the other a bare string: bare side- gets wrapped"- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]- wrapped ["absolute-datetime" {} "2024-01-01" "day"]- input ["between" {} field wrapped "2024-12-31"]]- (is (= ["between" {}- field- wrapped- ["absolute-datetime" {} "2024-12-31" "day"]]- (repair/repair trivial-mp input)))))- (testing "one bound a relative-datetime, the other a bare ISO string: bare side gets- wrapped (presence of temporal sibling triggers the rule)"+(deftest ^:parallel iso-datetime-bounds-keep-their-time-test+ (testing "bounds carrying a time portion keep it"+ (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]+ input ["between" {} field "2024-06-15T09:00:00" "2024-06-15T18:00:00"]]+ (is (= input (repair/repair trivial-mp input))))))++(deftest ^:parallel iso-date-bound-beside-temporal-clause-test+ (testing "a temporal sibling does not pull a bare bound into a wrapper"(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]rel ["relative-datetime" {} -7 "day"]input ["between" {} field rel "2024-12-31"]]- (is (= ["between" {} field rel- ["absolute-datetime" {} "2024-12-31" "day"]]- (repair/repair trivial-mp input))))))+ (is (= input (repair/repair trivial-mp input))))))-(deftest ^:parallel wrap-iso-date-bounds-numeric-untouched-test- (testing "both bounds numeric: not wrapped (between also works for numbers)"- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "TOTAL"]]- input ["between" {} field 10 100]]+(deftest ^:parallel non-temporal-between-bounds-untouched-test+ (testing "numeric and non-ISO string bounds are untouched"+ (doseq [input [["between" {} ["field" {} ["Sample" "PUBLIC" "ORDERS" "TOTAL"]] 10 100]+ ["between" {} ["field" {} ["Sample" "PUBLIC" "X"]] "hello" "world"]]](is (= input (repair/repair trivial-mp input))))))-(deftest ^:parallel wrap-iso-date-bounds-with-time-portion-test- (testing "yyyy-mm-ddThh:mm:ss bounds also get wrapped"- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]- input ["between" {} field "2024-06-15T09:00:00" "2024-06-15T18:00:00"]]- (is (= ["between" {}- field- ["absolute-datetime" {} "2024-06-15T09:00:00" "day"]- ["absolute-datetime" {} "2024-06-15T18:00:00" "day"]]- (repair/repair trivial-mp input))))))+(deftest ^:parallel iso-date-bound-is-a-temporal-sibling-test+ (testing "a bare ISO date counts as a temporal sibling, so bare 'now' is canonicalised"+ (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]]+ (is (= ["between" {} field "2024-01-01" ["now" {}]]+ (repair/repair trivial-mp ["between" {} field "2024-01-01" "now"])))+ (is (= ["between" {} field ["now" {}] "2024-12-31"]+ (repair/repair trivial-mp ["between" {} field "now" "2024-12-31"]))))))-(deftest ^:parallel wrap-iso-date-bounds-idempotent-test- (testing "wrapping is idempotent: two wraps == one wrap"- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]- input ["between" {} field "2024-01-01" "2024-12-31"]- once (repair/repair trivial-mp input)- twice (repair/repair trivial-mp once)]- (is (= once twice)))))+;;; ============================================================+;;; Pass 2.95 - hoist a temporal bucket from an absolute-datetime literal onto the ref+;;; ============================================================-(deftest ^:parallel wrap-iso-date-bounds-then-swap-test- (testing "wrap + swap compose: out-of-order bare ISO strings wrap then swap"- (let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]- input ["between" {} field "2024-12-31" "2024-01-01"]]- (is (= ["between" {}- field- ["absolute-datetime" {} "2024-01-01" "day"]- ["absolute-datetime" {} "2024-12-31" "day"]]- (repair/repair trivial-mp input))))))+(def ^:private created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]])++(defn- created-at-bucketed [unit]+ ["field" {"temporal-unit" unit} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]])++(defn- abs-dt [literal unit]+ ["absolute-datetime" {} literal unit])++(defn- week-start+ "`literal` truncated to the start of its week - what the hoist aligns a `week`-bucketed literal to.++ Computed rather than written out: `u.date/truncate` reads the site `start-of-week` setting, other+ namespaces mutate it, and the tests below are `^:parallel`. Under the default Sunday `2025-01-01`+ gives `2024-12-29`."+ [literal]+ (str (u.date/truncate (u.date/parse literal) :week)))++(defn- repair-in-stage+ "Run `repair` on a query carrying `stage-kvs`, returning the repaired first stage. The hoist pass+ is scoped to a stage's `filters` and `expressions`, so its inputs have to be given in one of those+ positions."+ [stage-kvs]+ (-> (repair/repair trivial-mp+ {"lib/type" "mbql/query"+ "stages" [(merge {"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}+ stage-kvs)]})+ (get "stages")+ first))++(defn- repair-filter [filter-clause]+ (first (get (repair-in-stage {"filters" [filter-clause]}) "filters")))++(defn- hoist-filter+ "Run only Pass 2.95 over a stage carrying `filter-clause`, returning the rewritten clause.++ Unlike [[repair-filter]] this skips Pass 6, so it can show what the pass itself declines - Pass 6's+ `unencodable-temporal-clause-error!` throws on most of those shapes."+ [filter-clause]+ (-> (#'repair/hoist-temporal-buckets*+ {"lib/type" "mbql/query"+ "stages" [{"lib/type" "mbql.stage/mbql" "filters" [filter-clause]}]})+ (get-in ["stages" 0 "filters"])+ first))++(deftest ^:parallel hoist-temporal-bucket-comparison-test+ (testing "a bucketed absolute-datetime literal moves its unit onto the field ref"+ (doseq [op ["=" "!=" "<" "<=" ">" ">="]]+ (is (= [op {} (created-at-bucketed "month") "2025-01-01"]+ (repair-filter [op {} created-at (abs-dt "2025-01-01" "month")]))+ op))))++(deftest ^:parallel hoist-temporal-bucket-units-test+ (testing "every date unit moves onto the field ref, and the literal aligns to the unit it gave up"+ ;; `week` is the only date unit `2025-01-01` is not already aligned to - it is a Wednesday+ (doseq [[unit literal] {"default" "2025-01-01"+ "day" "2025-01-01"+ "week" (week-start "2025-01-01")+ "month" "2025-01-01"+ "quarter" "2025-01-01"+ "year" "2025-01-01"}]+ (is (= ["=" {} (created-at-bucketed unit) literal]+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01" unit)]))+ unit)))+ (testing "a datetime literal also takes the sub-day units"+ (doseq [[unit literal] {"default" "2025-01-01T10:30:00"+ "second" "2025-01-01T10:30:00"+ "minute" "2025-01-01T10:30:00"+ "hour" "2025-01-01T10:00"+ "day" "2025-01-01T00:00"+ "month" "2025-01-01T00:00"}]+ (is (= ["=" {} (created-at-bucketed unit) literal]+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01T10:30:00" unit)]))+ unit))))++(deftest ^:parallel hoist-temporal-bucket-partial-date-test+ (testing "`yyyy-MM` and `yyyy` literals are widened to the first day they name"+ (is (= ["=" {} (created-at-bucketed "month") "2025-03-01"]+ (repair-filter ["=" {} created-at (abs-dt "2025-03" "month")])))+ (is (= ["=" {} (created-at-bucketed "year") "2025-01-01"]+ (repair-filter ["=" {} created-at (abs-dt "2025" "year")])))))++(deftest ^:parallel hoist-temporal-bucket-multi-value-test+ (testing "variadic `=` / `!=` and `in` / `not-in`: every literal carries the same bucket"+ (doseq [op ["=" "!=" "in" "not-in"]]+ (is (= [op {} (created-at-bucketed "month") "2025-01-01" "2025-03-01"]+ (repair-filter [op {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-03-01" "month")]))+ op)))+ (testing "literals bucketed by different units are left alone"+ (let [input ["in" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-03-01" "year")]]+ (is (= input (hoist-filter input)))))+ (testing "a mix of bucketed and bare literals is left alone"+ (let [input ["in" {} created-at (abs-dt "2025-01-01" "month") "2025-03-01"]]+ (is (= input (hoist-filter input))))))++(deftest ^:parallel hoist-temporal-bucket-expression-ref-test+ (testing "a custom-column ref takes the bucket the same way a field ref does"+ (is (= ["=" {} ["expression" {"temporal-unit" "month"} "Ship Date"] "2025-01-01"]+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-01-01" "month")])))))++;; The hoisted clause truncates the column, so the literal has to be truncated too or the clause+;; asks a different question - `date_trunc(month, col) = "2025-06-15"` is never true, and the+;; ordered heads land up to one bucket out. `optimize-temporal-filters` does this for a `field` ref+;; and never for an `expression` ref, so the pass does it itself. These are shape assertions; the+;; row-level proof that alignment reproduces the field-ref answer lives next to that middleware, in+;; `expression-ref-hoisted-bucket-needs-an-aligned-literal-test`.+(deftest ^:parallel hoist-temporal-bucket-aligns-literal-test+ (testing "the hoisted literal is truncated to the unit it gave up"+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-01"]+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15" "month")])))+ (is (= ["=" {} (created-at-bucketed "quarter") "2025-04-01"]+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15" "quarter")])))+ (is (= ["=" {} ["expression" {"temporal-unit" "month"} "Ship Date"] "2025-06-01"]+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-15" "month")]))))+ (testing "`during` names the unit containing its literal, so this is its mainline case"+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-01"]+ (repair-filter ["during" {} created-at "2025-06-15" "month"]))))+ (testing "both `between` bounds align"+ (is (= ["between" {} (created-at-bucketed "month") "2025-06-01" "2025-09-01"]+ (repair-filter ["between" {} created-at (abs-dt "2025-06-15" "month") (abs-dt "2025-09-20" "month")]))))+ (testing "so does every literal of a variadic comparison"+ (is (= ["in" {} (created-at-bucketed "month") "2025-06-01" "2025-09-01"]+ (repair-filter ["in" {} created-at (abs-dt "2025-06-15" "month") (abs-dt "2025-09-20" "month")]))))+ (testing "a literal already aligned comes back as written, not reformatted"+ (doseq [[literal unit] [["2025-06-01" "month"] ["2025-06-15" "day"]+ ["2025-01-01T10:30:00" "minute"] ["2025-01-01T10:30:00" "second"]]]+ (is (= ["=" {} (created-at-bucketed unit) literal]+ (repair-filter ["=" {} created-at (abs-dt literal unit)]))+ (str literal " " unit))))+ (testing "`default` names no unit, so its literal is never truncated"+ (is (= ["=" {} (created-at-bucketed "default") "2025-06-15"]+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15" "default")]))))+ (testing "a `Z`-suffixed literal truncates to `2025-06-01T00:00Z[UTC]`, which is not ISO, so it is kept as written"+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-15T10:00:00Z"]+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15T10:00:00Z" "month")]))))+ (testing "an offset literal does truncate to an ISO string, so it aligns"+ (is (= ["=" {} (created-at-bucketed "month") "2025-06-01T00:00+02:00"]+ (repair-filter ["=" {} created-at (abs-dt "2025-06-15T10:00:00+02:00" "month")])))))++(deftest ^:parallel hoist-temporal-bucket-year-on-expression-ref-test+ (testing "`year` is left on the literal when the ref is an `expression`"+ ;; `year` is both a truncation and an extraction unit, so `lib/type-of` calls a `year`-bucketed+ ;; ref with no type `:type/Integer` - which is exactly an `expression` ref - and the QP fails+ ;; trying to read the date literal as an integer. Declining leaves it to Pass 6.+ (doseq [input [["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-01" "year")]+ ["between" {} ["expression" {} "Ship Date"] (abs-dt "2025-01-01" "year") (abs-dt "2026-01-01" "year")]+ ["during" {} ["expression" {} "Ship Date"] "2025-06-01" "year"]]]+ (is (= input (hoist-filter input)) (pr-str input))))+ (testing "a `field` ref carries a type, so the same bucket still moves onto it"+ (is (= ["=" {} (created-at-bucketed "year") "2025-01-01"]+ (repair-filter ["=" {} created-at (abs-dt "2025-06-01" "year")])))+ (is (= ["=" {} (created-at-bucketed "year") "2025-01-01"]+ (repair-filter ["during" {} created-at "2025-06-01" "year"]))))+ (testing "every other unit still moves onto an `expression` ref"+ (doseq [[unit literal] {"day" "2025-06-15"+ "week" (week-start "2025-06-15")+ "month" "2025-06-01"+ "quarter" "2025-04-01"}]+ (is (= ["=" {} ["expression" {"temporal-unit" unit} "Ship Date"] literal]+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-15" unit)]))+ unit))))++(deftest ^:parallel hoist-temporal-bucket-between-test+ (testing "both bounds bucketed by the same unit: hoisted, bounds become plain strings"+ (is (= ["between" {} (created-at-bucketed "month") "2025-01-01" "2025-06-01"]+ (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "month")]))))+ (testing "hoisting composes with the swap pass for out-of-order bounds"+ (is (= ["between" {} (created-at-bucketed "month") "2025-01-01" "2025-06-01"]+ (repair-filter ["between" {} created-at (abs-dt "2025-06-01" "month") (abs-dt "2025-01-01" "month")]))))+ (testing "bounds bucketed by different units are left alone"+ (let [input ["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "year")]]+ (is (= input (hoist-filter input)))))+ (testing "one bound bucketed by a unit that changes it (`month`): left alone"+ (let [input ["between" {} created-at (abs-dt "2025-01-01" "month") "2025-06-01"]]+ (is (= input (hoist-filter input)))))+ (testing "one bound bucketed by `day` or `default`: the wrapper goes and the bound stays a plain date"+ (is (= ["between" {} created-at "2025-01-01" ["now" {}]]+ (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "day") "now"])))+ (is (= ["between" {} created-at "2025-01-01" "2025-06-30"]+ (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "default") "2025-06-30"])))))++(deftest ^:parallel hoist-temporal-bucket-during-test+ (testing "`during` becomes `=` against the bucketed field - what the clause already means"+ (doseq [[unit literal] {"day" "2025-01-01"+ "week" (week-start "2025-01-01")+ "month" "2025-01-01"+ "quarter" "2025-01-01"+ "year" "2025-01-01"}]+ (is (= ["=" {} (created-at-bucketed unit) literal]+ (repair-filter ["during" {} created-at "2025-01-01" unit]))+ unit)))+ (testing "a datetime literal with a sub-day unit is rewritten the same way"+ (is (= ["=" {} (created-at-bucketed "hour") "2025-01-01T10:00"]+ (repair-filter ["during" {} created-at "2025-01-01T10:30:00" "hour"]))))+ (testing "a sub-day unit on a date-only literal is left alone, like the comparison arm"+ (doseq [[literal unit] [["2025-01-01" "hour"] ["2025-01-01" "minute"] ["2025-01-01" "second"]+ ["2025-03" "hour"] ["2025" "hour"]]]+ (let [input ["during" {} created-at literal unit]]+ (is (= input (hoist-filter input)) (str literal " " unit)))))+ (testing "the `default` unit is left alone: `during` does not accept it, so Pass 6 reports it"+ (let [input ["during" {} created-at "2025-01-01" "default"]]+ (is (= input (hoist-filter input)))))+ (testing "a field that already carries a temporal-unit is never clobbered"+ (let [input ["during" {} (created-at-bucketed "day") "2025-01-01" "month"]]+ (is (= input (hoist-filter input))))))++(deftest ^:parallel hoist-temporal-bucket-only-comparison-heads-test+ (testing "a non-comparison clause of the same arity is not a comparison and is left alone"+ (doseq [head ["coalesce" "datetime-add" "concat"]]+ (let [input [head {} created-at (abs-dt "2025-01-01" "month")]]+ (is (= input (hoist-filter input)) head)))))++(deftest ^:parallel hoist-temporal-bucket-scope-test+ (testing "a stage's filters and expressions are rewritten; the same comparison elsewhere is not"+ ;; Pass 2.95 alone, not `repair`: Pass 6 rejects the shapes this pass declines.+ (let [cmp [">" {} created-at (abs-dt "2025-01-01" "month")]+ cmp' [">" {} (created-at-bucketed "month") "2025-01-01"]+ agg ["count-where" {} cmp]+ expr ["case" {"lib/expression-name" "E"} [[cmp 1]] 0]+ ob ["asc" {} ["sum-where" {} ["field" {} ["Sample" "PUBLIC" "ORDERS" "TOTAL"]] cmp]]+ brk ["case" {} [[cmp 1]] 0]+ fld ["case" {} [[cmp 2]] 0]+ stage (-> (#'repair/hoist-temporal-buckets*+ {"lib/type" "mbql/query"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "filters" [cmp]+ "aggregation" [agg]+ "expressions" [expr]+ "order-by" [ob]+ "breakout" [brk]+ "fields" [fld]}]})+ (get "stages")+ first)]+ (is (= [cmp'] (get stage "filters")))+ (is (= [agg] (get stage "aggregation")))+ (is (= [["case" {"lib/expression-name" "E"} [[cmp' 1]] 0]] (get stage "expressions")))+ (is (= [ob] (get stage "order-by")))+ (is (= [brk] (get stage "breakout")))+ (is (= [fld] (get stage "fields"))))))++(deftest ^:parallel hoist-temporal-bucket-join-scope-test+ ;; Pass 2.95 alone, not `repair`: Pass 6 rejects the conditions shape.+ (let [cmp [">" {} created-at (abs-dt "2025-01-01" "month")]+ cmp' [">" {} (created-at-bucketed "month") "2025-01-01"]+ stage (-> (#'repair/hoist-temporal-buckets*+ {"lib/type" "mbql/query"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "joins" [{"lib/type" "mbql/join"+ "alias" "J"+ "conditions" [cmp]+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "filters" [cmp]}]}]}]})+ (get "stages")+ first)]+ (testing "a join's own `conditions:` are not a stage key the optimizer rewrites: left as written"+ (is (= [cmp] (get-in stage ["joins" 0 "conditions"]))))+ (testing "a join's inner stage is a stage, and the optimizer covers it: hoisted"+ (is (= [cmp'] (get-in stage ["joins" 0 "stages" 0 "filters"]))))))++(deftest ^:parallel hoist-temporal-bucket-stage-marker-drift-test+ ;; The gate is `stage-like-map?` and not a `lib/type` match on purpose: Pass 2 only fills a+ ;; *missing* marker, so a stage the model mis-marked would drop out of a marker gate entirely.+ (testing "a stage the model mis-marked is still recognised structurally, so the marker complaint+ reaches the model instead of a misleading temporal one"+ (doseq [marker ["mbql/stage" "mbql.stage/mbql5"]]+ (let [stage (-> (#'repair/hoist-temporal-buckets*+ {"lib/type" "mbql/query"+ "stages" [{"lib/type" marker+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "filters" [[">" {} created-at (abs-dt "2025-01-01" "month")]]}]})+ (get "stages")+ first)]+ (is (= [[">" {} (created-at-bucketed "month") "2025-01-01"]] (get stage "filters")) marker)))))++(deftest ^:parallel hoist-temporal-bucket-nested-in-filter-test+ (testing "hoisting reaches comparisons nested under boolean combinators"+ (is (= ["and" {} ["=" {} (created-at-bucketed "month") "2025-01-01"]+ ["not" {} ["=" {} (created-at-bucketed "year") "2024-01-01"]]]+ (repair-filter ["and" {}+ ["=" {} created-at (abs-dt "2025-01-01" "month")]+ ["not" {} ["=" {} created-at (abs-dt "2024-01-01" "year")]]])))))++(deftest ^:parallel hoist-temporal-bucket-inside-case-test+ (testing "a comparison nested in a `case` inside a filter is hoisted - the QP optimizer walks+ filters to the same depth, so the two forms compile alike"+ (is (= [">" {} ["case" {} [[[">" {} (created-at-bucketed "month") "2025-01-01"] 1]] 0] 0]+ (repair-filter [">" {} ["case" {} [[[">" {} created-at (abs-dt "2025-01-01" "month")] 1]] 0] 0])))))++(deftest ^:parallel hoist-temporal-bucket-cross-stage-ref-test+ (testing "a cross-stage string-named ref is bucketed the same way as a portable-FK ref"+ (is (= ["=" {} ["field" {"temporal-unit" "month"} "CREATED_AT"] "2025-01-01"]+ (repair-filter ["=" {} ["field" {} "CREATED_AT"] (abs-dt "2025-01-01" "month")])))))++(deftest ^:parallel hoist-temporal-bucket-after-post-agg-split-test+ (testing "a post-aggregation filter that Pass 2.9 moves into a new stage is hoisted there"+ (let [stages (-> (repair/repair mp-temporal+ {"lib/type" "mbql/query"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "aggregation" [["max" {} created-at]]+ "breakout" [["field" {} ["Sample" "PUBLIC" "ORDERS" "USER_ID"]]]+ "filters" [["<" {} ["aggregation" {} 0]+ (abs-dt "2025-01-01" "month")]]}]})+ (get "stages"))+ [_ _ ref literal] (first (get (second stages) "filters"))]+ (is (= 2 (count stages)))+ (is (= "month" (get-in ref [1 "temporal-unit"])))+ (is (= "2025-01-01" literal)))))++(deftest ^:parallel hoist-temporal-bucket-guards-test+ (testing "a sub-day unit on a date-only literal is not valid MBQL 5: the pass declines it, Pass 6 reports it"+ (doseq [unit ["hour" "minute" "second" "millisecond"]]+ (let [input ["=" {} created-at (abs-dt "2025-01-01" unit)]]+ (is (= input (hoist-filter input)) unit))))+ (testing "extraction units are a different operation on a field ref: the pass declines them, Pass 6 reports them"+ (let [input ["=" {} created-at (abs-dt "2025-01-01" "month-of-year")]]+ (is (= input (hoist-filter input)))))+ ;; the one shape Pass 6 deliberately stays quiet about, so `repair-filter` (the full pipeline) is+ ;; what this block has to exercise+ (testing "a literal that does not parse is left alone, so the resolve-time check still reports it"+ (let [input ["=" {} created-at (abs-dt "2024-13-45" "month")]]+ (is (= input (repair-filter input)))))+ (testing "a field that already carries a temporal-unit is never clobbered"+ (let [input ["=" {} (created-at-bucketed "day") (abs-dt "2025-01-01" "month")]]+ (is (= input (hoist-filter input)))))+ (testing "a non-empty options map is never rewritten (it would annihilate an expression name; Pass 6 reports the clause)"+ (let [input ["=" {} created-at ["absolute-datetime" {"lib/expression-name" "Cutoff"} "2025-01-01" "day"]]]+ (is (= input (hoist-filter input))))))++(deftest ^:parallel hoist-temporal-bucket-normalisation-test+ (testing "the unit is matched case-insensitively, as lib normalises it"+ (is (= ["=" {} (created-at-bucketed "default") "2025-01-01"]+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01" "DEFAULT")]))))+ (testing "a `nil` temporal-unit counts as none - lib drops it"+ (is (= ["=" {} (created-at-bucketed "month") "2025-01-01"]+ (repair-filter ["=" {} ["field" {"temporal-unit" nil} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]+ (abs-dt "2025-01-01" "month")]))))+ (testing "a type hint on the literal does not block the rewrite"+ (is (= ["=" {} (created-at-bucketed "month") "2025-01-01"]+ (repair-filter ["=" {} created-at ["absolute-datetime" {"base-type" "type/Date"} "2025-01-01" "month"]])))))++(deftest ^:parallel hoist-temporal-bucket-idempotent-test+ (testing "hoisting is idempotent"+ (let [once (repair-filter ["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "month")])]+ (is (= once (repair-filter once))))))++(deftest ^:parallel unencodable-temporal-clause-error-test+ (testing "a temporal clause the hoist pass could not move becomes a retryable agent error"+ (doseq [[label stage-kvs]+ {"in `aggregation:`"+ {"aggregation" [["count-where" {} [">" {} created-at (abs-dt "2025-01-01" "month")]]]}+ "with the literal on the left"+ {"filters" [["=" {} (abs-dt "2025-01-01" "month") created-at]]}+ "compared to an already-bucketed ref"+ {"filters" [["=" {} (created-at-bucketed "day") (abs-dt "2025-01-01" "month")]]}+ "with an extraction unit"+ {"filters" [["=" {} created-at (abs-dt "2025-01-01" "month-of-year")]]}+ "`between` bounds bucketed by different units"+ {"filters" [["between" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-06-01" "year")]]}+ "`during` against an already-bucketed ref"+ {"filters" [["during" {} (created-at-bucketed "day") "2025-01-01" "month"]]}+ "`during` with a sub-day unit on a date-only literal"+ {"filters" [["during" {} created-at "2025-01-01" "hour"]]}+ ;; `millisecond` is a legal datetime bucket, but not one the QP turns into a range, so Pass+ ;; 2.95 declines it even on a datetime literal+ "`millisecond` on a datetime literal"+ {"filters" [["=" {} created-at (abs-dt "2025-01-01T10:00:00.123" "millisecond")]]}+ "`during` with `millisecond` on a datetime literal"+ {"filters" [["during" {} created-at "2025-01-01T10:00:00.123" "millisecond"]]}+ ;; a `value` clause keeps its raw string keys and still passes validation (BOT-2095)+ "a `value` clause carrying a unit"+ {"filters" [["=" {} created-at ["value" {"base-type" "type/DateTime" "unit" "day"} "2025-01-01"]]]}+ ;; E7 still reaches `expressions:` - Pass 2.95 declines an extraction unit everywhere+ "with an extraction unit in a `case` under `expressions:`"+ {"expressions" [["case" {"lib/expression-name" "E"}+ [[[">" {} created-at (abs-dt "2025-01-01" "month-of-year")] 1]] 0]]}}]+ (testing label+ (try+ (repair-in-stage stage-kvs)+ (is false "expected throw")+ (catch clojure.lang.ExceptionInfo e+ (let [d (ex-data e)]+ (is (true? (:agent-error? d)))+ (is (= :unencodable-temporal-clause (:error d)))))))))+ (testing "shapes that round-trip intact are never flagged"+ (is (= ["=" {} (created-at-bucketed "month") "2025-01-01"]+ (repair-filter ["=" {} created-at (abs-dt "2025-01-01" "month")])))+ (testing "a `value` clause with no unit, or a nil unit, survives the hop"+ (doseq [opts [{"base-type" "type/DateTime"}+ {"base-type" "type/DateTime" "unit" nil}]]+ (let [input ["=" {} created-at ["value" opts "2025-01-01"]]]+ (is (= input (repair-filter input)) (pr-str opts)))))+ (testing "a hoistable bucket in a `case` under `expressions:` is rewritten, not flagged"+ (is (= [["case" {"lib/expression-name" "E"}+ [[[">" {} (created-at-bucketed "month") "2025-01-01"] 1]] 0]]+ (get (repair-in-stage+ {"expressions" [["case" {"lib/expression-name" "E"}+ [[[">" {} created-at (abs-dt "2025-01-01" "month")] 1]] 0]]})+ "expressions")))))+ (testing "an unparseable literal is left for resolve's `:invalid-temporal-literal` to name"+ (let [input ["=" {} created-at (abs-dt "2024-13-45" "month")]]+ (is (= input (repair-filter input))))))++(deftest ^:parallel unencodable-temporal-clause-message-test+ (testing "the message steers to a half-open range, and names `year` on an `expression` ref as needing it"+ ;; `year` is also an extraction unit, so on an untyped `expression` ref the QP reads the bucketed+ ;; column as an integer and fails on the date literal; and `between` compiles to SQL BETWEEN wherever+ ;; `optimize-temporal-filters` does not run - every `expression` ref, all of `aggregation:` - so a+ ;; plain upper date keeps only midnight of the last day+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"`>=` its first day and `<` the first day after it.*not `between`.*`year` on an `expression` reference"+ (repair-filter ["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-06-01" "year")])))))-(deftest ^:parallel wrap-iso-date-bounds-non-iso-untouched-test- (testing "random strings (not yyyy-mm-dd shaped) are left alone"- (let [field ["field" {} ["Sample" "PUBLIC" "X"]]- input ["between" {} field "hello" "world"]]- (is (= input (repair/repair trivial-mp input))))))+;;; ============================================================+;;; Pass 1.867 - rewrite temporal-extract to its get-* equivalent+;;; ============================================================++(deftest ^:parallel rewrite-temporal-extract-test+ (testing "each extraction unit becomes its get-* clause, week modes passed on"+ (doseq [[unit expected] {"year-of-era" ["get-year" {} created-at]+ "quarter-of-year" ["get-quarter" {} created-at]+ "month-of-year" ["get-month" {} created-at]+ "day-of-month" ["get-day" {} created-at]+ "day-of-week" ["get-day-of-week" {} created-at]+ "day-of-week-iso" ["get-day-of-week" {} created-at "iso"]+ "hour-of-day" ["get-hour" {} created-at]+ "minute-of-hour" ["get-minute" {} created-at]+ "second-of-minute" ["get-second" {} created-at]+ "week-of-year-iso" ["get-week" {} created-at "iso"]+ "week-of-year-us" ["get-week" {} created-at "us"]+ "week-of-year-instance" ["get-week" {} created-at "instance"]}]+ (is (= ["=" {} expected 2]+ (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at unit] 2]))+ unit))))++(deftest ^:parallel rewrite-temporal-extract-preserves-options-test+ (testing "the clause's options survive the rewrite"+ (is (= ["get-month" {"lib/expression-name" "Mo"} created-at]+ (repair/repair trivial-mp ["temporal-extract" {"lib/expression-name" "Mo"} created-at "month-of-year"])))))++(deftest ^:parallel rewrite-temporal-extract-idempotent-test+ (testing "rewriting is idempotent"+ (let [once (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at "day-of-week"] 2])]+ (is (= once (repair/repair trivial-mp once))))))++(deftest ^:parallel rewrite-temporal-extract-week-mode-slot-test+ (testing "the optional week-mode slot is dropped: the unit alone decides, as in SQL compilation"+ (is (= ["=" {} ["get-day-of-week" {} created-at "iso"] 2]+ (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at "day-of-week-iso" "us"] 2])))+ (is (= ["=" {} ["get-month" {} created-at] 2]+ (repair/repair trivial-mp ["=" {} ["temporal-extract" {} created-at "month-of-year" "iso"] 2])))));;; ============================================================;;; Pass 1.865 - wrap bare "now" literals in temporal-comparison contexts@@ -771,11 +1204,8 @@(let [field-with-unit ["field" {"temporal-unit" "day"}["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]input ["between" {} field-with-unit "2024-01-01" "now"]]- ;; ISO date wrapped (Pass 1.86), bare 'now' wrapped (Pass 1.865).- (is (= ["between" {}- field-with-unit- ["absolute-datetime" {} "2024-01-01" "day"]- ["now" {}]]+ ;; the ISO bound stays exactly as written; only the bare 'now' is canonicalised+ (is (= ["between" {} field-with-unit "2024-01-01" ["now" {}]](repair/repair trivial-mp input)))))(testing "between(<numeric-field>, 'now', 'now'): no temporal sibling, leave alone";; The expression's column is non-temporal and bare 'now' is the only thing in sight;@@ -810,47 +1240,37 @@(is (= once twice)))))(deftest ^:parallel swap-between-bounds-iso-string-test- ;; Note: bare ISO strings are wrapped by Pass 1.86 ("wrap-iso-date-bounds*") *before*- ;; this swap pass runs, so the canonical post-repair form has \"[absolute-datetime ...]\"- ;; bounds. The swap still proceeds because \"swap-between-bounds*\" knows how to extract- ;; the inner ISO string for comparison.- (testing "ISO-8601 date strings ordered chronologically: wrapped + swapped"+ ;; Bare ISO strings now reach this pass unchanged; `swap-between-bounds*` compares them+ ;; directly via its `string?` arm (ISO-8601 lex order is chronological order).+ (testing "ISO-8601 date strings out of order: swapped, still bare"(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]input ["between" {} field "2024-12-31" "2024-01-01"]]- (is (= ["between" {}- field- ["absolute-datetime" {} "2024-01-01" "day"]- ["absolute-datetime" {} "2024-12-31" "day"]]+ (is (= ["between" {} field "2024-01-01" "2024-12-31"](repair/repair trivial-mp input)))))- (testing "ISO-8601 datetime strings: wrapped + swapped"+ (testing "ISO-8601 datetime strings out of order: swapped, time portion preserved"(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]input ["between" {} field "2024-06-15T18:00:00" "2024-06-15T09:00:00"]]- (is (= ["between" {}- field- ["absolute-datetime" {} "2024-06-15T09:00:00" "day"]- ["absolute-datetime" {} "2024-06-15T18:00:00" "day"]]+ (is (= ["between" {} field "2024-06-15T09:00:00" "2024-06-15T18:00:00"](repair/repair trivial-mp input)))))- (testing "in-order ISO strings: wrapped, no swap"+ (testing "in-order ISO strings: untouched"(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]input ["between" {} field "2024-01-01" "2024-12-31"]]- (is (= ["between" {}- field- ["absolute-datetime" {} "2024-01-01" "day"]- ["absolute-datetime" {} "2024-12-31" "day"]]- (repair/repair trivial-mp input))))))+ (is (= input (repair/repair trivial-mp input))))))(deftest ^:parallel swap-between-bounds-absolute-datetime-test+ ;; A bare clause outside a stage, so Pass 2.95 does not apply and the bounds stay+ ;; wrapped - the subject here is `between-bound-comparable`'s absolute-datetime arm, not hoisting.(testing "out-of-order absolute-datetime clauses: swap by inner ISO string"(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]- hi ["absolute-datetime" {} "2024-12-31" "day"]- lo ["absolute-datetime" {} "2024-01-01" "day"]+ hi ["absolute-datetime" {} "2024-12-31T10:00:00" "hour"]+ lo ["absolute-datetime" {} "2024-01-01T10:00:00" "hour"]input ["between" {} field hi lo]](is (= ["between" {} field lo hi](repair/repair trivial-mp input)))))(testing "in-order absolute-datetime clauses are left alone"(let [field ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]- lo ["absolute-datetime" {} "2024-01-01" "day"]- hi ["absolute-datetime" {} "2024-12-31" "day"]+ lo ["absolute-datetime" {} "2024-01-01T10:00:00" "hour"]+ hi ["absolute-datetime" {} "2024-12-31T10:00:00" "hour"]input ["between" {} field lo hi]](is (= input (repair/repair trivial-mp input))))))@@ -1411,11 +1831,33 @@;; deliberately wrap values in a vector to trigger Pass 1.82(gen/fmap (fn [[f vs]] ["in" {} f vs])(gen/tuple gen-field-clause (gen/vector gen/string-alphanumeric 1 4)))- ;; between with maybe-out-of-order ISO date strings (Pass 1.86 + 1.87)+ ;; between with maybe-out-of-order ISO date strings (Pass 1.87 swap)(gen/fmap (fn [[f a b]] ["between" {} f a b])(gen/tuple gen-field-clause-with-temporal(gen/elements ["2024-01-01" "2024-06-30" "2024-12-31"])(gen/elements ["2023-01-01" "2024-12-31" "now"])))+ ;; comparison against a bucketed absolute-datetime literal, hoistable and not (Pass 2.95)+ (gen/fmap (fn [[op f d unit]] [op {} f ["absolute-datetime" {} d unit]])+ (gen/tuple (gen/elements ["=" "!=" "<" "<=" ">" ">="])+ gen-field-clause+ (gen/elements ["2024-01-01" "2024-06-30" "2024-01-01T10:30:00"])+ (gen/elements ["default" "day" "month" "year" "hour" "month-of-year"])))+ ;; between with absolute-datetime bounds whose units may disagree (Pass 2.95)+ (gen/fmap (fn [[f a b ua ub]]+ ["between" {} f ["absolute-datetime" {} a ua] ["absolute-datetime" {} b ub]])+ (gen/tuple gen-field-clause+ (gen/elements ["2024-01-01" "2024-06-30"])+ (gen/elements ["2024-12-31" "2023-01-01"])+ (gen/elements ["day" "month" "year"])+ (gen/elements ["day" "month" "year"])))+ ;; temporal-extract, including the week-mode and unmapped units (Pass 1.867)+ (gen/fmap (fn [[f unit mode n]]+ ["=" {} (cond-> ["temporal-extract" {} f unit] mode (conj mode)) n])+ (gen/tuple gen-field-clause+ (gen/elements ["day-of-week" "month-of-year" "week-of-year-iso"+ "day-of-week-iso" "day-of-year"])+ (gen/elements [nil "iso" "us"])+ gen/small-integer));; post-aggregation filter that triggers Pass 2.9 split(gen/fmap (fn [n] [">" {} ["aggregation" {} 0] n])gen/small-integer)]))@@ -2555,6 +2997,9 @@;;; ============================================================;;; Pass 6 - friendly error messages (E1..E6)+;;;+;;; E7 (`unencodable-temporal-clause-error!`) is tested with Pass 2.95 above, in+;;; `unencodable-temporal-clause-error-test` - it only fires on what that pass declines.;;; ============================================================;;; ----- E1: [field, ...] in aggregation block ---------------------------------------diff --git a/test/metabase/agent_lib/representations/resolve_test.clj b/test/metabase/agent_lib/representations/resolve_test.cljindex 29b9398bc7b..e93c37dc8cc 100644--- a/test/metabase/agent_lib/representations/resolve_test.clj+++ b/test/metabase/agent_lib/representations/resolve_test.clj@@ -185,6 +185,116 @@(is (re-find #"Invalid temporal literal" (ex-message ex)))(is (true? (:agent-error? (ex-data ex)))))))+;;; ============================================================+;;; bare temporal literals: validation follows the compared column's type+;;; ============================================================++(def ^:private created-at ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"])+(def ^:private total ["Sample" "PUBLIC" "ORDERS" "TOTAL"])+(def ^:private category ["Sample" "PUBLIC" "PRODUCTS" "CATEGORY"])++(def ^:private coerced-mp+ "[[mock-metadata]] plus `TS_TEXT`, an ISO-8601 text column an admin coerced to a datetime — the+ shape in which `:base-type` and `:effective-type` legitimately disagree."+ (lib.tu/mock-metadata-provider+ (update mock-metadata :fields conj+ {:id 112 :name "TS_TEXT" :table-id 10 :base-type :type/Text+ :effective-type :type/DateTime :coercion-strategy :Coercion/ISO8601->DateTime})))++(defn- resolve-stage+ "Resolve a single-stage query over `table` carrying `stage-keys`, returning the resolved stage."+ [table stage-keys]+ (-> (repr.resolve/resolve-query mp {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [(assoc stage-keys+ "lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" table])]})+ (get-in [:stages 0])))++(defn- resolve-error+ "The `:error` [[resolve-stage]] throws for `stage-keys`, or `nil` when the stage resolves cleanly."+ [table stage-keys]+ (try (resolve-stage table stage-keys) nil+ (catch clojure.lang.ExceptionInfo e (:error (ex-data e)))))++(deftest validate-temporal-literals-leaves-a-text-between-alone-test+ (testing "date-shaped bounds on a text column are a legal range, not malformed dates"+ (let [clause (-> (resolve-stage "PRODUCTS"+ {"filters" [["between" {} ["field" {} category]+ "2024-01-01 batch A" "2024-06-01 batch Z"]]})+ (get-in [:filters 0]))]+ (is (= :between (nth clause 0)))+ (is (= ["2024-01-01 batch A" "2024-06-01 batch Z"] (subvec clause 3))))))++(deftest validate-temporal-literals-rejects-a-bad-between-bound-test+ (testing "a bound compared to a temporal column is still checked"+ (is (= :invalid-temporal-literal+ (resolve-error "ORDERS" {"filters" [["between" {} ["field" {} created-at]+ "2024-13-45" "2024-12-31"]]})))))++(deftest validate-temporal-literals-checks-a-between-on-an-untyped-ref-test+ (testing "an `expression` ref carries no type here, which is no reason to stop checking its bounds"+ (is (= :invalid-temporal-literal+ (resolve-error "ORDERS"+ {"expressions" [["datetime-add" {"lib/expression-name" "Shifted"}+ ["field" {} created-at] 3 "day"]]+ "filters" [["between" {} ["expression" {} "Shifted"]+ "2024-13-45" "2024-12-31"]]})))))++(deftest validate-temporal-literals-ignores-a-model-authored-base-type-test+ (testing "the metadata provider is authoritative — a wrong stamp does not excuse a bad literal"+ ;; CREATED_AT is :type/DateTime in the provider, and the QP resolves it from there too;+ ;; `annotate-field-types` leaves a ref that already carries `:base-type` alone.+ (is (= :invalid-temporal-literal+ (resolve-error "ORDERS" {"filters" [["between" {} ["field" {"base-type" "type/Text"} created-at]+ "2024-13-45" "2024-12-31"]]})))))++(deftest validate-temporal-literals-rejects-a-bad-literal-on-a-temporal-comparison-test+ (testing "a bare literal compared to a temporal ref is checked, whatever the comparison head"+ (doseq [clause [["=" {} ["field" {"temporal-unit" "month"} created-at] "2024-13-45"]+ [">=" {} ["field" {} created-at] "2024-13-45"]+ ["in" {} ["field" {} created-at] "2024-13-45" "2024-02-01"]]]+ (testing (pr-str clause)+ (is (= :invalid-temporal-literal (resolve-error "ORDERS" {"filters" [clause]})))))))++(deftest validate-temporal-literals-skips-a-known-non-temporal-comparison-test+ (testing "a date-shaped string compared to a text column is a text comparison"+ (is (nil? (resolve-error "PRODUCTS" {"filters" [["=" {} ["field" {} category]+ "2024-01-01 batch A"]]}))))+ (testing "a temporal operand elsewhere in the clause does not make this literal a date"+ ;; `=` takes any number of operands with no pairwise type check, so a temporal one can sit+ ;; beside a text one. Rejecting here would be the very defect this pass was fixed for.+ (is (nil? (resolve-error "PRODUCTS" {"filters" [["=" {} ["field" {} category]+ ["field" {} created-at]+ "2024-01-01 batch A"]]})))))++(deftest validate-temporal-literals-descends-past-an-inert-comparison-test+ (testing "a comparison that is not itself checkable must not swallow a nested one that is"+ ;; the operand test has to live in the *pattern*: with a head-only pattern this outer `=` matches,+ ;; `match-many` stops descending, and the nested comparison's literal is never seen.+ (is (= :invalid-temporal-literal+ (resolve-error "ORDERS"+ {"filters" [["=" {} ["field" {} total]+ ["case" {} [[[">=" {} ["field" {} created-at] "2024-13-45"] 1]]+ 0]]]})))))++(deftest operand-temporality-types-a-coerced-column-test+ (testing "a `year` bucket does not make a coerced column look like text — the provider is read"+ (is (= :temporal+ (#'repr.resolve/operand-temporality+ coerced-mp+ [:field {:temporal-unit :year :base-type :type/Text :effective-type :type/DateTime} 112]))))+ (testing "a cross-stage ref carries the storage type and the coercion result, and is temporal"+ ;; the pair a coerced column resolves to is not a contradiction: the coercion result is the+ ;; type the QP compares a literal against, so one temporal member is enough.+ (is (= :temporal+ (#'repr.resolve/operand-temporality+ coerced-mp+ [:field {:base-type :type/Text :effective-type :type/DateTime} "TS_TEXT"]))))+ (testing "an operand nothing here can type stays distinguishable from one typed as not temporal"+ (is (= :unknown (#'repr.resolve/operand-temporality coerced-mp [:expression {} "Shifted"])))+ (is (= :non-temporal (#'repr.resolve/operand-temporality coerced-mp [:field {} 111])))))+;;; ============================================================;;; fields (projection);;; ============================================================diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.cljindex a36379f7bcf..fd16929d1a0 100644--- a/test/metabase/metabot/tools/construct_representations_test.clj+++ b/test/metabase/metabot/tools/construct_representations_test.clj@@ -19,6 +19,7 @@[metabase.lib.test-util :as lib.tu][metabase.mcp.v2.message :as message][metabase.mcp.v2.recovery-hints :as v2-hints]+ [metabase.metabot.agent.links :as links][metabase.metabot.db :as metabot.db][metabase.metabot.tools.construct :as construct][metabase.metabot.tools.entity-details :as entity-details]@@ -26,7 +27,8 @@[metabase.models.interface :as mi][metabase.models.serialization.resolve :as serdes.resolve][metabase.models.serialization.resolve.mp :as resolve.mp]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.util.json :as json]))(set! *warn-on-reflection* true)@@ -96,16 +98,21 @@(f)))(def ^:private mp-coerced- "ORDERS with a UNIX-seconds column coerced to a timestamp: `base-type :type/Integer`,- `effective-type :type/DateTime`. Exercises the editor gate's unsuppressed expression- type check against coerced columns (the #41122 false-positive class that- `*suppress-expression-type-check?*` exists for)."+ "ORDERS with two columns an admin coerced to a timestamp - UNIX seconds and ISO-8601 text - each+ paired with a plain column of the same `base-type`, so a test can vary the coercion alone.++ Exercises the editor gate's unsuppressed expression type check against coerced columns (the+ #41122 false-positive class that `*suppress-expression-type-check?*` exists for), and the+ temporal-bucket gate, whose whole job is telling a coerced column from a non-temporal one."(lib.tu/mock-metadata-provider{:database {:id 1 :name "Sample"}:tables [{:id 10 :name "ORDERS" :schema "PUBLIC" :db-id 1}]:fields [{:id 100 :name "ID" :table-id 10 :base-type :type/Integer}{:id 104 :name "CREATED_AT_UNIX" :table-id 10 :base-type :type/Integer- :effective-type :type/DateTime :coercion-strategy :Coercion/UNIXSeconds->DateTime}]}))+ :effective-type :type/DateTime :coercion-strategy :Coercion/UNIXSeconds->DateTime}+ {:id 105 :name "STATUS" :table-id 10 :base-type :type/Text}+ {:id 106 :name "CREATED_AT_ISO" :table-id 10 :base-type :type/Text+ :effective-type :type/DateTime :coercion-strategy :Coercion/ISO8601->DateTime}]}))(defn- with-coerced-mp-and-stubs! [f](with-redefs [lib-be/application-database-metadata-provider (fn [_db-id] mp-coerced)@@ -114,6 +121,23 @@api/query-check allow-read-check](f)))+(def ^:private mp-temporal+ "ORDERS with a plain `:type/DateTime` column, for the temporal-clause round-trip tests."+ (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "Sample"}+ :tables [{:id 10 :name "ORDERS" :schema "PUBLIC" :db-id 1}]+ :fields [{:id 100 :name "ID" :table-id 10 :base-type :type/Integer}+ {:id 102 :name "USER_ID" :table-id 10 :base-type :type/Integer}+ {:id 103 :name "CREATED_AT" :table-id 10 :base-type :type/DateTime}+ {:id 104 :name "STATUS" :table-id 10 :base-type :type/Text}]}))++(defn- with-temporal-mp-and-stubs! [f]+ (with-redefs [lib-be/application-database-metadata-provider (fn [_db-id] mp-temporal)+ construct/resolve-database-id-from-first-stage (fn [_] 1)+ api/read-check allow-read-check+ api/query-check allow-read-check]+ (f)))+(defn- query-data"Build the keyword-keyed external-query map the tool accepts. Tests author the query as astring-keyed map (the form documented in earlier YAML-era tests); we walk-keywordize so the@@ -1694,3 +1718,332 @@"source-table" ["Sample" "PUBLIC" "ORDERS"]"aggregation" [["count" {}]]}]})))))(is (false? @seen)))))++(defn- legacy-json-wire+ "Run the tool on a single-stage ORDERS query carrying `stage-kvs`, returning `[before wire]`.++ `before` is the MBQL 5 query the tool built; `wire` is that query converted to legacy the way+ `links` does for the `/question#` hash and JSON round-tripped."+ [stage-kvs]+ (let [result (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [(merge {"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}+ stage-kvs)]}))+ before (get-in result [:structured-output :query])]+ [before (-> before links/->legacy-mbql json/encode (json/decode true))]))++(defn- stage-clause-counts+ "Per-stage counts of the clause vectors on a query, for comparing a query with its round-trip."+ [query]+ (mapv (fn [stage]+ (into (sorted-map)+ (for [k [:filters :aggregation :expressions :breakout :order-by :fields :joins]+ :when (contains? stage k)]+ [k (count (get stage k))])))+ (:stages query)))++(defn- assert-survives-json-hop! [[before wire]]+ ;; `->legacy-mbql` returns the MBQL 5 query unchanged if conversion throws, and a+ ;; JSON-round-tripped MBQL 5 query passes `lib/query` - so without this the+ ;; assertion below could hold while proving nothing.+ (is (= "query" (:type wire))+ "->legacy-mbql fell back to MBQL 5 instead of converting")+ ;; ... and `lib/query` does not throw on a wire whose clause `clean-stage-schema-errors` deleted,+ ;; so "no throw" is not survival. Compare the clauses - by count, since uuids and idents are+ ;; regenerated across the hop and deep equality is not available. A clause swapped for a different+ ;; clause of the same arity would slip through; deletion, the failure mode here, would not.+ (let [after (lib/query mp-temporal wire)]+ (is (map? after) "did not survive the JSON round-trip")+ (is (= (stage-clause-counts before) (stage-clause-counts after))+ "a clause was silently dropped by the round-trip")))++;; A Metabot-built question reaches the frontend as legacy MBQL inside a base64 `/question#` hash,+;; so it makes a JSON hop that turns every keyword into a string. Legacy normalization has no+;; `:decode/normalize` on the temporal-unit enums used by `:absolute-datetime`, `:during` or+;; `:temporal-extract`, so a query carrying one came back un-normalized and the question page 400'd+;; on `/api/dataset/query_metadata`. The JSON hop is what makes this reproduce - converting to+;; legacy alone does not, which is why the plain legacy round-trip gates above never caught it+;; (BOT-2095).+(deftest legacy-json-round-trip-temporal-filters-test+ (with-temporal-mp-and-stubs!+ (fn []+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]+ abs-dt (fn [literal unit] ["absolute-datetime" {} literal unit])]+ (doseq [[label filter-clause]+ {"bare date range"+ ["between" {} created-at "2025-01-01" "2025-12-31"]+ "absolute-datetime literal carrying a bucket"+ ["=" {} created-at (abs-dt "2025-01-01" "month")]+ "absolute-datetime literal with the default unit"+ ["=" {} created-at (abs-dt "2025-01-01" "default")]+ "absolute-datetime datetime literal with the default unit"+ [">" {} created-at (abs-dt "2025-01-01T10:30:00" "default")]+ "absolute-datetime datetime literal bucketed by hour"+ ["=" {} created-at (abs-dt "2025-01-01T10:00:00" "hour")]+ "absolute-datetime year-month literal"+ ["=" {} created-at (abs-dt "2025-01" "month")]+ "several bucketed literals in one `in`"+ ["in" {} created-at (abs-dt "2025-01-01" "month") (abs-dt "2025-03-01" "month")]+ "between with one redundant bucket and `now`"+ ["between" {} created-at (abs-dt "2025-01-01" "day") "now"]+ "during"+ ["during" {} created-at "2025-01-01" "month"]+ "during with a sub-day unit"+ ["during" {} created-at "2025-01-01T10:00:00" "hour"]+ "temporal-extract"+ ["=" {} ["temporal-extract" {} created-at "day-of-week"] 2]+ "temporal-extract carrying a week mode"+ ["=" {} ["temporal-extract" {} created-at "week-of-year-iso"] 2]+ "temporal-extract with the optional week-mode slot"+ ["=" {} ["temporal-extract" {} created-at "day-of-week" "iso"] 2]}]+ (testing label+ (assert-survives-json-hop! (legacy-json-wire {"filters" [filter-clause]+ "aggregation" [["count" {}]]}))))+ (testing "a bucketed literal compared to a custom column"+ (assert-survives-json-hop!+ (legacy-json-wire {"expressions" [["datetime-add" {"lib/expression-name" "Ship Date"} created-at 3 "day"]]+ "filters" [["=" {} ["expression" {} "Ship Date"] (abs-dt "2025-01-01" "month")]]+ "aggregation" [["count" {}]]})))+ (testing "a bucketed literal inside a `case` under `expressions`"+ (assert-survives-json-hop!+ (legacy-json-wire {"expressions" [["case" {"lib/expression-name" "Jan"}+ [[["=" {} created-at (abs-dt "2025-01-01" "month")] 1]] 0]]+ "aggregation" [["count" {}]]})))+ (testing "a post-aggregation filter, which repair moves into a second stage"+ (assert-survives-json-hop!+ (legacy-json-wire {"aggregation" [["max" {} created-at]]+ "breakout" [["field" {} ["Sample" "PUBLIC" "ORDERS" "USER_ID"]]]+ "filters" [["<" {} ["aggregation" {} 0] (abs-dt "2025-01-01" "month")]]})))))))++;; A bucketed temporal clause that Pass 2.95 cannot hoist does not survive the `/question#` hash the+;; frontend opens: 400 in dev, and in production `lib/query`'s cleaner deletes the clause and the+;; question silently answers something else. Pass 6 turns each into a retryable agent error (BOT-2095).+(deftest unencodable-temporal-clause-reaches-the-agent-test+ (with-temporal-mp-and-stubs!+ (fn []+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]+ abs-dt (fn [literal unit] ["absolute-datetime" {} literal unit])]+ (doseq [[label stage-kvs]+ {"in `count-where`"+ {"aggregation" [["count-where" {} [">" {} created-at (abs-dt "2025-01-01" "month")]]]}+ "in a join condition"+ {"joins" [{"lib/type" "mbql/join"+ "alias" "O2"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}]+ "conditions" [[">" {} created-at (abs-dt "2025-01-01" "month")]]}]+ "aggregation" [["count" {}]]}+ "a `value` clause carrying a unit"+ {"filters" [["=" {} created-at+ ["value" {"base-type" "type/DateTime" "unit" "day"} "2025-01-01"]]]+ "aggregation" [["count" {}]]}}]+ (testing label+ (try+ (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [(merge {"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}+ stage-kvs)]}))+ (is false "expected throw")+ (catch clojure.lang.ExceptionInfo e+ (let [d (ex-data e)]+ (is (true? (:agent-error? d)))+ (is (= :unencodable-temporal-clause (:error d))))))))))))++;; Repair moves a bucket onto the ref without knowing the column's type; lib accepts a bucket on a+;; text column, and the QP would then drop it silently on a `field` ref and keep it on an+;; `expression` ref, where it compiles to nonsense. The gate after resolve turns either into an+;; agent-facing error (the pre-hoist shape used to be rejected by the expression editor gate).+(deftest temporal-bucket-on-non-temporal-column-test+ (with-temporal-mp-and-stubs!+ (fn []+ (let [status ["field" {} ["Sample" "PUBLIC" "ORDERS" "STATUS"]]+ abs-dt (fn [literal unit] ["absolute-datetime" {} literal unit])+ label-expr ["concat" {"lib/expression-name" "Label"} status "x"]]+ (doseq [[label stage-kvs]+ {"hoisted from a bucketed literal"+ {"filters" [["<" {} status (abs-dt "2025-01-01" "month")]]+ "aggregation" [["count" {}]]}+ "written by the model"+ {"filters" [["=" {} ["field" {"temporal-unit" "month"} ["Sample" "PUBLIC" "ORDERS" "STATUS"]] "2025-01-01"]]+ "aggregation" [["count" {}]]}+ "hoisted onto a text custom column"+ {"expressions" [label-expr]+ "filters" [["<" {} ["expression" {} "Label"] (abs-dt "2025-01-01" "month")]]+ "aggregation" [["count" {}]]}+ "written by the model onto a text custom column"+ {"expressions" [label-expr]+ "filters" [["=" {} ["expression" {"temporal-unit" "month"} "Label"] "2025-01-01"]]+ "aggregation" [["count" {}]]}+ "hoisted onto a text custom column inside an `expressions` definition"+ {"expressions" [label-expr+ ["case" {"lib/expression-name" "Cohort"}+ [[["<" {} ["expression" {} "Label"] (abs-dt "2025-01-01" "month")] "a"]] "b"]]+ "aggregation" [["count" {}]]}+ ;; an extraction unit stamps the *bucket's* type on `effective-type`, so the gate+ ;; has to look past it to the column's own type - here still Text+ "written by the model with the bucket's own type stamped on the ref"+ {"expressions" [label-expr]+ "breakout" [["expression" {"temporal-unit" "month-of-year"+ "effective-type" "type/Integer"} "Label"]]+ "aggregation" [["count" {}]]}+ ;; the join condition's left-hand ref belongs to the *parent* stage, so the gate has+ ;; to type it there - the join's own stage has no `Label`+ "written by the model onto a text custom column in a join condition"+ {"expressions" [label-expr]+ "joins" [{"lib/type" "mbql/join"+ "alias" "O2"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}]+ "conditions" [["=" {} ["expression" {"temporal-unit" "month"} "Label"]+ ["field" {"join-alias" "O2"} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]]]}]+ "aggregation" [["count" {}]]}}]+ (testing label+ (try+ (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [(merge {"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}+ stage-kvs)]}))+ (is false "expected throw")+ (catch clojure.lang.ExceptionInfo e+ (let [d (ex-data e)]+ (is (true? (:agent-error? d)))+ (is (= :temporal-unit-on-non-temporal-column (:error d))))))))+ (testing "the message names a custom column as such"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"`Label` is a Text custom column"+ (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "expressions" [label-expr]+ "filters" [["<" {} ["expression" {} "Label"] (abs-dt "2025-01-01" "month")]]+ "aggregation" [["count" {}]]}]}))))+ (is (thrown-with-msg?+ ;; Text, not the `Integer` the extraction bucket stamped on `effective-type`+ clojure.lang.ExceptionInfo #"`Label` is a Text custom column"+ (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "expressions" [label-expr]+ "breakout" [["expression" {"temporal-unit" "month-of-year"+ "effective-type" "type/Integer"} "Label"]]+ "aggregation" [["count" {}]]}]}))))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"`Status` is a Text column"+ (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "filters" [["<" {} status (abs-dt "2025-01-01" "month")]]+ "aggregation" [["count" {}]]}]}))))))+ ;; The gate is only useful if it leaves working queries alone. Typing an `expression` ref needs+ ;; the ref's stage, and a join condition mixes two of them: this query runs today, and a gate+ ;; that resolved every join ref in the join's own stage would reject it with `No expression+ ;; named "Ship"`.+ (testing "a join condition bucketing a temporal custom column from the parent stage is accepted"+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]+ result (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "expressions" [["datetime-add" {"lib/expression-name" "Ship"} created-at 3 "day"]]+ "joins" [{"lib/type" "mbql/join"+ "alias" "O2"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}]+ "conditions" [["=" {}+ ["expression" {"temporal-unit" "month"} "Ship"]+ ["field" {"join-alias" "O2" "temporal-unit" "month"}+ ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]]]}]+ "aggregation" [["count" {}]]}]}))]+ (is (some? (get-in result [:structured-output :query])))))+ ;; The mirror of the Text case above: an extraction unit stamps `:type/Integer` on+ ;; `effective-type` while `base-type` keeps the column's own type. This query runs and answers+ ;; correctly, so reading the stamped type would reject a working query.+ (testing "a `day-of-week` bucket on a temporal custom column carrying the bucket's type is accepted"+ (let [created-at ["field" {} ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]+ result (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "expressions" [["datetime-add" {"lib/expression-name" "Ship"} created-at 3 "day"]]+ "breakout" [["expression" {"temporal-unit" "day-of-week"+ "effective-type" "type/Integer"+ "base-type" "type/DateTime"} "Ship"]]+ "aggregation" [["count" {}]]}]}))]+ (is (some? (get-in result [:structured-output :query])))))+ ;; `with-temporal-bucket` records the pre-bucket type in `:lib/original-effective-type` and+ ;; stamps the bucketed one on `:effective-type`, so an extraction unit makes a datetime column+ ;; look like an integer. The gate strips the bucket before asking for the type; reading the+ ;; ref's own options instead would reject this legitimate query builder output.+ (testing "a `month-of-year` bucket on a field ref the query builder produced is accepted"+ (let [q0 (lib/query mp-temporal (lib.metadata/table mp-temporal 10))+ col (first (filter #(= 103 (:id %)) (lib/filterable-columns q0)))+ ref (lib/with-temporal-bucket (lib/ref col) :month-of-year)+ q (lib/filter q0 (lib/= ref 6))]+ (is (= [:type/Integer :type/DateTime]+ ((juxt :effective-type :lib/original-effective-type) (second ref))))+ (is (= q (repr.repair/assert-temporal-buckets-on-temporal-columns! q))))))))++;; A coerced column's `:effective-type` is its own post-coercion type, not the artifact of an+;; extraction bucket, so the gate has to keep it: strip it and an ISO-8601 timestamp column reads+;; back as `:type/Text`. Pass 2.95 funnels every way of asking "filter this column by year" into the+;; bucketed-ref shape below, so rejecting it left the model with no phrasing that worked - and advice+;; ("bucket a date / datetime column instead") that pointed back at the same column.+(deftest temporal-bucket-on-coerced-column-accepted-test+ (with-coerced-mp-and-stubs!+ (fn []+ (doseq [column ["CREATED_AT_UNIX" "CREATED_AT_ISO"]+ :let [bare ["field" {} ["Sample" "PUBLIC" "ORDERS" column]]+ bucketed ["field" {"temporal-unit" "year"} ["Sample" "PUBLIC" "ORDERS" column]]]+ [shape filters]+ {"a bucketed literal" [["=" {} bare ["absolute-datetime" {} "2024-01-01" "year"]]]+ "a `during`" [["during" {} bare "2024-01-01" "year"]]+ "the bucket on the ref" [["=" {} bucketed "2024-01-01"]]+ "a bucketed `between`" [["between" {} bucketed "2024-01-01" "2024-12-31"]]}]+ (testing (str column " bucketed by year, written as " shape)+ (let [result (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "filters" filters+ "aggregation" [["count" {}]]}]}))]+ (is (some? (get-in result [:structured-output :query])))))))))++;; The pin for the test above: `year` is an extraction unit as well as a truncation one, and what+;; buys a coerced column its bucket is the coercion, not the unit. `ID` and `STATUS` are the+;; uncoerced twins of the two columns accepted above - same `base-type`, no `effective-type`.+(deftest temporal-bucket-by-year-on-non-temporal-column-rejected-test+ (with-coerced-mp-and-stubs!+ (fn []+ (doseq [[column type-name] {"ID" "Integer", "STATUS" "Text"}]+ (testing (str column " bucketed by year")+ (try+ (construct/execute-representations-query+ (query-data {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "filters" [["=" {} ["field" {"temporal-unit" "year"}+ ["Sample" "PUBLIC" "ORDERS" column]]+ "2024-01-01"]]+ "aggregation" [["count" {}]]}]}))+ (is false "expected throw")+ (catch clojure.lang.ExceptionInfo e+ (let [d (ex-data e)]+ (is (true? (:agent-error? d)))+ (is (= :temporal-unit-on-non-temporal-column (:error d)))+ (is (str/includes? (ex-message e) (str "is a " type-name " column")))))))))))diff --git a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.cljindex 1b6710c7382..c7e4075b9c7 100644--- a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj+++ b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj@@ -571,6 +571,118 @@[:field (meta/id :checkins :date) {:base-type :type/Date, :temporal-unit unit}][:absolute-datetime #t "2014-05-08" unit]]}))))))))+(deftest ^:parallel optimize-date-not-equals-null-semantics-test+ (testing (str "`!=` against a DATE column bucketed by `day` is NOT interchangeable between "+ "bucket-on-the-ref and bucket-on-the-literal: only the former keeps the `!=`, and "+ "so the `IS NULL` disjunct `sql.qp` adds to every `!=`. Pass 2.95 in agent-lib "+ "repair hoists the bucket onto the ref for exactly this reason (BOT-2095).")+ (testing "bucket on the ref: stays `!=`, so NULL rows are included"+ (is (=? {:query {:filter [:!=+ [:field (meta/id :checkins :date) {:base-type :type/Date, :temporal-unit :default}]+ [:absolute-datetime #t "2014-05-08" :default]]}}+ (optimize-query+ (lib.tu.macros/mbql-query checkins+ {:filter [:!=+ [:field (meta/id :checkins :date) {:base-type :type/Date, :temporal-unit :day}]+ [:absolute-datetime #t "2014-05-08" :day]]})))))+ (testing "bucket on the literal: becomes a negated range, which excludes NULL rows"+ (is (=? {:query {:filter [:or+ [:< [:field (meta/id :checkins :date) {:base-type :type/Date}]+ [:absolute-datetime #t "2014-05-08" :default]]+ [:>= [:field (meta/id :checkins :date) {:base-type :type/Date}]+ [:absolute-datetime #t "2014-05-09" :default]]]}}+ (optimize-query+ (lib.tu.macros/mbql-query checkins+ {:filter [:!=+ [:field (meta/id :checkins :date) {:base-type :type/Date}]+ [:absolute-datetime #t "2014-05-08" :day]]})))))))++(deftest ^:parallel optimize-untyped-expression-ref-not-optimized-test+ (testing (str "`temporal-ref?` reads the column type off the ref's own options, and resolve stamps "+ "none on an `expression` ref, so the literal-side form is left unoptimized on an "+ "untyped ref and rewritten on a typed one. Pass 2.95 in agent-lib repair hoists the "+ "bucket onto the ref, which is what makes the two agree again (BOT-2095).")+ (testing "untyped ref, bucket on the literal: left alone"+ (let [clause [:> [:expression "date"] [:absolute-datetime #t "2025-06-01" :month]]]+ (is (= clause (optimize-filters clause)))))+ (testing "typed ref, same clause: rewritten into a range"+ (is (= [:>= [:expression "date" {:base-type :type/DateTime}] [:absolute-datetime #t "2025-07-01" :default]]+ (optimize-filters [:> [:expression "date" {:base-type :type/DateTime}]+ [:absolute-datetime #t "2025-06-01" :month]]))))+ ;; The hoisted form Pass 2.95 emits is `<ref bucketed by month> > "2025-06-01"`, and by the time+ ;; this middleware runs `wrap-value-literals` has turned that bare string into a `:day` literal+ ;; WITHOUT copying the ref's unit onto it - so the units disagree and nothing is optimized. Write+ ;; the literal as `:default` here instead and the clause *is* optimized: a different fact, pinned+ ;; under this test's name. Nor is this half about typing - the typed ref behaves identically.+ (testing "bucket on the ref: left alone whether or not the ref carries a type"+ (doseq [opts [{:temporal-unit :month}+ {:base-type :type/DateTime, :temporal-unit :month}]]+ (let [clause [:> [:expression "date" opts] [:absolute-datetime #t "2025-06-01" :day]]]+ (is (= clause (optimize-filters clause)) (pr-str opts)))))))++(deftest ^:parallel expression-ref-hoisted-bucket-needs-an-aligned-literal-test+ (testing (str "This middleware truncates BOTH sides of a comparison, so a literal sitting in the "+ "middle of its bucket costs a `field` ref nothing. It never fires on an `expression` "+ "ref, where only the column is truncated - so agent-lib repair's Pass 2.95 truncates "+ "the literal itself when it hoists a bucket onto a ref. Without that the hoisted "+ "clause asks a different question and answers it silently (BOT-2095).")+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ created-at (lib.metadata/field mp (mt/id :people :created_at))+ ;; `Ship` is `created_at` shifted by zero days, so the field-ref answer is the right one+ base (-> (lib/query mp (lib.metadata/table mp (mt/id :people)))+ (lib/expression "Ship" (lib/datetime-add created-at 0 :day)))+ cnt (fn [filter-clause]+ (-> base+ (lib/filter filter-clause)+ (lib/aggregate (lib/count))+ qp/process-query+ mt/rows+ ffirst))+ fld (fn [unit] (lib/with-temporal-bucket (lib/ref created-at) unit))+ expr (fn [unit] (lib/with-temporal-bucket (lib/expression-ref base "Ship") unit))]+ (doseq [[label ground-clause unaligned-clause aligned-clause]+ [["="+ (lib/= (fld :month) "2018-06-15")+ (lib/= (expr :month) "2018-06-15")+ (lib/= (expr :month) "2018-06-01")]+ ["<"+ (lib/< (fld :month) "2018-06-15")+ (lib/< (expr :month) "2018-06-15")+ (lib/< (expr :month) "2018-06-01")]+ [">="+ (lib/>= (fld :month) "2018-06-15")+ (lib/>= (expr :month) "2018-06-15")+ (lib/>= (expr :month) "2018-06-01")]+ ["between"+ (lib/between (fld :month) "2018-06-15" "2018-09-20")+ (lib/between (expr :month) "2018-06-15" "2018-09-20")+ (lib/between (expr :month) "2018-06-01" "2018-09-01")]]]+ (testing label+ (let [ground (cnt ground-clause)]+ (is (pos? ground)+ "the field-ref clause has to match some rows, or the comparison below is vacuous")+ (is (= ground (cnt aligned-clause))+ "the aligned literal reproduces the field-ref answer")+ (is (not= ground (cnt unaligned-clause))+ "the unaligned literal does not - this is the defect Pass 2.95 aligns away"))))+ ;; This is what licenses Pass 2.95 applying the alignment unconditionally instead of only+ ;; when the ref is an `expression` - the middleware truncates the literal for a `field` ref+ ;; anyway, so pre-truncating it cannot move the answer.+ (testing "aligning the literal is a no-op for a field ref"+ (doseq [[label aligned unaligned]+ [["=" (lib/= (fld :month) "2018-06-01") (lib/= (fld :month) "2018-06-15")]+ ["!=" (lib/!= (fld :month) "2018-06-01") (lib/!= (fld :month) "2018-06-15")]+ ["<" (lib/< (fld :month) "2018-06-01") (lib/< (fld :month) "2018-06-15")]+ ["<=" (lib/<= (fld :month) "2018-06-01") (lib/<= (fld :month) "2018-06-15")]+ [">" (lib/> (fld :month) "2018-06-01") (lib/> (fld :month) "2018-06-15")]+ [">=" (lib/>= (fld :month) "2018-06-01") (lib/>= (fld :month) "2018-06-15")]+ ["quarter" (lib/= (fld :quarter) "2018-04-01") (lib/= (fld :quarter) "2018-06-15")]+ ["between" (lib/between (fld :month) "2018-06-01" "2018-09-01")+ (lib/between (fld :month) "2018-06-15" "2018-09-20")]]]+ (testing label+ (is (= (cnt unaligned) (cnt aligned))))))))))+(deftest ^:parallel do-not-change-unit-of-relative-datetime-to-default-test(testing "Never change the unit of a relative datetime to :default. That would not make any sense."(is (= {:database (meta/id)