Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Parse :absolute-datetime string literals in repr resolve (#76436)

Mihael Konjević898f75a2b115
diff --git a/src/metabase/agent_lib/representations/resolve.clj b/src/metabase/agent_lib/representations/resolve.clj
index c8c41eaee01..c69cfd8853a 100644
--- a/src/metabase/agent_lib/representations/resolve.clj
+++ b/src/metabase/agent_lib/representations/resolve.clj
@@ -34,7 +34,11 @@
[metabase.lib.schema :as lib.schema]
[metabase.models.serialization.resolve :as resolve]
[metabase.models.serialization.resolve.mp :as resolve.mp]
- [metabase.util.log :as log]))
+ [metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.log :as log]
+ [metabase.util.match :as match]))
(set! *warn-on-reflection* true)
@@ -99,6 +103,46 @@
node))
pmbql-query))
+(defn- assert-parseable-temporal-literal!
+ "Throw an agent-facing error if temporal string `s` is not a real date / datetime / year /
+ year-month — i.e. one [[u.date/parse]] can read. The query schema only checks structure with a
+ regex, so a value like `\"2024-13-45\"` passes normalization but is not a valid date."
+ [s]
+ (try
+ (u.date/parse s)
+ nil
+ (catch Exception e
+ ;; `s` is untrusted agent input — bound it before it lands in an exception message or ex-data so a
+ ;; pathological literal can't flood logs. A real temporal literal is well under this length.
+ (let [s' (u/truncate s 64)]
+ (throw (ex-info (tru "Invalid temporal literal {0} in :absolute-datetime — use an ISO-8601 date, datetime, year, or year-month."
+ (pr-str s'))
+ {:agent-error? true
+ :status-code 400
+ :error :invalid-temporal-literal
+ :literal s'}
+ e))))))
+
+(defn- validate-absolute-datetime-literals
+ "Validate (without coercing) the string literals in `:absolute-datetime` clauses, so malformed
+ temporal input fails fast here — where the agent sees it and can correct — instead of surviving to
+ query execution.
+
+ The actual string → `java.time` coercion is intentionally NOT done here: it happens later in the
+ QP's `wrap-value-literals`, where the comparison field's type and the report timezone are
+ available. This pass only rejects literals that can't possibly parse. Returns the query unchanged."
+ [pmbql-query]
+ ;; `match/match-many`, not `lib.walk/walk-clauses`: the latter is a `mu/defn` that validates its
+ ;; whole-query argument against `::lib.schema/query`, so under test instrumentation it would throw a
+ ;; raw "Invalid input" on an otherwise-malformed query (e.g. an `:offset` in `:expressions`) here,
+ ;; pre-empting the friendlier not-runnable gate downstream. We only need to inspect literals, and
+ ;; matching a bare `[:absolute-datetime _ s _]` vector suffices — the same shape check the sibling
+ ;; `annotate-field-types` pass uses.
+ (match/match-many pmbql-query
+ [:absolute-datetime _ (s :guard string?) _]
+ (assert-parseable-temporal-literal! s))
+ pmbql-query)
+
(defn resolve-query
"Convert a parsed (string-keyed, portable) representations query into a canonical, numeric-ID,
`:lib/uuid`-stamped MBQL 5 query attached to `metadata-provider`.
@@ -117,7 +161,8 @@
resolved (resolve/import-mbql resolver kw-form)
with-mp (assoc resolved :lib/metadata metadata-provider)]
(-> (lib.normalize/normalize ::lib.schema/query with-mp)
- (annotate-field-types metadata-provider)))))
+ (annotate-field-types metadata-provider)
+ validate-absolute-datetime-literals))))
;;; ============================================================
;;; Export final pMBQL back to portable representations
diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.clj
index 6b1a580d380..6d7f4ac96c6 100644
--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj
+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj
@@ -220,6 +220,64 @@
(def ^:private raw-value? (complement lib/clause?))
+;; Some queries carry temporal literals in their portable / wire form — e.g. Metabot's representations
+;; repair wraps `between` bounds as `[:absolute-datetime {} "2024-01-01" :day]`, and on CLJS a string
+;; is the only representation. The comparison arms below parse temporal strings that arrive as *raw*
+;; values, but would otherwise skip ones already wrapped in `:absolute-datetime`, leaving a bare string
+;; that later middleware (e.g. `optimize-temporal-filters`) chokes on. This predicate lets those arms
+;; detect such clauses; the unit travels with the string so a `:year`/`:month` literal keeps its bucket
+;; instead of collapsing to the field's default when re-parsed.
+(defn- string-valued-absolute-datetime
+ "If `x` is an `:absolute-datetime` clause whose literal is still an (unparsed) string, return a
+ `[string unit]` pair of that inner string and the clause's temporal unit; otherwise `nil`."
+ [x]
+ (when (lib/clause-of-type? x :absolute-datetime)
+ (let [[_tag _opts v unit] x]
+ (when (string? v) [v unit]))))
+
+(defn- wrappable-literal?
+ "A raw value, or a string-valued `:absolute-datetime` that still needs parsing."
+ [x]
+ (or (raw-value? x)
+ (some? (string-valued-absolute-datetime x))))
+
+(defn- wrap-literal-against-field
+ "Wrap a single comparison literal `x` against the type info of the `field` it is compared to.
+
+ A raw value, or a string-valued `:absolute-datetime`, is re-parsed through [[add-type-info]] so it
+ picks up the field's effective type and the report timezone — same as a raw string would. The
+ comparison field's own bucket still wins, but when the field is *unbucketed* the literal's own unit
+ is used as the fallback instead of `:default`, so e.g. `[:absolute-datetime {} \"2024\" :year]`
+ against an unbucketed field keeps its `:year` bucket instead of collapsing to a single instant. A
+ field carrying an explicit `:default` unit counts as unbucketed here — `add-default-temporal-unit`
+ stamps `:default` onto every temporal field ref on `:temporal/requires-default-unit` drivers, and an
+ LLM can author it directly — so the literal's unit still wins in that case.
+ Anything else — an already-parsed `:absolute-datetime`, `:now`, `:relative-datetime`, another
+ field — is returned untouched, so a *mixed* `:between` only wraps the bound(s) that need it."
+ [query path field x]
+ (if-let [[s unit] (string-valued-absolute-datetime x)]
+ (let [col (*type-info* query path field)]
+ ;; `contains?`, not `(#{nil :default} (:unit col))`: a set used as a fn returns the *found value*,
+ ;; so it would report `nil` (falsy) for an unbucketed `nil`-unit field and drop the fallback.
+ (add-type-info s (cond-> col
+ (and unit (contains? #{nil :default} (:unit col))) (assoc :unit unit))))
+ (if (raw-value? x)
+ (add-type-info x (*type-info* query path field))
+ x)))
+
+(defn- parse-fieldless-literal
+ "Parse a string-valued `:absolute-datetime` standalone — using its own unit and no field type — for a
+ comparison of two literals where there is no field to wrap against (e.g. `[:= <abs-datetime-string>
+ <abs-datetime-string>]`). Without this the bare string survives to execution and later middleware
+ (e.g. `optimize-temporal-filters`) chokes on it. Anything else is returned untouched."
+ [x]
+ (if-let [[s unit] (string-valued-absolute-datetime x)]
+ ;; pre-parse to a `java.time` value (no field type / report timezone to apply here) so the result
+ ;; is the literal's own value — a date-only string stays a `LocalDate`, not a timezone-dependent
+ ;; instant — rather than routing through the tz-aware string path.
+ (add-type-info (u.date/parse s) {:unit (or unit :default)})
+ x))
+
(defn- wrap-value-literals-in-clause
[query path clause]
(match/match-one clause
@@ -231,13 +289,23 @@
(add-type-info x {:base-type x-type :effective-type x-type})
(add-type-info y {:base-type y-type :effective-type y-type})])
- ;; field and literal
- [(tag :guard #{:= :!= :< :> :<= :>=}) opts field (x :guard raw-value?)]
- [tag opts field (add-type-info x (*type-info* query path field))]
+ ;; field and literal. `field` is guarded as *not* a wrappable literal so that a comparison of two
+ ;; literals (e.g. `[:= <abs-datetime-string> <abs-datetime-string>]`) can't bind a literal as the
+ ;; `field` and leave it unwrapped — it falls through unchanged instead.
+ [(tag :guard #{:= :!= :< :> :<= :>=}) opts (field :guard (not (wrappable-literal? field))) (x :guard wrappable-literal?)]
+ [tag opts field (wrap-literal-against-field query path field x)]
;; literal and field (literal on LHS)
- [(tag :guard #{:= :!= :< :> :<= :>=}) opts (x :guard raw-value?) field]
- [tag opts (add-type-info x (*type-info* query path field)) field]
+ [(tag :guard #{:= :!= :< :> :<= :>=}) opts (x :guard wrappable-literal?) (field :guard (not (wrappable-literal? field)))]
+ [tag opts (wrap-literal-against-field query path field x) field]
+
+ ;; two literals, no field to wrap against — still parse a string-valued `:absolute-datetime` (using
+ ;; its own unit) so a bare string can't survive to execution. Guarded on
+ ;; `string-valued-absolute-datetime` so `[:= 1 2]`, two already-parsed bounds, etc. fall through
+ ;; unchanged; raw values stay raw since there's no field type to attach.
+ [(tag :guard #{:= :!= :< :> :<= :>=}) opts x y]
+ (when (or (string-valued-absolute-datetime x) (string-valued-absolute-datetime y))
+ [tag opts (parse-fieldless-literal x) (parse-fieldless-literal y)])
[:datetime-diff opts (x :guard string?) (y :guard string?) unit]
[:datetime-diff opts (add-type-info (u.date/parse x) nil) (add-type-info (u.date/parse y) nil) unit]
@@ -245,12 +313,16 @@
[(tag :guard #{:datetime-add :datetime-subtract :convert-timezone :temporal-extract}) opts (field :guard string?) & args]
(into [tag opts (add-type-info (u.date/parse field) nil)] args)
- [:between opts field (min-val :guard raw-value?) (max-val :guard raw-value?)]
- [:between
- opts
- field
- (add-type-info min-val (*type-info* query path field))
- (add-type-info max-val (*type-info* query path field))]
+ ;; wrap each bound independently so a *mixed* `:between` — e.g. a string-valued `:absolute-datetime`
+ ;; on one side and `:now` / `:relative-datetime` / an already-parsed `:absolute-datetime` on the
+ ;; other — still gets the wrappable bound coerced instead of falling through unwrapped.
+ [:between opts field min-val max-val]
+ (when (or (wrappable-literal? min-val) (wrappable-literal? max-val))
+ [:between
+ opts
+ field
+ (wrap-literal-against-field query path field min-val)
+ (wrap-literal-against-field query path field max-val)])
[(tag :guard #{:starts-with :ends-with :contains}) opts field (s :guard string?) & more]
(let [s (add-type-info s (*type-info* query path field), :parse-datetime-strings? false)]
diff --git a/test/metabase/agent_lib/representations/resolve_test.clj b/test/metabase/agent_lib/representations/resolve_test.clj
index 51ab9765c07..07c3c4c7c9a 100644
--- a/test/metabase/agent_lib/representations/resolve_test.clj
+++ b/test/metabase/agent_lib/representations/resolve_test.clj
@@ -133,6 +133,44 @@
(testing "query passes lib.schema/query"
(is (schema-valid? q)))))
+;;; ============================================================
+;;; :absolute-datetime literal validation (fail-fast for the agent)
+;;; ============================================================
+
+(defn- resolve-absolute-datetime
+ "Resolve a query whose only filter is `[= CREATED_AT [absolute-datetime <literal> <unit>]]` and
+ return the (still-string) literal at position 2 of the resolved `:absolute-datetime` clause."
+ [literal unit]
+ (let [parsed {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "filters" [["=" {}
+ ["field" {"temporal-unit" "month"}
+ ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]
+ ["absolute-datetime" {} literal unit]]]}]}
+ q (repr.resolve/resolve-query mp parsed)]
+ (nth (get-in q [:stages 0 :filters 0 3]) 2)))
+
+(deftest validate-absolute-datetime-literals-accepts-all-formats-test
+ (testing "every valid :absolute-datetime string shape passes validation and is left as a string"
+ (doseq [[literal unit] [["2024-01-15" "day"]
+ ["2024-01-15T10:30:00" "default"]
+ ["2024-01-15T10:30:00+02:00" "default"]
+ ["2024" "year"]
+ ["2024-03" "month"]]]
+ (testing (str literal " / " unit)
+ ;; validation does not coerce — the string survives for the QP to parse at execution time
+ (is (= literal (resolve-absolute-datetime literal unit)))))))
+
+(deftest validate-absolute-datetime-literals-rejects-invalid-test
+ (testing "a structurally-ISO but out-of-range literal fails fast with an agent-facing error"
+ (let [ex (try (resolve-absolute-datetime "2024-13-45" "day") nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? ex))
+ (is (re-find #"Invalid temporal literal" (ex-message ex)))
+ (is (true? (:agent-error? (ex-data ex)))))))
+
;;; ============================================================
;;; fields (projection)
;;; ============================================================
diff --git a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
index f727a3b0469..e148a1ba38b 100644
--- a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
+++ b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
@@ -194,6 +194,186 @@
:query))
"Datetime literal strings should get parsed in the current report timezone.")))
+(deftest ^:parallel wrap-prewrapped-absolute-datetime-strings-test
+ (testing (str "an `:absolute-datetime` whose literal is still a string (e.g. Metabot's "
+ "representations repair wraps `between` bounds as "
+ "`[:absolute-datetime {} \"2024-01-01\" :day]`) gets parsed to a `java.time` value, "
+ "in the comparison field's type and unit — same as a raw string would")
+ (is (=? {:filters [[:between
+ {}
+ [:field {:temporal-unit :month} (meta/id :checkins :date)]
+ [:absolute-datetime {} (t/local-date "2024-01-01") :month]
+ [:absolute-datetime {} (t/local-date "2024-12-31") :month]]]}
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:between
+ [:field (meta/id :checkins :date) {:temporal-unit :month}]
+ [:absolute-datetime "2024-01-01" :day]
+ [:absolute-datetime "2024-12-31" :day]]}))
+ wrap-value-literals
+ :stages
+ first))))
+ (testing "an `:absolute-datetime` whose literal is already a `java.time` value is left unchanged"
+ (let [parsed (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:= [:field (meta/id :checkins :date) {:temporal-unit :month}]
+ [:absolute-datetime (t/local-date "2024-01-01") :month]]}))]
+ (is (= (:stages parsed) (:stages (wrap-value-literals parsed)))))))
+
+(deftest wrap-prewrapped-absolute-datetime-strings-tz-test
+ (testing (str "a pre-wrapped local string is coerced using the report timezone, by the field's "
+ "type — the type/tz awareness that can only happen here in the QP, at execution time")
+ (mt/with-report-timezone-id! "US/Pacific"
+ (is (= (:query
+ (lib.tu.macros/mbql-query checkins
+ {:source-query {:source-table $$checkins}
+ :filter [:=
+ [:field "DATE" {:temporal-unit :day, :base-type :type/DateTimeWithZoneID}]
+ [:absolute-datetime (t/zoned-date-time "2018-10-01T00:00-07:00[US/Pacific]") :day]]}))
+ (-> (lib.tu.macros/mbql-query checkins
+ {:source-query {:source-table $$checkins}
+ :filter [:= [:field "DATE" {:temporal-unit :day, :base-type :type/DateTimeWithZoneID}]
+ [:absolute-datetime "2018-10-01" :day]]})
+ (assoc-in [:settings :report-timezone] "US/Pacific")
+ (wrap-value-literals "US/Pacific")
+ :query))
+ "pre-wrapped string coerced to the report timezone for a DateTimeWithZoneID field"))))
+
+(defn- year-literal-filter-query
+ "Build a pMBQL query whose `:=` filter compares an *unbucketed* `checkins.date` against a string-valued
+ `[:absolute-datetime {} `s` `unit`]`. Year / year-month string literals can't be expressed through the
+ legacy `mbql-query` macro, so we build a valid query and swap the literal into the already-wrapped
+ pMBQL clause."
+ [s unit]
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:= [:field (meta/id :checkins :date) nil]
+ [:absolute-datetime "2024-01-01" :day]]}))
+ (assoc-in [:stages 0 :filters 0 3 2] s)
+ (assoc-in [:stages 0 :filters 0 3 3] unit)))
+
+(deftest ^:parallel wrap-prewrapped-absolute-datetime-unit-test
+ (testing (str "a string-valued `:absolute-datetime` keeps its own temporal unit when re-parsed against "
+ "an *unbucketed* field, so a year / year-month literal is not collapsed to a single "
+ ":default-bucketed instant")
+ (doseq [[unit s expected] [[:year "2024" "2024-01-01"]
+ [:month "2024-03" "2024-03-01"]]]
+ (is (=? {:filters [[:= {}
+ [:field {} (meta/id :checkins :date)]
+ [:absolute-datetime {} (t/local-date expected) unit]]]}
+ (-> (year-literal-filter-query s unit)
+ wrap-value-literals
+ :stages
+ first))
+ (str (name unit) " literal keeps its unit against an unbucketed field"))))
+ (testing "the comparison field's own bucket still wins over the literal's unit when the field is bucketed"
+ (is (=? {:filters [[:= {}
+ [:field {:temporal-unit :month} (meta/id :checkins :date)]
+ [:absolute-datetime {} (t/local-date "2024-01-01") :month]]]}
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:= [:field (meta/id :checkins :date) {:temporal-unit :month}]
+ [:absolute-datetime "2024-01-01" :day]]}))
+ wrap-value-literals
+ :stages
+ first)))))
+
+(deftest ^:parallel wrap-prewrapped-absolute-datetime-default-unit-test
+ (testing (str "a field carrying an explicit `:default` temporal unit counts as *unbucketed*, so the "
+ "literal's own unit still wins instead of collapsing to a single `:default`-bucketed "
+ "instant. `add-default-temporal-unit` stamps `:default` onto every temporal field ref on "
+ "`:temporal/requires-default-unit` drivers (Druid family), so the unit-preservation "
+ "feature would otherwise be fully defeated there.")
+ (doseq [[unit s expected] [[:year "2024" "2024-01-01"]
+ [:month "2024-03" "2024-03-01"]]]
+ (is (=? {:filters [[:= {}
+ [:field {:temporal-unit :default} (meta/id :checkins :date)]
+ [:absolute-datetime {} (t/local-date expected) unit]]]}
+ (-> (year-literal-filter-query s unit)
+ (assoc-in [:stages 0 :filters 0 2 1 :temporal-unit] :default)
+ wrap-value-literals
+ :stages
+ first))
+ (str (name unit) " literal keeps its unit against a `:default`-bucketed field")))))
+
+(deftest ^:parallel wrap-prewrapped-absolute-datetime-mixed-between-test
+ (testing "a mixed `:between` wraps only the string-valued bound and passes the other through untouched"
+ (testing "string bound + already-parsed `:absolute-datetime` bound"
+ (is (=? {:filters [[:between {}
+ [:field {} (meta/id :checkins :date)]
+ [:absolute-datetime {} (t/local-date "2024-01-01") :day]
+ [:absolute-datetime {} (t/local-date "2024-12-31") :month]]]}
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:between
+ [:field (meta/id :checkins :date) nil]
+ [:absolute-datetime "2024-01-01" :day]
+ [:absolute-datetime (t/local-date "2024-12-31") :month]]}))
+ wrap-value-literals
+ :stages
+ first))
+ "the string bound is parsed; the already-parsed bound is left as-is"))
+ (testing "string bound + `:relative-datetime` bound"
+ (is (=? {:filters [[:between {}
+ [:field {} (meta/id :checkins :date)]
+ [:absolute-datetime {} (t/local-date "2024-01-01") :day]
+ [:relative-datetime {} 0 :day]]]}
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:between
+ [:field (meta/id :checkins :date) nil]
+ [:absolute-datetime "2024-01-01" :day]
+ [:relative-datetime 0 :day]]}))
+ wrap-value-literals
+ :stages
+ first))
+ "the `:relative-datetime` bound is left untouched"))
+ (testing "raw (unwrapped) string bound + `:relative-datetime` bound"
+ (is (=? {:filters [[:between {}
+ [:field {} (meta/id :checkins :date)]
+ [:absolute-datetime {} (t/local-date "2026-01-01") :default]
+ [:relative-datetime {} 0 :day]]]}
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:between
+ [:field (meta/id :checkins :date) nil]
+ "2026-01-01"
+ [:relative-datetime 0 :day]]}))
+ wrap-value-literals
+ :stages
+ first))
+ "the raw string bound still gets wrapped, same as when the `:relative-datetime` bound isn't present"))))
+
+(deftest ^:parallel wrap-prewrapped-absolute-datetime-two-literals-test
+ (testing (str "a comparison with a string-valued `:absolute-datetime` on *both* sides (no field to "
+ "wrap against) still has each literal parsed using its own unit — neither is mis-bound "
+ "as the `field` and silently dropped, and no bare string survives to execution")
+ (is (=? {:filters [[:= {}
+ [:absolute-datetime {} (t/local-date "2024-01-01") :day]
+ [:absolute-datetime {} (t/local-date "2025-01-01") :day]]]}
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:= [:absolute-datetime "2024-01-01" :day]
+ [:absolute-datetime "2025-01-01" :day]]}))
+ wrap-value-literals
+ :stages
+ first))))
+ (testing "two already-parsed `:absolute-datetime` literals are left unchanged (nothing to parse)"
+ (let [parsed (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query checkins
+ {:filter [:= [:absolute-datetime (t/local-date "2024-01-01") :day]
+ [:absolute-datetime (t/local-date "2025-01-01") :day]]}))]
+ (is (= (:stages parsed) (:stages (wrap-value-literals parsed)))))))
+
(deftest ^:parallel string-filters-test
(testing "string filters like `starts-with` should not parse datetime strings for obvious reasons"
(is (= (lib.tu.macros/mbql-query checkins