The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Parse :absolute-datetime string literals in repr resolve (#76436)
Mihael Konjević898f75a2b115
diff --git a/src/metabase/agent_lib/representations/resolve.clj b/src/metabase/agent_lib/representations/resolve.cljindex c8c41eaee01..c69cfd8853a 100644--- a/src/metabase/agent_lib/representations/resolve.clj+++ b/src/metabase/agent_lib/representations/resolve.clj@@ -34,7 +34,11 @@[metabase.lib.schema :as lib.schema][metabase.models.serialization.resolve :as resolve][metabase.models.serialization.resolve.mp :as resolve.mp]- [metabase.util.log :as log]))+ [metabase.util :as u]+ [metabase.util.date-2 :as u.date]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.log :as log]+ [metabase.util.match :as match]))(set! *warn-on-reflection* true)@@ -99,6 +103,46 @@node))pmbql-query))+(defn- assert-parseable-temporal-literal!+ "Throw an agent-facing error if temporal string `s` is not a real date / datetime / year /+ year-month — i.e. one [[u.date/parse]] can read. The query schema only checks structure with a+ regex, so a value like `\"2024-13-45\"` passes normalization but is not a valid date."+ [s]+ (try+ (u.date/parse s)+ nil+ (catch Exception e+ ;; `s` is untrusted agent input — bound it before it lands in an exception message or ex-data so a+ ;; pathological literal can't flood logs. A real temporal literal is well under this length.+ (let [s' (u/truncate s 64)]+ (throw (ex-info (tru "Invalid temporal literal {0} in :absolute-datetime — use an ISO-8601 date, datetime, year, or year-month."+ (pr-str s'))+ {:agent-error? true+ :status-code 400+ :error :invalid-temporal-literal+ :literal s'}+ e))))))++(defn- validate-absolute-datetime-literals+ "Validate (without coercing) the string literals in `:absolute-datetime` clauses, so malformed+ temporal input fails fast here — where the agent sees it and can correct — instead of surviving to+ query execution.++ The actual string → `java.time` coercion is intentionally NOT done here: it happens later in the+ QP's `wrap-value-literals`, where the comparison field's type and the report timezone are+ available. This pass only rejects literals that can't possibly parse. Returns the query unchanged."+ [pmbql-query]+ ;; `match/match-many`, not `lib.walk/walk-clauses`: the latter is a `mu/defn` that validates its+ ;; whole-query argument against `::lib.schema/query`, so under test instrumentation it would throw a+ ;; raw "Invalid input" on an otherwise-malformed query (e.g. an `:offset` in `:expressions`) here,+ ;; pre-empting the friendlier not-runnable gate downstream. We only need to inspect literals, and+ ;; matching a bare `[:absolute-datetime _ s _]` vector suffices — the same shape check the sibling+ ;; `annotate-field-types` pass uses.+ (match/match-many pmbql-query+ [:absolute-datetime _ (s :guard string?) _]+ (assert-parseable-temporal-literal! s))+ pmbql-query)+(defn resolve-query"Convert a parsed (string-keyed, portable) representations query into a canonical, numeric-ID,`:lib/uuid`-stamped MBQL 5 query attached to `metadata-provider`.@@ -117,7 +161,8 @@resolved (resolve/import-mbql resolver kw-form)with-mp (assoc resolved :lib/metadata metadata-provider)](-> (lib.normalize/normalize ::lib.schema/query with-mp)- (annotate-field-types metadata-provider)))))+ (annotate-field-types metadata-provider)+ validate-absolute-datetime-literals))));;; ============================================================;;; Export final pMBQL back to portable representationsdiff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.cljindex 6b1a580d380..6d7f4ac96c6 100644--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj@@ -220,6 +220,64 @@(def ^:private raw-value? (complement lib/clause?))+;; Some queries carry temporal literals in their portable / wire form — e.g. Metabot's representations+;; repair wraps `between` bounds as `[:absolute-datetime {} "2024-01-01" :day]`, and on CLJS a string+;; is the only representation. The comparison arms below parse temporal strings that arrive as *raw*+;; values, but would otherwise skip ones already wrapped in `:absolute-datetime`, leaving a bare string+;; that later middleware (e.g. `optimize-temporal-filters`) chokes on. This predicate lets those arms+;; detect such clauses; the unit travels with the string so a `:year`/`:month` literal keeps its bucket+;; instead of collapsing to the field's default when re-parsed.+(defn- string-valued-absolute-datetime+ "If `x` is an `:absolute-datetime` clause whose literal is still an (unparsed) string, return a+ `[string unit]` pair of that inner string and the clause's temporal unit; otherwise `nil`."+ [x]+ (when (lib/clause-of-type? x :absolute-datetime)+ (let [[_tag _opts v unit] x]+ (when (string? v) [v unit]))))++(defn- wrappable-literal?+ "A raw value, or a string-valued `:absolute-datetime` that still needs parsing."+ [x]+ (or (raw-value? x)+ (some? (string-valued-absolute-datetime x))))++(defn- wrap-literal-against-field+ "Wrap a single comparison literal `x` against the type info of the `field` it is compared to.++ A raw value, or a string-valued `:absolute-datetime`, is re-parsed through [[add-type-info]] so it+ picks up the field's effective type and the report timezone — same as a raw string would. The+ comparison field's own bucket still wins, but when the field is *unbucketed* the literal's own unit+ is used as the fallback instead of `:default`, so e.g. `[:absolute-datetime {} \"2024\" :year]`+ against an unbucketed field keeps its `:year` bucket instead of collapsing to a single instant. A+ field carrying an explicit `:default` unit counts as unbucketed here — `add-default-temporal-unit`+ stamps `:default` onto every temporal field ref on `:temporal/requires-default-unit` drivers, and an+ LLM can author it directly — so the literal's unit still wins in that case.+ Anything else — an already-parsed `:absolute-datetime`, `:now`, `:relative-datetime`, another+ field — is returned untouched, so a *mixed* `:between` only wraps the bound(s) that need it."+ [query path field x]+ (if-let [[s unit] (string-valued-absolute-datetime x)]+ (let [col (*type-info* query path field)]+ ;; `contains?`, not `(#{nil :default} (:unit col))`: a set used as a fn returns the *found value*,+ ;; so it would report `nil` (falsy) for an unbucketed `nil`-unit field and drop the fallback.+ (add-type-info s (cond-> col+ (and unit (contains? #{nil :default} (:unit col))) (assoc :unit unit))))+ (if (raw-value? x)+ (add-type-info x (*type-info* query path field))+ x)))++(defn- parse-fieldless-literal+ "Parse a string-valued `:absolute-datetime` standalone — using its own unit and no field type — for a+ comparison of two literals where there is no field to wrap against (e.g. `[:= <abs-datetime-string>+ <abs-datetime-string>]`). Without this the bare string survives to execution and later middleware+ (e.g. `optimize-temporal-filters`) chokes on it. Anything else is returned untouched."+ [x]+ (if-let [[s unit] (string-valued-absolute-datetime x)]+ ;; pre-parse to a `java.time` value (no field type / report timezone to apply here) so the result+ ;; is the literal's own value — a date-only string stays a `LocalDate`, not a timezone-dependent+ ;; instant — rather than routing through the tz-aware string path.+ (add-type-info (u.date/parse s) {:unit (or unit :default)})+ x))+(defn- wrap-value-literals-in-clause[query path clause](match/match-one clause@@ -231,13 +289,23 @@(add-type-info x {:base-type x-type :effective-type x-type})(add-type-info y {:base-type y-type :effective-type y-type})])- ;; field and literal- [(tag :guard #{:= :!= :< :> :<= :>=}) opts field (x :guard raw-value?)]- [tag opts field (add-type-info x (*type-info* query path field))]+ ;; field and literal. `field` is guarded as *not* a wrappable literal so that a comparison of two+ ;; literals (e.g. `[:= <abs-datetime-string> <abs-datetime-string>]`) can't bind a literal as the+ ;; `field` and leave it unwrapped — it falls through unchanged instead.+ [(tag :guard #{:= :!= :< :> :<= :>=}) opts (field :guard (not (wrappable-literal? field))) (x :guard wrappable-literal?)]+ [tag opts field (wrap-literal-against-field query path field x)];; literal and field (literal on LHS)- [(tag :guard #{:= :!= :< :> :<= :>=}) opts (x :guard raw-value?) field]- [tag opts (add-type-info x (*type-info* query path field)) field]+ [(tag :guard #{:= :!= :< :> :<= :>=}) opts (x :guard wrappable-literal?) (field :guard (not (wrappable-literal? field)))]+ [tag opts (wrap-literal-against-field query path field x) field]++ ;; two literals, no field to wrap against — still parse a string-valued `:absolute-datetime` (using+ ;; its own unit) so a bare string can't survive to execution. Guarded on+ ;; `string-valued-absolute-datetime` so `[:= 1 2]`, two already-parsed bounds, etc. fall through+ ;; unchanged; raw values stay raw since there's no field type to attach.+ [(tag :guard #{:= :!= :< :> :<= :>=}) opts x y]+ (when (or (string-valued-absolute-datetime x) (string-valued-absolute-datetime y))+ [tag opts (parse-fieldless-literal x) (parse-fieldless-literal y)])[:datetime-diff opts (x :guard string?) (y :guard string?) unit][:datetime-diff opts (add-type-info (u.date/parse x) nil) (add-type-info (u.date/parse y) nil) unit]@@ -245,12 +313,16 @@[(tag :guard #{:datetime-add :datetime-subtract :convert-timezone :temporal-extract}) opts (field :guard string?) & args](into [tag opts (add-type-info (u.date/parse field) nil)] args)- [:between opts field (min-val :guard raw-value?) (max-val :guard raw-value?)]- [:between- opts- field- (add-type-info min-val (*type-info* query path field))- (add-type-info max-val (*type-info* query path field))]+ ;; wrap each bound independently so a *mixed* `:between` — e.g. a string-valued `:absolute-datetime`+ ;; on one side and `:now` / `:relative-datetime` / an already-parsed `:absolute-datetime` on the+ ;; other — still gets the wrappable bound coerced instead of falling through unwrapped.+ [:between opts field min-val max-val]+ (when (or (wrappable-literal? min-val) (wrappable-literal? max-val))+ [:between+ opts+ field+ (wrap-literal-against-field query path field min-val)+ (wrap-literal-against-field query path field max-val)])[(tag :guard #{:starts-with :ends-with :contains}) opts field (s :guard string?) & more](let [s (add-type-info s (*type-info* query path field), :parse-datetime-strings? false)]diff --git a/test/metabase/agent_lib/representations/resolve_test.clj b/test/metabase/agent_lib/representations/resolve_test.cljindex 51ab9765c07..07c3c4c7c9a 100644--- a/test/metabase/agent_lib/representations/resolve_test.clj+++ b/test/metabase/agent_lib/representations/resolve_test.clj@@ -133,6 +133,44 @@(testing "query passes lib.schema/query"(is (schema-valid? q)))))+;;; ============================================================+;;; :absolute-datetime literal validation (fail-fast for the agent)+;;; ============================================================++(defn- resolve-absolute-datetime+ "Resolve a query whose only filter is `[= CREATED_AT [absolute-datetime <literal> <unit>]]` and+ return the (still-string) literal at position 2 of the resolved `:absolute-datetime` clause."+ [literal unit]+ (let [parsed {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "filters" [["=" {}+ ["field" {"temporal-unit" "month"}+ ["Sample" "PUBLIC" "ORDERS" "CREATED_AT"]]+ ["absolute-datetime" {} literal unit]]]}]}+ q (repr.resolve/resolve-query mp parsed)]+ (nth (get-in q [:stages 0 :filters 0 3]) 2)))++(deftest validate-absolute-datetime-literals-accepts-all-formats-test+ (testing "every valid :absolute-datetime string shape passes validation and is left as a string"+ (doseq [[literal unit] [["2024-01-15" "day"]+ ["2024-01-15T10:30:00" "default"]+ ["2024-01-15T10:30:00+02:00" "default"]+ ["2024" "year"]+ ["2024-03" "month"]]]+ (testing (str literal " / " unit)+ ;; validation does not coerce — the string survives for the QP to parse at execution time+ (is (= literal (resolve-absolute-datetime literal unit)))))))++(deftest validate-absolute-datetime-literals-rejects-invalid-test+ (testing "a structurally-ISO but out-of-range literal fails fast with an agent-facing error"+ (let [ex (try (resolve-absolute-datetime "2024-13-45" "day") nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? ex))+ (is (re-find #"Invalid temporal literal" (ex-message ex)))+ (is (true? (:agent-error? (ex-data ex)))))))+;;; ============================================================;;; fields (projection);;; ============================================================diff --git a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj b/test/metabase/query_processor/middleware/wrap_value_literals_test.cljindex f727a3b0469..e148a1ba38b 100644--- a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj+++ b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj@@ -194,6 +194,186 @@:query))"Datetime literal strings should get parsed in the current report timezone.")))+(deftest ^:parallel wrap-prewrapped-absolute-datetime-strings-test+ (testing (str "an `:absolute-datetime` whose literal is still a string (e.g. Metabot's "+ "representations repair wraps `between` bounds as "+ "`[:absolute-datetime {} \"2024-01-01\" :day]`) gets parsed to a `java.time` value, "+ "in the comparison field's type and unit — same as a raw string would")+ (is (=? {:filters [[:between+ {}+ [:field {:temporal-unit :month} (meta/id :checkins :date)]+ [:absolute-datetime {} (t/local-date "2024-01-01") :month]+ [:absolute-datetime {} (t/local-date "2024-12-31") :month]]]}+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:between+ [:field (meta/id :checkins :date) {:temporal-unit :month}]+ [:absolute-datetime "2024-01-01" :day]+ [:absolute-datetime "2024-12-31" :day]]}))+ wrap-value-literals+ :stages+ first))))+ (testing "an `:absolute-datetime` whose literal is already a `java.time` value is left unchanged"+ (let [parsed (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:= [:field (meta/id :checkins :date) {:temporal-unit :month}]+ [:absolute-datetime (t/local-date "2024-01-01") :month]]}))]+ (is (= (:stages parsed) (:stages (wrap-value-literals parsed)))))))++(deftest wrap-prewrapped-absolute-datetime-strings-tz-test+ (testing (str "a pre-wrapped local string is coerced using the report timezone, by the field's "+ "type — the type/tz awareness that can only happen here in the QP, at execution time")+ (mt/with-report-timezone-id! "US/Pacific"+ (is (= (:query+ (lib.tu.macros/mbql-query checkins+ {:source-query {:source-table $$checkins}+ :filter [:=+ [:field "DATE" {:temporal-unit :day, :base-type :type/DateTimeWithZoneID}]+ [:absolute-datetime (t/zoned-date-time "2018-10-01T00:00-07:00[US/Pacific]") :day]]}))+ (-> (lib.tu.macros/mbql-query checkins+ {:source-query {:source-table $$checkins}+ :filter [:= [:field "DATE" {:temporal-unit :day, :base-type :type/DateTimeWithZoneID}]+ [:absolute-datetime "2018-10-01" :day]]})+ (assoc-in [:settings :report-timezone] "US/Pacific")+ (wrap-value-literals "US/Pacific")+ :query))+ "pre-wrapped string coerced to the report timezone for a DateTimeWithZoneID field"))))++(defn- year-literal-filter-query+ "Build a pMBQL query whose `:=` filter compares an *unbucketed* `checkins.date` against a string-valued+ `[:absolute-datetime {} `s` `unit`]`. Year / year-month string literals can't be expressed through the+ legacy `mbql-query` macro, so we build a valid query and swap the literal into the already-wrapped+ pMBQL clause."+ [s unit]+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:= [:field (meta/id :checkins :date) nil]+ [:absolute-datetime "2024-01-01" :day]]}))+ (assoc-in [:stages 0 :filters 0 3 2] s)+ (assoc-in [:stages 0 :filters 0 3 3] unit)))++(deftest ^:parallel wrap-prewrapped-absolute-datetime-unit-test+ (testing (str "a string-valued `:absolute-datetime` keeps its own temporal unit when re-parsed against "+ "an *unbucketed* field, so a year / year-month literal is not collapsed to a single "+ ":default-bucketed instant")+ (doseq [[unit s expected] [[:year "2024" "2024-01-01"]+ [:month "2024-03" "2024-03-01"]]]+ (is (=? {:filters [[:= {}+ [:field {} (meta/id :checkins :date)]+ [:absolute-datetime {} (t/local-date expected) unit]]]}+ (-> (year-literal-filter-query s unit)+ wrap-value-literals+ :stages+ first))+ (str (name unit) " literal keeps its unit against an unbucketed field"))))+ (testing "the comparison field's own bucket still wins over the literal's unit when the field is bucketed"+ (is (=? {:filters [[:= {}+ [:field {:temporal-unit :month} (meta/id :checkins :date)]+ [:absolute-datetime {} (t/local-date "2024-01-01") :month]]]}+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:= [:field (meta/id :checkins :date) {:temporal-unit :month}]+ [:absolute-datetime "2024-01-01" :day]]}))+ wrap-value-literals+ :stages+ first)))))++(deftest ^:parallel wrap-prewrapped-absolute-datetime-default-unit-test+ (testing (str "a field carrying an explicit `:default` temporal unit counts as *unbucketed*, so the "+ "literal's own unit still wins instead of collapsing to a single `:default`-bucketed "+ "instant. `add-default-temporal-unit` stamps `:default` onto every temporal field ref on "+ "`:temporal/requires-default-unit` drivers (Druid family), so the unit-preservation "+ "feature would otherwise be fully defeated there.")+ (doseq [[unit s expected] [[:year "2024" "2024-01-01"]+ [:month "2024-03" "2024-03-01"]]]+ (is (=? {:filters [[:= {}+ [:field {:temporal-unit :default} (meta/id :checkins :date)]+ [:absolute-datetime {} (t/local-date expected) unit]]]}+ (-> (year-literal-filter-query s unit)+ (assoc-in [:stages 0 :filters 0 2 1 :temporal-unit] :default)+ wrap-value-literals+ :stages+ first))+ (str (name unit) " literal keeps its unit against a `:default`-bucketed field")))))++(deftest ^:parallel wrap-prewrapped-absolute-datetime-mixed-between-test+ (testing "a mixed `:between` wraps only the string-valued bound and passes the other through untouched"+ (testing "string bound + already-parsed `:absolute-datetime` bound"+ (is (=? {:filters [[:between {}+ [:field {} (meta/id :checkins :date)]+ [:absolute-datetime {} (t/local-date "2024-01-01") :day]+ [:absolute-datetime {} (t/local-date "2024-12-31") :month]]]}+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:between+ [:field (meta/id :checkins :date) nil]+ [:absolute-datetime "2024-01-01" :day]+ [:absolute-datetime (t/local-date "2024-12-31") :month]]}))+ wrap-value-literals+ :stages+ first))+ "the string bound is parsed; the already-parsed bound is left as-is"))+ (testing "string bound + `:relative-datetime` bound"+ (is (=? {:filters [[:between {}+ [:field {} (meta/id :checkins :date)]+ [:absolute-datetime {} (t/local-date "2024-01-01") :day]+ [:relative-datetime {} 0 :day]]]}+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:between+ [:field (meta/id :checkins :date) nil]+ [:absolute-datetime "2024-01-01" :day]+ [:relative-datetime 0 :day]]}))+ wrap-value-literals+ :stages+ first))+ "the `:relative-datetime` bound is left untouched"))+ (testing "raw (unwrapped) string bound + `:relative-datetime` bound"+ (is (=? {:filters [[:between {}+ [:field {} (meta/id :checkins :date)]+ [:absolute-datetime {} (t/local-date "2026-01-01") :default]+ [:relative-datetime {} 0 :day]]]}+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:between+ [:field (meta/id :checkins :date) nil]+ "2026-01-01"+ [:relative-datetime 0 :day]]}))+ wrap-value-literals+ :stages+ first))+ "the raw string bound still gets wrapped, same as when the `:relative-datetime` bound isn't present"))))++(deftest ^:parallel wrap-prewrapped-absolute-datetime-two-literals-test+ (testing (str "a comparison with a string-valued `:absolute-datetime` on *both* sides (no field to "+ "wrap against) still has each literal parsed using its own unit — neither is mis-bound "+ "as the `field` and silently dropped, and no bare string survives to execution")+ (is (=? {:filters [[:= {}+ [:absolute-datetime {} (t/local-date "2024-01-01") :day]+ [:absolute-datetime {} (t/local-date "2025-01-01") :day]]]}+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:= [:absolute-datetime "2024-01-01" :day]+ [:absolute-datetime "2025-01-01" :day]]}))+ wrap-value-literals+ :stages+ first))))+ (testing "two already-parsed `:absolute-datetime` literals are left unchanged (nothing to parse)"+ (let [parsed (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query checkins+ {:filter [:= [:absolute-datetime (t/local-date "2024-01-01") :day]+ [:absolute-datetime (t/local-date "2025-01-01") :day]]}))]+ (is (= (:stages parsed) (:stages (wrap-value-literals parsed)))))))+(deftest ^:parallel string-filters-test(testing "string filters like `starts-with` should not parse datetime strings for obvious reasons"(is (= (lib.tu.macros/mbql-query checkins