Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[match] Implement replace-all to avoid infinite &recurs (#82403)

Oleksandr Yakushev86b05b360996
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 3d155df2f07..a079b05f2d1 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -819,6 +819,7 @@
metabase.util.match/match-many hooks.metabase.util.match/match-many
metabase.util.match/matches? hooks.metabase.util.match/matches?
metabase.util.match/replace hooks.metabase.util.match/replace
+ metabase.util.match/replace-all hooks.metabase.util.match/replace-all
metabase.util.match/replace-in hooks.metabase.util.match/replace-in
metabase.util/case-enum hooks.metabase.util/case-enum
metabase.util/format-color hooks.metabase.util/format-color
diff --git a/.clj-kondo/src/hooks/metabase/util/match.clj b/.clj-kondo/src/hooks/metabase/util/match.clj
index 272e297db0a..f589742a09f 100644
--- a/.clj-kondo/src/hooks/metabase/util/match.clj
+++ b/.clj-kondo/src/hooks/metabase/util/match.clj
@@ -145,6 +145,7 @@
(def match-many match-one)
(def replace match-one)
+(def replace-all match-one)
(defn replace-in [context]
;; Everything is the same as in match-one, just drop the middle argument.
diff --git a/.cljfmt.edn b/.cljfmt.edn
index 2de0c4b0f5e..5415436824f 100644
--- a/.cljfmt.edn
+++ b/.cljfmt.edn
@@ -36,6 +36,7 @@
metabase.driver-api.core/match-many :all
metabase.driver-api.core/match-one :all
metabase.driver-api.core/replace :all
+ metabase.driver-api.core/replace-all :all
metabase.driver-api.core/replace-in :all
metabase.lib.drill-thru.test-util/test-drill-variants-with-merged-args :all
metabase.lib.drill-thru.test-util/test-returns-drill :all
@@ -206,6 +207,7 @@
metabase.util.match/match-one [[:inner 0]]
metabase.util.match/match-many [[:inner 0]]
metabase.util.match/replace [[:inner 0]]
+ metabase.util.match/replace-all [[:inner 0]]
metabase.util.match/replace-in [[:inner 0]]
metabase.models.interface/define-batched-hydration-method [[:inner 0]]
metabase.models.interface/define-simple-hydration-method [[:inner 0]]
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.clj
index 0662fe8c3a5..41b6c061740 100644
--- a/src/metabase/agent_lib/representations/repair.clj
+++ b/src/metabase/agent_lib/representations/repair.clj
@@ -73,10 +73,10 @@
(defn- ensure-clause-options*
"Fix a clause-like vector missing options map at position 2 in nested clause-like vectors."
[form]
- (match/replace form
+ (match/replace-all form
(:and [op] (_ :guard clause-like?)) [op {}]
- (:and [op nil & _] (_ :guard clause-like?)) (&recur (assoc &match 1 {}))
- (:and [op (non-map :guard (not (map? non-map))) & args] (_ :guard clause-like?)) (&recur (into [op {} non-map] args))))
+ (:and [op nil & _] (_ :guard clause-like?)) (assoc &match 1 {})
+ (:and [op (non-map :guard (not (map? non-map))) & args] (_ :guard clause-like?)) (into [op {} non-map] args)))
;;; ============================================================
;;; Pass 1.7 -- unwrap nested `[field opts [field inner-opts target]]` clauses.
@@ -98,9 +98,9 @@
(defn- unwrap-nested-field-clauses*
[form]
- (match/replace form
+ (match/replace-all form
["field" (outer :guard map?) ["field" (inner :guard map?) target]]
- (&recur ["field" (into inner outer) target])))
+ ["field" (into inner outer) target]))
;;; ============================================================
;;; Pass 1.75 -- strip stray double-quotes from portable-FK field references.
@@ -139,9 +139,9 @@
(defn- dequote-field-targets*
[form]
- (match/replace form
+ (match/replace-all form
["field" (opts :guard map?) (target :guard vector?)]
- ["field" opts (&recur (dequote-field-target target))]))
+ ["field" opts (dequote-field-target target)]))
;;; ============================================================
;;; Pass 1.81 -- canonicalise common operator-name aliases.
@@ -203,13 +203,13 @@
(defn- rewrite-operator-name-aliases*
[form]
- (match/replace form
+ (match/replace-all form
[(op :guard (and (string? op)
(let [lower (u/lower-case-en op)]
(and (contains? operator-name-aliases lower)
(not= op (operator-name-aliases lower))))))
(_ :guard map?) & _]
- (&recur (assoc &match 0 (operator-name-aliases (u/lower-case-en op))))))
+ (assoc &match 0 (operator-name-aliases (u/lower-case-en op)))))
;;; ============================================================
;;; Pass 1.8 -- canonicalise temporal-bucket extraction aliases.
@@ -236,9 +236,9 @@
(defn- rewrite-temporal-bucket-aliases*
[form]
- (match/replace form
+ (match/replace-all form
[(op :guard (and (string? op) (temporal-bucket-extraction-aliases op))) (_ :guard map?) & _]
- (&recur (assoc &match 0 (temporal-bucket-extraction-aliases (u/lower-case-en op))))))
+ (assoc &match 0 (temporal-bucket-extraction-aliases (u/lower-case-en op)))))
;;; ============================================================
;;; Pass 1.815 -- drop unsupported `get-day-of-week` week-mode arguments.
@@ -308,9 +308,9 @@
(defn- rewrite-direction-aliases*
[form]
- (match/replace form
+ (match/replace-all form
[(head :guard direction-clause-head?) (_ :guard map?) _]
- (&recur (assoc &match 0 (get direction-aliases (u/lower-case-en head))))))
+ (assoc &match 0 (get direction-aliases (u/lower-case-en head)))))
;;; ============================================================
;;; Pass 1.87 -- rewrite known misspelled `lib/type` markers to their canonical value.
@@ -326,9 +326,9 @@
(defn- rewrite-lib-type-aliases*
[form]
- (match/replace form
+ (match/replace-all form
{"lib/type" (canonical :guard lib-type-aliases)}
- (&recur (assoc &match "lib/type" (lib-type-aliases canonical)))))
+ (assoc &match "lib/type" (lib-type-aliases canonical))))
;;; ============================================================
;;; Pass 1.88 -- merge a trailing extra options-map into the position-1 options.
@@ -424,12 +424,9 @@
(defn- merge-trailing-options*
[form]
- (match/replace form
+ (match/replace-all form
(_ :guard needs-trailing-options-merge?)
- ;; &recur: dropping the trailing element changes the count, so the guard no longer fires
- ;; on the result -- this just resumes descent into the clause's own args, where a nested
- ;; clause might need the same fix.
- (&recur (merge-trailing-options &match))))
+ (merge-trailing-options &match)))
;;; ============================================================
;;; Pass 1.89 -- merge a trailing options-map into position-1 on N-ary string-search filters
@@ -581,12 +578,12 @@
_ nil))
(defn- normalise-case-clauses* [form]
- (match/replace form
+ (match/replace-all form
[(head :guard #{"case" "if"}) (opts :guard map?) & args]
(let [canonical-args (if-let [[branches default] (classify-case-args args)]
(canonical-case-args branches default)
args)]
- (into [head opts] (map #(&recur % nil)) canonical-args))))
+ (into [head opts] canonical-args))))
;;; ============================================================
;;; Pass 1.82 -- normalise filter clauses where the LLM passed a values-list as a single
@@ -684,14 +681,13 @@
otherwise mis-identify a 2-element values-list (e.g. `[\"alice\" \"bob\"]`) as a bare
clause and corrupt it by inserting `{}` between the two scalars."
[form]
- (match/replace form
+ (match/replace-all form
(_ :guard in-not-in-values-list-clause)
(let [[head opts lhs values] (in-not-in-values-list-clause &match)]
- (&recur (splat-in-values-clause head opts lhs values)))
-
+ (splat-in-values-clause head opts lhs values))
(_ :guard eq-values-list-clause)
(let [[head opts lhs values] (eq-values-list-clause &match)]
- (&recur (splat-in-values-clause (=->in-head head) opts lhs values)))))
+ (splat-in-values-clause (=->in-head head) opts lhs values))))
;;; ============================================================
;;; Pass 1.83 -- unwrap boolean wrapper clauses.
@@ -734,8 +730,8 @@
(defn- unwrap-boolean-wrappers*
[form]
- (match/replace form
- (_ :guard boolean-wrapper-clause?) (&recur (unwrap-boolean-wrapper &match))))
+ (match/replace-all form
+ (_ :guard boolean-wrapper-clause?) (unwrap-boolean-wrapper &match)))
;;; ============================================================
;;; Pass 1.87 -- swap out-of-order literal bounds in `between` clauses.
@@ -799,11 +795,9 @@
(defn- swap-between-bounds*
[form]
- (match/replace form
+ (match/replace-all form
(_ :guard swappable-between-clause?)
- ;; &recur: after the swap the bounds compare in order, so `swappable-between-clause?` no
- ;; longer fires on the result -- this just resumes descent into the clause.
- (&recur (-> &match (assoc 3 (nth &match 4)) (assoc 4 (nth &match 3))))))
+ (-> &match (assoc 3 (nth &match 4)) (assoc 4 (nth &match 3)))))
;;; ============================================================
;;; Temporal-literal helpers, shared by the passes below.
@@ -1385,10 +1379,10 @@
(defn- normalise-fields-shape*
[form]
- (match/replace form
+ (match/replace-all form
{"fields" (:and single-clause
[(_ :guard string?) (_ :guard map?) & _])}
- (&recur (assoc &match "fields" [single-clause]))))
+ (assoc &match "fields" [single-clause])))
;;; ============================================================
;;; Pass 1.5 -- normalize `expressions:` shape (map -> sequential; stamp `lib/expression-name`)
@@ -1428,14 +1422,14 @@
into the canonical sequential shape with `lib/expression-name` stamped from the map key.
Idempotent: sequential input passes through unchanged."
[form]
- (match/replace form
+ (match/replace-all form
{"expressions" (exprs :guard map?)}
- (&recur (assoc &match "expressions"
- (into []
- (keep (fn [[expr-name clause]]
- (when (expression-clause? clause)
- (stamp-expression-name clause expr-name))))
- exprs)))))
+ (assoc &match "expressions"
+ (into []
+ (keep (fn [[expr-name clause]]
+ (when (expression-clause? clause)
+ (stamp-expression-name clause expr-name))))
+ exprs))))
;;; ============================================================
;;; Pass 1.9 -- stamp top-level `database:` from the first stage
@@ -1518,11 +1512,11 @@
(or (top-level-query-map? m) (join-like-map? m) (stage-like-map? m))))
(defn- ensure-lib-types* [form]
- (match/replace form
+ (match/replace-all form
(_ :guard needs-lib-type-marker?)
- (&recur (assoc &match "lib/type" (cond (top-level-query-map? &match) "mbql/query"
- (join-like-map? &match) "mbql/join"
- (stage-like-map? &match) "mbql.stage/mbql")))))
+ (assoc &match "lib/type" (cond (top-level-query-map? &match) "mbql/query"
+ (join-like-map? &match) "mbql/join"
+ (stage-like-map? &match) "mbql.stage/mbql"))))
;;; ============================================================
;;; Pass 2.7 -- rewrite inline aggregation expressions in `order-by` to aggregation refs
@@ -1558,9 +1552,9 @@
Uses the same `clause-like?` predicate as the options-insertion pass, so FK-paths and
non-clause vectors are left alone."
[form]
- (match/replace form
+ (match/replace-all form
(:and (_ :guard clause-like?) [op (_ :guard map?) & args])
- (&recur (into [op] args))))
+ (into [op] args)))
(defn- ensure-aggregation-uuid
"Return a tuple `[stamped-aggregation uuid]`. If the aggregation already has a `lib/uuid` in
diff --git a/src/metabase/lib/filter/desugar.cljc b/src/metabase/lib/filter/desugar.cljc
index 0a3654b052e..f660cd5c34b 100644
--- a/src/metabase/lib/filter/desugar.cljc
+++ b/src/metabase/lib/filter/desugar.cljc
@@ -322,10 +322,10 @@
the query itself. Filtering should be done based on the number, rather than the name."
[time-config :- ::lib.schema.common/time-config
expression :- ::clause]
- (match/replace expression
- [:month-name opts expr] (&recur (temporal-case-expression time-config expr opts :month-of-year 12))
- [:quarter-name opts expr] (&recur (temporal-case-expression time-config expr opts :quarter-of-year 4))
- [:day-name opts expr] (&recur (temporal-case-expression time-config expr opts :day-of-week 7))))
+ (match/replace-all expression
+ [:month-name opts expr] (temporal-case-expression time-config expr opts :month-of-year 12)
+ [:quarter-name opts expr] (temporal-case-expression time-config expr opts :quarter-of-year 4)
+ [:day-name opts expr] (temporal-case-expression time-config expr opts :day-of-week 7)))
(mu/defn- desugar-expression :- ::clause
"Rewrite various 'syntactic sugar' expressions like `:/` with more than two args into something simpler for drivers
diff --git a/src/metabase/lib/filter/desugar/jvm.clj b/src/metabase/lib/filter/desugar/jvm.clj
index d2fac076d39..6e464618a4e 100644
--- a/src/metabase/lib/filter/desugar/jvm.clj
+++ b/src/metabase/lib/filter/desugar/jvm.clj
@@ -140,16 +140,13 @@
(mu/defn desugar-host-and-domain :- ::lib.schema.mbql-clause/clause
"Unwrap host and domain."
[expression :- ::lib.schema.mbql-clause/clause]
- (match/replace expression
+ (match/replace-all expression
[:host opts expr]
;; TODO (Cam 8/18/25) -- seems weird that we don't support Regex literals in the regex clauses and have to call (str ...)
- (&recur [:regex-match-first opts expr (str host-regex)])
-
+ [:regex-match-first opts expr (str host-regex)]
[:domain opts expr]
- (&recur [:regex-match-first opts expr (str domain-regex)])
-
+ [:regex-match-first opts expr (str domain-regex)]
[:subdomain opts expr]
- (&recur [:regex-match-first opts expr (str subdomain-regex)])
-
+ [:regex-match-first opts expr (str subdomain-regex)]
[:path opts expr]
- (&recur [:regex-match-first opts expr (str path-regex)])))
+ [:regex-match-first opts expr (str path-regex)]))
diff --git a/src/metabase/util/match.clj b/src/metabase/util/match.clj
index 37038c753cc..f1cb69c5ce2 100644
--- a/src/metabase/util/match.clj
+++ b/src/metabase/util/match.clj
@@ -427,10 +427,23 @@
~form
~(when contains-&parents? []))))
+(defmacro replace-all
+ "Like [[replace]], but walks the `form` from the inside outwards, thus applying the transform multiple times (in a
+ \"postwalk\" fashion). Doesn't support `&parents` anaphora yet. NB: unlike [[replace]], this macro walks map keys
+ individually. Make sure that the match body doesn't trigger on map keys. "
+ [form & clauses]
+ (let [replace-fn-symb (gensym "replace-")
+ contains-&parents? (contains-symbol? clauses '&parents)]
+ (when contains-&parents?
+ (throw (ex-info "&parents is not supported by replace-all." {})))
+ `(perf/postwalk (fn ~replace-fn-symb [~'&match]
+ (match-one ~'&match
+ ~@clauses
+ ~'_ ~'&match))
+ ~form)))
+
(defmacro replace-in
"Like `replace`, but only replaces things in the part of `x` in the keypath `ks` (i.e. the way to `update-in` works.)"
{:style/indent :defn}
[x ks & patterns-and-results]
`(metabase.util.match.impl/update-in-unless-empty ~x ~ks (fn [x#] (replace x# ~@patterns-and-results))))
-
-;; TODO - it would be useful to have something like a `replace-all` function as well
diff --git a/test/metabase/util/match_test.cljc b/test/metabase/util/match_test.cljc
index 2b74309cfa4..b89bf95b89f 100644
--- a/test/metabase/util/match_test.cljc
+++ b/test/metabase/util/match_test.cljc
@@ -415,7 +415,7 @@
[a b c] (when (> a 1)
(+ a b c)))))))
-(t/deftest ^:parallel parents-tests
+(t/deftest ^:parallel parents-test
(t/is (= [:foo :bar :baz :qux]
(match/match-one [:foo [:bar [:baz {:qux 42}]]]
42 &parents)))
@@ -423,3 +423,16 @@
[:foo :bar :baz :corge]]
(match/match-many [:foo [:bar [:baz {:qux 42 :corge 42}]]]
42 &parents))))
+
+(t/deftest ^:parallel replace-all-test
+ (t/is (= [:field {:updated true} [:field {:updated true} [:field {:updated true} 1]]]
+ (match/replace-all [:field {} [:field {} [:field {} 1]]]
+ [:field opts arg] [:field (assoc opts :updated true) arg])))
+ (t/testing "applies outward to already updated values"
+ (t/is (= :needle
+ (match/replace-all [[[[[[[[[[:needle]]]]]]]]]]
+ [needle] needle))))
+ (t/testing "careful - applies to keys too"
+ (t/is (= [:field {"key1_updated" 1} [:field {"key2_updated" 2} nil]]
+ (match/replace-all [:field {"key1" 1} [:field {"key2" 2} nil]]
+ (_ :guard string?) (str &match "_updated"))))))