Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Normalize aggregation references, sort maps and remove all uuids (#67521)

metamben6d1cb34859b5
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index 9f7ea944f6e..80b8a225334 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -226,7 +226,7 @@
(defn- encode-mbql-stage-for-hashing [stage]
(-> stage
common/encode-map-for-hashing
- lib.schema.util/indexed-order-bys-for-stage
+ lib.schema.util/indexed-aggregation-refs-for-stage
;; preserve these keys because we want to hash two identical queries from different source cards
;; differently (see [[metabase.query-processor.middleware.cache-test/multiple-models-e2e-test]]) and this is a
;; reliable way to differentiate them since it gets populated by the QP.
diff --git a/src/metabase/lib/schema/util.cljc b/src/metabase/lib/schema/util.cljc
index 203e4cea0e6..a6e6e8abe27 100644
--- a/src/metabase/lib/schema/util.cljc
+++ b/src/metabase/lib/schema/util.cljc
@@ -129,21 +129,67 @@
(map? x) (dissoc :lib/uuid)))
x))
-(defn indexed-order-bys-for-stage
- "Convert all order-bys in a stage to refer to aggregations by index instead of uuid"
- [{:keys [aggregation order-by] :as stage}]
- (if (and aggregation order-by)
- (let [agg-lookups (->> aggregation
- (map-indexed (fn [i [_type {agg-uuid :lib/uuid}]]
- [agg-uuid i]))
- (into {}))]
- (update stage :order-by (fn [order-bys]
- (mapv (fn [[_dir _opts [order-type agg-opts agg-uuid] :as order-by]]
- (if (= order-type :aggregation)
- (assoc order-by 2 [order-type agg-opts (agg-lookups agg-uuid)])
- order-by))
- order-bys))))
- stage))
+(defn sorted-maps
+ "Recursively convert all maps in `x` to sorted maps using the same comparator
+ as [[metabase.lib.schema.common/unfussy-sorted-map]]. This ensures consistent
+ JSON serialization order regardless of how maps were constructed."
+ [x sorted-map-fn]
+ (perf/postwalk
+ (fn [x]
+ (if (and (map? x) (not (sorted? x)))
+ (into (sorted-map-fn) x)
+ x))
+ x))
+
+(defn- aggregation-ref?
+ "Check if `x` is an aggregation ref clause `[:aggregation opts uuid-string]`."
+ [x]
+ (and (vector? x)
+ (= :aggregation (first x))
+ (>= (count x) 3)
+ (map? (second x))
+ (string? (nth x 2))))
+
+(defn- replace-aggregation-ref-uuids
+ "Walk `form` recursively, replacing aggregation ref UUIDs with indices.
+ Only descends into vectors (MBQL clauses), not maps."
+ [form agg-lookups]
+ (cond
+ (aggregation-ref? form)
+ (update form 2 #(get agg-lookups % %))
+
+ (vector? form)
+ (mapv #(replace-aggregation-ref-uuids % agg-lookups) form)
+
+ :else
+ form))
+
+(def ^:private stage-keys-with-clauses
+ "Stage keys that can contain MBQL clauses with aggregation refs."
+ [:aggregation :breakout :expressions :fields :filters :order-by])
+
+(defn indexed-aggregation-refs-for-stage
+ "Convert all aggregation refs in a stage to refer to aggregations by index instead of uuid.
+ This ensures that semantically identical queries with different UUIDs hash the same."
+ [{:keys [aggregation joins] :as stage}]
+ (if-not aggregation
+ stage
+ (let [agg-lookups (into {}
+ (map-indexed (fn [i [_type {agg-uuid :lib/uuid}]]
+ [agg-uuid i]))
+ aggregation)
+ replace-in-clauses (fn [clauses]
+ (mapv #(replace-aggregation-ref-uuids % agg-lookups) clauses))
+ result (cond-> (reduce (fn [stage k]
+ (cond-> stage
+ (get stage k) (update k replace-in-clauses)))
+ stage
+ stage-keys-with-clauses)
+ ;; Also handle join conditions
+ joins (update :joins (fn [joins]
+ (mapv #(m/update-existing % :conditions replace-in-clauses)
+ joins))))]
+ result)))
(defn pred-matches-form?
"Check if `form` or any of its children forms match `pred`. This function is used for validation; during normal
diff --git a/src/metabase/lib_be/hash.clj b/src/metabase/lib_be/hash.clj
index ac232923931..d3f6e085802 100644
--- a/src/metabase/lib_be/hash.clj
+++ b/src/metabase/lib_be/hash.clj
@@ -7,6 +7,7 @@
[metabase.lib-be.models.transforms :as lib-be.models.transforms]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.util :as lib.schema.util]
[metabase.util.json :as json]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]))
@@ -15,30 +16,35 @@
(mc/encoder
::lib.schema/query
(mtx/transformer
- {:name :for-hashing}
- ;; remove Lib UUID from anywhere that might still have it
- {:encoders {:map (fn [m]
- (cond-> m
- (:lib/uuid m) (dissoc :lib/uuid)))}}
- ;; convert anything that's not a sorted map to one
- {:encoders {:map (fn [m]
- (when (map? m)
- (if (sorted? m)
- m
- (into (lib.schema.common/unfussy-sorted-map) m))))}})))
+ {:name :for-hashing})))
(defn- encoder []
(mr/cached ::encoder ::lib.schema/query encoder*))
(defn- encode [query]
- ((encoder) query))
+ ;; First apply Malli schema-based encoders (e.g. for parameters, aggregation refs).
+ ;; Then use remove-lib-uuids and sorted-maps as generic post-processing steps.
+ ;; The post-processing is necessary because Malli encoders only apply to values
+ ;; that match specific schemas. Values typed as :any (like args to :+ clauses)
+ ;; pass through without encoding, so nested maps with :lib/uuid keys won't have
+ ;; them stripped and won't be converted to sorted maps.
+ (-> query
+ ((encoder))
+ lib.schema.util/remove-lib-uuids
+ (lib.schema.util/sorted-maps lib.schema.common/unfussy-sorted-map)))
+
+(mu/defn query->hash-input :- :map
+ "Normalize and strip `query` to the canonical form used for hashing."
+ [query :- :map]
+ (-> query
+ (cond-> (not= (keyword (:type query)) :internal)
+ (as-> $query (lib-be.models.transforms/normalize-query nil $query {:strict? true})))
+ encode))
(mu/defn query-hash :- bytes?
"Return a 256-bit SHA3 hash of `query` as a key for the cache. (This is returned as a byte array.)"
^bytes [query :- :map]
(-> query
- (cond-> (not= (keyword (:type query)) :internal)
- (as-> $query (lib-be.models.transforms/normalize-query nil $query {:strict? true})))
- encode
+ query->hash-input
json/encode
buddy-hash/sha3-256))
diff --git a/test/metabase/lib_be/hash_test.clj b/test/metabase/lib_be/hash_test.clj
index 2752e277ece..a758a014c97 100644
--- a/test/metabase/lib_be/hash_test.clj
+++ b/test/metabase/lib_be/hash_test.clj
@@ -3,6 +3,7 @@
[buddy.core.codecs :as codecs]
[clojure.test :refer :all]
[metabase.lib-be.hash :as lib-be.hash]
+ [metabase.lib.core :as lib]
[metabase.lib.test-metadata :as meta]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -223,3 +224,187 @@
:database Integer/MAX_VALUE
:stages [{:lib/type :mbql.stage/mbql
:source-table 14303}]})))))
+
+(defn- query-with-aggregation-order-by
+ "Build a query with an aggregation and order-by referencing that aggregation.
+ Each call generates fresh UUIDs."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (as-> $q (lib/order-by $q (lib/aggregation-ref $q 0)))))
+
+(defn- query-with-aggregation-expression
+ "Build a query with aggregations and an expression that references them.
+ Each call generates fresh UUIDs."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (lib/aggregate (lib/sum (meta/field-metadata :venues :price)))
+ (as-> $q
+ ;; Add an aggregation expression that references the first two aggregations
+ (lib/aggregate $q (lib/+ (lib/aggregation-ref $q 0)
+ (lib/aggregation-ref $q 1))))))
+
+(defn- query-with-aggregation-in-breakout
+ "Try to build a query with an aggregation ref in breakout."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (as-> $q (lib/breakout $q (lib/aggregation-ref $q 0)))))
+
+(defn- query-with-aggregation-in-filter
+ "Try to build a query with an aggregation ref in a filter."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (as-> $q (lib/filter $q (lib/> (lib/aggregation-ref $q 0) 5)))))
+
+(defn- query-with-aggregation-in-fields
+ "Try to build a query with an aggregation ref in fields."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (as-> $q (lib/with-fields $q [(lib/aggregation-ref $q 0)]))))
+
+(defn- query-with-aggregation-in-join-condition
+ "Try to build a query with an aggregation ref in a join condition."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (as-> $q
+ (lib/join $q (lib/join-clause (meta/table-metadata :categories)
+ [(lib/= (lib/aggregation-ref $q 0)
+ (meta/field-metadata :categories :id))])))))
+
+(defn- query-with-simple-join
+ "Build a query with a simple join (no aggregation refs)."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/join (lib/join-clause (meta/table-metadata :categories)
+ [(lib/= (meta/field-metadata :venues :category-id)
+ (meta/field-metadata :categories :id))]))))
+
+(defn- query-with-join-subquery-aggregation
+ "Build a query with a join that has a subquery with aggregations and order-by.
+ This tests that aggregation refs in join stages are properly handled by the encoder."
+ []
+ (let [subquery (-> (lib/query meta/metadata-provider (meta/table-metadata :categories))
+ (lib/aggregate (lib/count))
+ (lib/breakout (meta/field-metadata :categories :name))
+ (as-> $q (lib/order-by $q (lib/aggregation-ref $q 0))))]
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/join (lib/join-clause subquery
+ [(lib/= (meta/field-metadata :venues :name)
+ (meta/field-metadata :categories :name))])))))
+
+(deftest ^:parallel mbql5-aggregation-ref-uuid-test
+ (testing "MBQL 5 queries with aggregation references should hash the same regardless of UUID values"
+ ;; In MBQL 5, aggregation refs look like [:aggregation {:lib/uuid "ref-uuid"} "agg-uuid"]
+ ;; where "agg-uuid" is the :lib/uuid of the aggregation being referenced.
+ ;; Two semantically identical queries created at different times will have different UUIDs,
+ ;; but should produce the same hash.
+ (are [query-fn] (= (query-hash-hex (query-fn))
+ (query-hash-hex (query-fn)))
+ query-with-aggregation-order-by
+ query-with-aggregation-expression
+ ;; The following are nonsensical but allowed by the schema
+ query-with-aggregation-in-breakout
+ query-with-aggregation-in-filter
+ query-with-aggregation-in-fields
+ query-with-aggregation-in-join-condition
+ query-with-simple-join
+ query-with-join-subquery-aggregation)
+ (testing "ordering by different aggregations should produce different hashes"
+ (letfn [(query-order-by-aggregation [agg-index]
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (lib/aggregate (lib/sum (meta/field-metadata :venues :price)))
+ (as-> $q (lib/order-by $q (lib/aggregation-ref $q agg-index)))))]
+ (is (not= (query-hash-hex (query-order-by-aggregation 0))
+ (query-hash-hex (query-order-by-aggregation 1))))))))
+
+;; Test cases for :any typed schema positions that can contain :lib/uuid
+;; These test the issue where Malli encoders don't apply to values typed as :any
+
+(defn- query-with-arithmetic-expression
+ "Build a query with an arithmetic expression like (+ field1 field2).
+ The :+ clause schema uses [:+ {:min 2} :any] for its args, so nested
+ field refs with :lib/uuid won't get their options maps encoded."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "sum-fields" (lib/+ (meta/field-metadata :venues :price)
+ (meta/field-metadata :venues :id)))))
+
+(defn- query-with-nested-arithmetic
+ "Build a query with nested arithmetic like (+ (* field1 2) field2).
+ Tests that deeply nested clauses with :lib/uuid are handled."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "nested" (lib/+ (lib/* (meta/field-metadata :venues :price) 2)
+ (meta/field-metadata :venues :id)))))
+
+(defn- query-with-case-expression
+ "Build a query with a case expression containing field refs.
+ Tests that case/if expressions properly strip :lib/uuid from nested clauses."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "case-expr"
+ (lib/case [[(lib/> (meta/field-metadata :venues :price) 5)
+ (meta/field-metadata :venues :id)]]
+ (meta/field-metadata :venues :price)))))
+
+(defn- query-with-coalesce-expression
+ "Build a query with a coalesce expression containing field refs.
+ Tests that coalesce expressions properly strip :lib/uuid from nested clauses."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "coalesce-expr"
+ (lib/coalesce (meta/field-metadata :venues :price)
+ (meta/field-metadata :venues :id)))))
+
+(defn- query-with-concat-expression
+ "Build a query with a concat expression.
+ Tests string expressions with :any typed args."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "concat-expr"
+ (lib/concat (meta/field-metadata :venues :name)
+ (meta/field-metadata :venues :name)))))
+
+(deftest ^:parallel any-typed-schema-positions-test
+ (testing "Expressions using :any typed schema positions should hash consistently"
+ ;; The :+ clause schema uses [:+ {:min 2} :any] for its args, so nested
+ ;; field refs with :lib/uuid need post-processing to strip UUIDs.
+ (are [query-fn] (= (query-hash-hex (query-fn))
+ (query-hash-hex (query-fn)))
+ query-with-arithmetic-expression
+ query-with-nested-arithmetic
+ query-with-case-expression
+ query-with-coalesce-expression
+ query-with-concat-expression)))
+
+(deftest ^:parallel maps-under-any-schema-are-sorted-test
+ (testing "Maps nested under :any typed schema positions should be sorted for consistent hashing"
+ (testing "field ref options inside :+ args (uses [:+ {:min 2} :any])"
+ ;; The :+ clause schema uses [:+ {:min 2} :any] for its args, so the options
+ ;; map inside a :field ref won't be sorted by the Malli encoder.
+ ;; We use array-map to create maps with guaranteed different key orders.
+ (let [query-1 {:lib/type :mbql/query
+ :database 1
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table 1
+ :expressions [[:+ (array-map :lib/uuid "expr-uuid"
+ :lib/expression-name "sum")
+ [:field (array-map :a 1 :b 2) 100]
+ [:field (array-map :c 3 :d 4) 200]]]}]}
+ query-2 {:lib/type :mbql/query
+ :database 1
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table 1
+ :expressions [[:+ (array-map :lib/expression-name "sum"
+ :lib/uuid "expr-uuid")
+ [:field (array-map :b 2 :a 1) 100]
+ [:field (array-map :d 4 :c 3) 200]]]}]}]
+ (is (= (query-hash-hex query-1)
+ (query-hash-hex query-2))
+ "Maps with same content but different key order should hash the same")))))