The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Normalize aggregation references, sort maps and remove all uuids (#67521)
metamben6d1cb34859b5
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 9f7ea944f6e..80b8a225334 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -226,7 +226,7 @@(defn- encode-mbql-stage-for-hashing [stage](-> stagecommon/encode-map-for-hashing- lib.schema.util/indexed-order-bys-for-stage+ lib.schema.util/indexed-aggregation-refs-for-stage;; preserve these keys because we want to hash two identical queries from different source cards;; differently (see [[metabase.query-processor.middleware.cache-test/multiple-models-e2e-test]]) and this is a;; reliable way to differentiate them since it gets populated by the QP.diff --git a/src/metabase/lib/schema/util.cljc b/src/metabase/lib/schema/util.cljcindex 203e4cea0e6..a6e6e8abe27 100644--- a/src/metabase/lib/schema/util.cljc+++ b/src/metabase/lib/schema/util.cljc@@ -129,21 +129,67 @@(map? x) (dissoc :lib/uuid)))x))-(defn indexed-order-bys-for-stage- "Convert all order-bys in a stage to refer to aggregations by index instead of uuid"- [{:keys [aggregation order-by] :as stage}]- (if (and aggregation order-by)- (let [agg-lookups (->> aggregation- (map-indexed (fn [i [_type {agg-uuid :lib/uuid}]]- [agg-uuid i]))- (into {}))]- (update stage :order-by (fn [order-bys]- (mapv (fn [[_dir _opts [order-type agg-opts agg-uuid] :as order-by]]- (if (= order-type :aggregation)- (assoc order-by 2 [order-type agg-opts (agg-lookups agg-uuid)])- order-by))- order-bys))))- stage))+(defn sorted-maps+ "Recursively convert all maps in `x` to sorted maps using the same comparator+ as [[metabase.lib.schema.common/unfussy-sorted-map]]. This ensures consistent+ JSON serialization order regardless of how maps were constructed."+ [x sorted-map-fn]+ (perf/postwalk+ (fn [x]+ (if (and (map? x) (not (sorted? x)))+ (into (sorted-map-fn) x)+ x))+ x))++(defn- aggregation-ref?+ "Check if `x` is an aggregation ref clause `[:aggregation opts uuid-string]`."+ [x]+ (and (vector? x)+ (= :aggregation (first x))+ (>= (count x) 3)+ (map? (second x))+ (string? (nth x 2))))++(defn- replace-aggregation-ref-uuids+ "Walk `form` recursively, replacing aggregation ref UUIDs with indices.+ Only descends into vectors (MBQL clauses), not maps."+ [form agg-lookups]+ (cond+ (aggregation-ref? form)+ (update form 2 #(get agg-lookups % %))++ (vector? form)+ (mapv #(replace-aggregation-ref-uuids % agg-lookups) form)++ :else+ form))++(def ^:private stage-keys-with-clauses+ "Stage keys that can contain MBQL clauses with aggregation refs."+ [:aggregation :breakout :expressions :fields :filters :order-by])++(defn indexed-aggregation-refs-for-stage+ "Convert all aggregation refs in a stage to refer to aggregations by index instead of uuid.+ This ensures that semantically identical queries with different UUIDs hash the same."+ [{:keys [aggregation joins] :as stage}]+ (if-not aggregation+ stage+ (let [agg-lookups (into {}+ (map-indexed (fn [i [_type {agg-uuid :lib/uuid}]]+ [agg-uuid i]))+ aggregation)+ replace-in-clauses (fn [clauses]+ (mapv #(replace-aggregation-ref-uuids % agg-lookups) clauses))+ result (cond-> (reduce (fn [stage k]+ (cond-> stage+ (get stage k) (update k replace-in-clauses)))+ stage+ stage-keys-with-clauses)+ ;; Also handle join conditions+ joins (update :joins (fn [joins]+ (mapv #(m/update-existing % :conditions replace-in-clauses)+ joins))))]+ result)))(defn pred-matches-form?"Check if `form` or any of its children forms match `pred`. This function is used for validation; during normaldiff --git a/src/metabase/lib_be/hash.clj b/src/metabase/lib_be/hash.cljindex ac232923931..d3f6e085802 100644--- a/src/metabase/lib_be/hash.clj+++ b/src/metabase/lib_be/hash.clj@@ -7,6 +7,7 @@[metabase.lib-be.models.transforms :as lib-be.models.transforms][metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.util :as lib.schema.util][metabase.util.json :as json][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]))@@ -15,30 +16,35 @@(mc/encoder::lib.schema/query(mtx/transformer- {:name :for-hashing}- ;; remove Lib UUID from anywhere that might still have it- {:encoders {:map (fn [m]- (cond-> m- (:lib/uuid m) (dissoc :lib/uuid)))}}- ;; convert anything that's not a sorted map to one- {:encoders {:map (fn [m]- (when (map? m)- (if (sorted? m)- m- (into (lib.schema.common/unfussy-sorted-map) m))))}})))+ {:name :for-hashing})))(defn- encoder [](mr/cached ::encoder ::lib.schema/query encoder*))(defn- encode [query]- ((encoder) query))+ ;; First apply Malli schema-based encoders (e.g. for parameters, aggregation refs).+ ;; Then use remove-lib-uuids and sorted-maps as generic post-processing steps.+ ;; The post-processing is necessary because Malli encoders only apply to values+ ;; that match specific schemas. Values typed as :any (like args to :+ clauses)+ ;; pass through without encoding, so nested maps with :lib/uuid keys won't have+ ;; them stripped and won't be converted to sorted maps.+ (-> query+ ((encoder))+ lib.schema.util/remove-lib-uuids+ (lib.schema.util/sorted-maps lib.schema.common/unfussy-sorted-map)))++(mu/defn query->hash-input :- :map+ "Normalize and strip `query` to the canonical form used for hashing."+ [query :- :map]+ (-> query+ (cond-> (not= (keyword (:type query)) :internal)+ (as-> $query (lib-be.models.transforms/normalize-query nil $query {:strict? true})))+ encode))(mu/defn query-hash :- bytes?"Return a 256-bit SHA3 hash of `query` as a key for the cache. (This is returned as a byte array.)"^bytes [query :- :map](-> query- (cond-> (not= (keyword (:type query)) :internal)- (as-> $query (lib-be.models.transforms/normalize-query nil $query {:strict? true})))- encode+ query->hash-inputjson/encodebuddy-hash/sha3-256))diff --git a/test/metabase/lib_be/hash_test.clj b/test/metabase/lib_be/hash_test.cljindex 2752e277ece..a758a014c97 100644--- a/test/metabase/lib_be/hash_test.clj+++ b/test/metabase/lib_be/hash_test.clj@@ -3,6 +3,7 @@[buddy.core.codecs :as codecs][clojure.test :refer :all][metabase.lib-be.hash :as lib-be.hash]+ [metabase.lib.core :as lib][metabase.lib.test-metadata :as meta][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -223,3 +224,187 @@:database Integer/MAX_VALUE:stages [{:lib/type :mbql.stage/mbql:source-table 14303}]})))))++(defn- query-with-aggregation-order-by+ "Build a query with an aggregation and order-by referencing that aggregation.+ Each call generates fresh UUIDs."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (as-> $q (lib/order-by $q (lib/aggregation-ref $q 0)))))++(defn- query-with-aggregation-expression+ "Build a query with aggregations and an expression that references them.+ Each call generates fresh UUIDs."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (lib/aggregate (lib/sum (meta/field-metadata :venues :price)))+ (as-> $q+ ;; Add an aggregation expression that references the first two aggregations+ (lib/aggregate $q (lib/+ (lib/aggregation-ref $q 0)+ (lib/aggregation-ref $q 1))))))++(defn- query-with-aggregation-in-breakout+ "Try to build a query with an aggregation ref in breakout."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (as-> $q (lib/breakout $q (lib/aggregation-ref $q 0)))))++(defn- query-with-aggregation-in-filter+ "Try to build a query with an aggregation ref in a filter."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (as-> $q (lib/filter $q (lib/> (lib/aggregation-ref $q 0) 5)))))++(defn- query-with-aggregation-in-fields+ "Try to build a query with an aggregation ref in fields."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (as-> $q (lib/with-fields $q [(lib/aggregation-ref $q 0)]))))++(defn- query-with-aggregation-in-join-condition+ "Try to build a query with an aggregation ref in a join condition."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (as-> $q+ (lib/join $q (lib/join-clause (meta/table-metadata :categories)+ [(lib/= (lib/aggregation-ref $q 0)+ (meta/field-metadata :categories :id))])))))++(defn- query-with-simple-join+ "Build a query with a simple join (no aggregation refs)."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/join (lib/join-clause (meta/table-metadata :categories)+ [(lib/= (meta/field-metadata :venues :category-id)+ (meta/field-metadata :categories :id))]))))++(defn- query-with-join-subquery-aggregation+ "Build a query with a join that has a subquery with aggregations and order-by.+ This tests that aggregation refs in join stages are properly handled by the encoder."+ []+ (let [subquery (-> (lib/query meta/metadata-provider (meta/table-metadata :categories))+ (lib/aggregate (lib/count))+ (lib/breakout (meta/field-metadata :categories :name))+ (as-> $q (lib/order-by $q (lib/aggregation-ref $q 0))))]+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/join (lib/join-clause subquery+ [(lib/= (meta/field-metadata :venues :name)+ (meta/field-metadata :categories :name))])))))++(deftest ^:parallel mbql5-aggregation-ref-uuid-test+ (testing "MBQL 5 queries with aggregation references should hash the same regardless of UUID values"+ ;; In MBQL 5, aggregation refs look like [:aggregation {:lib/uuid "ref-uuid"} "agg-uuid"]+ ;; where "agg-uuid" is the :lib/uuid of the aggregation being referenced.+ ;; Two semantically identical queries created at different times will have different UUIDs,+ ;; but should produce the same hash.+ (are [query-fn] (= (query-hash-hex (query-fn))+ (query-hash-hex (query-fn)))+ query-with-aggregation-order-by+ query-with-aggregation-expression+ ;; The following are nonsensical but allowed by the schema+ query-with-aggregation-in-breakout+ query-with-aggregation-in-filter+ query-with-aggregation-in-fields+ query-with-aggregation-in-join-condition+ query-with-simple-join+ query-with-join-subquery-aggregation)+ (testing "ordering by different aggregations should produce different hashes"+ (letfn [(query-order-by-aggregation [agg-index]+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (lib/aggregate (lib/sum (meta/field-metadata :venues :price)))+ (as-> $q (lib/order-by $q (lib/aggregation-ref $q agg-index)))))]+ (is (not= (query-hash-hex (query-order-by-aggregation 0))+ (query-hash-hex (query-order-by-aggregation 1))))))))++;; Test cases for :any typed schema positions that can contain :lib/uuid+;; These test the issue where Malli encoders don't apply to values typed as :any++(defn- query-with-arithmetic-expression+ "Build a query with an arithmetic expression like (+ field1 field2).+ The :+ clause schema uses [:+ {:min 2} :any] for its args, so nested+ field refs with :lib/uuid won't get their options maps encoded."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "sum-fields" (lib/+ (meta/field-metadata :venues :price)+ (meta/field-metadata :venues :id)))))++(defn- query-with-nested-arithmetic+ "Build a query with nested arithmetic like (+ (* field1 2) field2).+ Tests that deeply nested clauses with :lib/uuid are handled."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "nested" (lib/+ (lib/* (meta/field-metadata :venues :price) 2)+ (meta/field-metadata :venues :id)))))++(defn- query-with-case-expression+ "Build a query with a case expression containing field refs.+ Tests that case/if expressions properly strip :lib/uuid from nested clauses."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "case-expr"+ (lib/case [[(lib/> (meta/field-metadata :venues :price) 5)+ (meta/field-metadata :venues :id)]]+ (meta/field-metadata :venues :price)))))++(defn- query-with-coalesce-expression+ "Build a query with a coalesce expression containing field refs.+ Tests that coalesce expressions properly strip :lib/uuid from nested clauses."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "coalesce-expr"+ (lib/coalesce (meta/field-metadata :venues :price)+ (meta/field-metadata :venues :id)))))++(defn- query-with-concat-expression+ "Build a query with a concat expression.+ Tests string expressions with :any typed args."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "concat-expr"+ (lib/concat (meta/field-metadata :venues :name)+ (meta/field-metadata :venues :name)))))++(deftest ^:parallel any-typed-schema-positions-test+ (testing "Expressions using :any typed schema positions should hash consistently"+ ;; The :+ clause schema uses [:+ {:min 2} :any] for its args, so nested+ ;; field refs with :lib/uuid need post-processing to strip UUIDs.+ (are [query-fn] (= (query-hash-hex (query-fn))+ (query-hash-hex (query-fn)))+ query-with-arithmetic-expression+ query-with-nested-arithmetic+ query-with-case-expression+ query-with-coalesce-expression+ query-with-concat-expression)))++(deftest ^:parallel maps-under-any-schema-are-sorted-test+ (testing "Maps nested under :any typed schema positions should be sorted for consistent hashing"+ (testing "field ref options inside :+ args (uses [:+ {:min 2} :any])"+ ;; The :+ clause schema uses [:+ {:min 2} :any] for its args, so the options+ ;; map inside a :field ref won't be sorted by the Malli encoder.+ ;; We use array-map to create maps with guaranteed different key orders.+ (let [query-1 {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 1+ :expressions [[:+ (array-map :lib/uuid "expr-uuid"+ :lib/expression-name "sum")+ [:field (array-map :a 1 :b 2) 100]+ [:field (array-map :c 3 :d 4) 200]]]}]}+ query-2 {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 1+ :expressions [[:+ (array-map :lib/expression-name "sum"+ :lib/uuid "expr-uuid")+ [:field (array-map :b 2 :a 1) 100]+ [:field (array-map :d 4 :c 3) 200]]]}]}]+ (is (= (query-hash-hex query-1)+ (query-hash-hex query-2))+ "Maps with same content but different key order should hash the same")))))