Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix query normalization issues in v57 (#66722)

Cam Sauld2d11e9ed02e
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index dbf34e827e7..e1a2714ee6c 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -709,6 +709,7 @@
metabase.lib.field.resolution
metabase.lib.filter
metabase.lib.filter.desugar.jvm ; only so [[metabase.legacy-mbql.jvm-util]] can use defined regexes
+ metabase.lib.hierarchy
metabase.lib.init
metabase.lib.join.util
metabase.lib.metadata
diff --git a/resources/migrations/057_update_migrations.yaml b/resources/migrations/057_update_migrations.yaml
index 3dae882b2b5..046ddf8e5d9 100644
--- a/resources/migrations/057_update_migrations.yaml
+++ b/resources/migrations/057_update_migrations.yaml
@@ -2604,6 +2604,18 @@ databaseChangeLog:
path: instance_analytics_views/content/v5/h2-content.sql
relativeToChangelogFile: true
+ - changeSet:
+ id: v57.2025-12-09T12:45:01
+ author: dpsutton
+ comment: >-
+ Copy legacy MBQL queries from report_card.legacy_query to report_card.dataset_query where the parsed query is erroneously '{}'
+ changes:
+ - sql: >-
+ UPDATE report_card
+ SET dataset_query = legacy_query
+ where dataset_query = '{}' and legacy_query is not null;
+ rollback: # nothing to undo
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/legacy_mbql/normalize.cljc b/src/metabase/legacy_mbql/normalize.cljc
index d002495c297..c1433783b75 100644
--- a/src/metabase/legacy_mbql/normalize.cljc
+++ b/src/metabase/legacy_mbql/normalize.cljc
@@ -2,20 +2,20 @@
"Normalize MBQL 1-4 to MBQL 4."
(:require
#?@(:clj
- ([potemkin :as p]))
+ ([metabase.legacy-mbql.schema.helpers :as helpers]
+ [potemkin :as p]))
[metabase.legacy-mbql.schema :as mbql.s]
- [metabase.legacy-mbql.schema.helpers :as helpers]
[metabase.lib.normalize :as lib.normalize]
[metabase.util.malli :as mu]))
+#?(:clj (comment helpers/keep-me))
+
;; this is only for the sake of a few random namespaces that use this still
#?(:clj
(p/import-vars [helpers is-clause?]))
(defn- infer-schema [x]
- (or (when (sequential? x)
- (when-let [tag (helpers/effective-clause-tag x)]
- (keyword "metabase.legacy-mbql.schema" (name tag))))
+ (or (mbql.s/infer-mbql-clause-schema x)
::mbql.s/Query))
(defn normalize
diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljc
index 14934b1c23e..e3ac1af2f41 100644
--- a/src/metabase/legacy_mbql/schema.cljc
+++ b/src/metabase/legacy_mbql/schema.cljc
@@ -17,7 +17,10 @@
[malli.core :as mc]
[metabase.legacy-mbql.schema.helpers :as helpers :refer [is-clause?]]
[metabase.legacy-mbql.schema.macros :refer [defclause defclause* one-of]]
+ [metabase.lib.hierarchy :as lib.hierarchy]
+ [metabase.lib.normalize :as lib.normalize]
[metabase.lib.schema.actions :as lib.schema.actions]
+ [metabase.lib.schema.aggregation :as lib.schema.aggregation]
[metabase.lib.schema.binning :as lib.schema.binning]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.constraints :as lib.schema.constraints]
@@ -42,22 +45,23 @@
[metabase.util.performance :as perf :refer [every? select-keys #?(:clj doseq) some mapv update-keys empty? not-empty]]
[metabase.util.time :as u.time]))
-;; A NOTE ABOUT METADATA:
-;;
-;; Clauses below are marked with the following tags for documentation purposes:
-;;
-;; * Clauses marked `^:sugar` are syntactic sugar primarily intended to make generating queries easier on the
-;; frontend. These clauses are automatically rewritten as simpler clauses by the `desugar` or `expand-macros`
-;; middleware. Thus driver implementations do not need to handle these clauses.
-;;
-;; * Clauses marked `^:internal` are automatically generated by `wrap-value-literals` or other middleware from values
-;; passed in. They are not intended to be used by the frontend when generating a query. These add certain
-;; information that simplify driver implementations. When writing MBQL queries yourself you should pretend these
-;; clauses don't exist.
-;;
-;; * Clauses marked ` require a certain set of features to be used. At some date in
-;; the future we will likely add middleware that uses this metadata to automatically validate that a driver has the
-;; features needed to run the query in question.
+(defn infer-mbql-clause-schema
+ "Infer the schema for something that is (probably) an MBQL clause."
+ [x]
+ (when (sequential? x)
+ (when-let [tag (helpers/effective-clause-tag x)]
+ (keyword "metabase.legacy-mbql.schema" (name tag)))))
+
+(defn- normalize-mbql-clause [x]
+ (when-let [schema (infer-mbql-clause-schema x)]
+ (lib.normalize/normalize schema x)))
+
+(mr/def ::AnyMBQLClause
+ "Schema for ANY valid MBQL clause"
+ [:fn
+ {:error/message "Valid MBQL clause"
+ :decode/normalize normalize-mbql-clause}
+ helpers/mbql-clause?])
(mr/def ::options-style
"For convenience of converting back and forth between MBQL 5, from now on we can record info about the options style
@@ -1292,16 +1296,27 @@
;;
;; [:+ [:sum [:field 10 nil]] [:sum [:field 20 nil]]]
+(defn- aggregation-expression?
+ "A clause is a valid aggregation if it is an aggregation clause, or it is an expression that transitively contains
+ a single aggregation clause.
+
+ (This is mostly copied from [[metabase.lib.schema.aggregation/aggregation-expression?]]"
+ [x]
+ (when (vector? x)
+ (when-let [[tag & args] x]
+ (or (lib.hierarchy/isa? tag ::lib.schema.aggregation/aggregation-clause-tag)
+ ;; Case has the following shape [:case [[cond expr]...] default-expr?]
+ (if (= :case tag)
+ (or (some aggregation-expression? (ffirst args))
+ (some aggregation-expression? (fnext args)))
+ (some aggregation-expression? args))))))
+
(mr/def ::UnnamedAggregation
- [:multi
- {:error/message "unnamed aggregation clause or numeric expression"
- :dispatch (fn [x]
- (if (is-clause? numeric-functions x)
- :numeric-expression
- :else))}
- [:numeric-expression [:ref ::NumericExpression]]
- [:else (one-of aggregation avg cum-sum distinct distinct-where stddev sum min max metric share count-where
- sum-where case if median percentile var cum-count count offset)]])
+ [:and
+ [:ref ::AnyMBQLClause]
+ [:fn
+ {:error/message "Aggregations should contain at least one aggregation function."}
+ aggregation-expression?]])
(mr/def ::AggregationOptionsOptions
"Additional options for any aggregation clause when wrapping it in `:aggregation-options`."
diff --git a/src/metabase/lib_be/hash.clj b/src/metabase/lib_be/hash.clj
index 279fb040b24..ac232923931 100644
--- a/src/metabase/lib_be/hash.clj
+++ b/src/metabase/lib_be/hash.clj
@@ -37,7 +37,8 @@
"Return a 256-bit SHA3 hash of `query` as a key for the cache. (This is returned as a byte array.)"
^bytes [query :- :map]
(-> query
- (cond-> (not= (keyword (:type query)) :internal) lib-be.models.transforms/normalize-query)
+ (cond-> (not= (keyword (:type query)) :internal)
+ (as-> $query (lib-be.models.transforms/normalize-query nil $query {:strict? true})))
encode
json/encode
buddy-hash/sha3-256))
diff --git a/src/metabase/lib_be/models/transforms.clj b/src/metabase/lib_be/models/transforms.clj
index 3fa165f6967..d3e5ae5674d 100644
--- a/src/metabase/lib_be/models/transforms.clj
+++ b/src/metabase/lib_be/models/transforms.clj
@@ -29,47 +29,62 @@
([query]
(normalize-query nil query))
+ ([metadata-providerable query]
+ (normalize-query metadata-providerable query nil))
+
([metadata-providerable :- [:maybe ::lib.metadata.protocols/metadata-providerable]
- query :- [:maybe :map]]
- (let [metadata-providerable (or metadata-providerable
- (when-let [mp (:lib/metadata query)]
- (when (lib/metadata-provider? mp)
- mp))
- lib.metadata.jvm/application-database-metadata-provider)]
- (cond
- (empty? query)
- {}
+ query :- [:maybe :map]
+ {:keys [strict?]} :- [:maybe
+ [:map
+ {:closed true}
+ [:strict? {:optional true, :default false} [:maybe :boolean]]]]]
+ (try
+ (let [metadata-providerable (or metadata-providerable
+ (when-let [mp (:lib/metadata query)]
+ (when (lib/metadata-provider? mp)
+ mp))
+ lib.metadata.jvm/application-database-metadata-provider)]
+ (cond
+ (empty? query)
+ {}
- (not (has-normalized-key? query :database))
- (throw (ex-info "Query must include :database" {:query query}))
+ (not (has-normalized-key? query :database))
+ (throw (ex-info "Query must include :database" {:query query}))
- (not (has-any-of-these-normalized-keys? query #{:lib/type :type}))
- (throw (ex-info "Query must include :lib/type or :type" {:query query}))
+ (not (has-any-of-these-normalized-keys? query #{:lib/type :type}))
+ (throw (ex-info "Query must include :lib/type or :type" {:query query}))
- (and (has-normalized-key? query :lib/type)
- (has-any-of-these-normalized-keys? query #{:type :query :native}))
- (throw (ex-info "MBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type"
- {:query query}))
+ (and (has-normalized-key? query :lib/type)
+ (has-any-of-these-normalized-keys? query #{:type :query :native}))
+ (throw (ex-info "MBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type"
+ {:query query}))
- (and (has-normalized-key? query :type)
- (has-normalized-key? query :stages))
- (throw (ex-info "MBQL 5 :stages is not allowed in an MBQL 4 query with :type" {:query query}))
+ (and (has-normalized-key? query :type)
+ (has-normalized-key? query :stages))
+ (throw (ex-info "MBQL 5 :stages is not allowed in an MBQL 4 query with :type" {:query query}))
- (lib/cached-metadata-provider-with-cache? (:lib/metadata query))
- (lib/normalize ::lib.schema/query query)
+ (lib/cached-metadata-provider-with-cache? (:lib/metadata query))
+ (lib/normalize ::lib.schema/query query)
- :else
- (->> query
- (lib-be.bootstrap/resolve-database (when (lib/metadata-provider? metadata-providerable)
- metadata-providerable))
- (lib/query metadata-providerable))))))
+ :else
+ (->> query
+ (lib-be.bootstrap/resolve-database (when (lib/metadata-provider? metadata-providerable)
+ metadata-providerable))
+ (lib/query metadata-providerable))))
+ ;; return an empty map if we are unable to normalize the query correctly to prevent breaking things downstream,
+ ;; unless strict mode is on (when we are saving a query)
+ (catch Throwable e
+ (when strict?
+ (throw e))
+ (log/errorf e "Error normalizing query %s" (pr-str query))
+ {}))))
(defn- transform-query-in [query]
(when-not (map? query)
(throw (ex-info (format "Query must be a map, got ^%s %s" (.getCanonicalName (class query)) (pr-str query))
{:query query, :status-code 400})))
(-> query
- normalize-query
+ (as-> $query (normalize-query nil $query {:strict? true}))
lib/prepare-for-serialization
mi/json-in))
@@ -82,7 +97,7 @@
(normalize-query query))
(catch Throwable e
(log/errorf e "Error deserializing dataset_query from app DB: %s" (ex-message e))
- nil))))
+ {}))))
(def transform-query
"Toucan 2 transform spec for Card `dataset_query` and other columns that store MBQL."
diff --git a/test/metabase/legacy_mbql/schema_test.cljc b/test/metabase/legacy_mbql/schema_test.cljc
index e46901d048d..f28bf2546e7 100644
--- a/test/metabase/legacy_mbql/schema_test.cljc
+++ b/test/metabase/legacy_mbql/schema_test.cljc
@@ -317,3 +317,17 @@
(testing "Remove deprecated keys like :model/inner_ident automatically (GIT-8399)"
(is (= {:base_type :type/*, :name "X", :display_name "X"}
(lib/normalize ::mbql.s/legacy-column-metadata {:name "X", :model/inner_ident "wow"})))))
+
+(deftest ^:parallel normalize-aggregation-inside-non-aggregation-function-test
+ (is (= [:concat "$" [:round [:sum [:field "cost_per_customer" {:base-type :type/Decimal}]]]]
+ (lib/normalize
+ ::mbql.s/Aggregation
+ ["concat" "$" ["round" ["sum" ["field" "cost_per_customer" {"base-type" "type/Decimal"}]]]])))
+ (is (= [:aggregation-options
+ [:concat "$" [:round [:sum [:field "cost_per_customer" {:base-type :type/Decimal}]]]]
+ {:name "Sum", :display-name "Sum"}]
+ (lib/normalize
+ ::mbql.s/Aggregation
+ ["aggregation-options"
+ ["concat" "$" ["round" ["sum" ["field" "cost_per_customer" {"base-type" "type/Decimal"}]]]]
+ {"name" "Sum", "display-name" "Sum"}]))))
diff --git a/test/metabase/lib_be/hash_test.clj b/test/metabase/lib_be/hash_test.clj
index fc9a80aff28..2752e277ece 100644
--- a/test/metabase/lib_be/hash_test.clj
+++ b/test/metabase/lib_be/hash_test.clj
@@ -5,8 +5,11 @@
[metabase.lib-be.hash :as lib-be.hash]
[metabase.lib.test-metadata :as meta]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util.malli :as mu]))
+(use-fixtures :once (fixtures/initialize :db))
+
(defn- query-hash-hex [query]
(codecs/bytes->hex
(mu/disable-enforcement
diff --git a/test/metabase/lib_be/models/transforms_test.clj b/test/metabase/lib_be/models/transforms_test.clj
index ce17f8ffedb..92f678afb1a 100644
--- a/test/metabase/lib_be/models/transforms_test.clj
+++ b/test/metabase/lib_be/models/transforms_test.clj
@@ -8,7 +8,7 @@
(deftest ^:parallel handle-bad-template-tags-test
(testing (str "an malformed template tags map like the one below is invalid. Rather than potentially destroy an entire API "
"response because of one malformed Card, dump the error to the logs and return nil.")
- (is (= nil
+ (is (= {}
((:out lib-be/transform-query)
(json/encode
{:database 1
@@ -16,7 +16,7 @@
:native {:template-tags 1000}}))))))
(deftest ^:parallel template-tag-validate-saves-test
- (testing "on the other hand we should be a little more strict on the way and disallow you from saving the invalid stuff"
+ (testing "on the other hand we should be a little more strict on the way in and disallow you from saving the invalid stuff"
;; TODO -- we should make sure this returns a good error message so we don't have to dig thru the exception chain.
(is (thrown?
Exception
@@ -30,33 +30,27 @@
((:out lib-be/transform-query) "{}"))))
(deftest ^:parallel normalize-busted-query-test
- ;; this validation should be done even in prod
(mu/disable-enforcement
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"\QQuery must include :database\E"
- (lib-be/normalize-query {:query {:source-table "card__117"}})))))
+ (is (= {}
+ (lib-be/normalize-query {:query {:source-table "card__117"}})))))
(deftest ^:parallel normalize-busted-query-test-2
- ;; this validation should be done even in prod
(mu/disable-enforcement
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"\QQuery must include :lib/type or :type\E"
- (lib-be/normalize-query {:database 1, :query {:source-table "card__117"}})))))
+ (is (= {}
+ (lib-be/normalize-query {:database 1, :query {:source-table "card__117"}})))))
(deftest ^:parallel normalize-busted-query-test-3
- ;; this validation should be done even in prod
(mu/disable-enforcement
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"\QMBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type\E"
- (lib-be/normalize-query {:database 1, :lib/type :mbql/query, :query {:source-table "card__117"}})))))
+ (is (= {}
+ (lib-be/normalize-query {:database 1, :lib/type :mbql/query, :query {:source-table "card__117"}})))))
(deftest ^:parallel normalize-busted-query-test-4
- ;; this validation should be done even in prod
(mu/disable-enforcement
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"\QMBQL 5 :stages is not allowed in an MBQL 4 query with :type\E"
- (lib-be/normalize-query {:database 1, :type :query, :stages []})))))
+ (is (= {}
+ (lib-be/normalize-query {:database 1, :type :query, :stages []})))))
+
+(deftest ^:parallel normalize-busted-query-test-5
+ (testing "A totally broken query should get normalized to a map rather than return a string or nil"
+ (mu/disable-enforcement
+ (is (= {}
+ ((:out lib-be/transform-query) "WOW THIS IS A MESSED UP DATASET_QUERY!"))))))
diff --git a/test/metabase/segments/models/segment_test.clj b/test/metabase/segments/models/segment_test.clj
index cfb5e143394..cfcd3c8d913 100644
--- a/test/metabase/segments/models/segment_test.clj
+++ b/test/metabase/segments/models/segment_test.clj
@@ -49,7 +49,8 @@
(t2/query-one {:update :segment
:set {:definition (json/encode {:filter "X"})}
:where [:= :id segment-id]})
- (is (nil? (:definition (t2/select-one :model/Segment :id segment-id)))))))
+ (is (= {}
+ (:definition (t2/select-one :model/Segment :id segment-id)))))))
(testing "...but should still throw them on insert"
(is (thrown? Exception
(t2/insert! :model/Segment {:table_id (mt/id :venues)