The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix query normalization issues in v57 (#66722)
Cam Sauld2d11e9ed02e
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex dbf34e827e7..e1a2714ee6c 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -709,6 +709,7 @@metabase.lib.field.resolutionmetabase.lib.filtermetabase.lib.filter.desugar.jvm ; only so [[metabase.legacy-mbql.jvm-util]] can use defined regexes+ metabase.lib.hierarchymetabase.lib.initmetabase.lib.join.utilmetabase.lib.metadatadiff --git a/resources/migrations/057_update_migrations.yaml b/resources/migrations/057_update_migrations.yamlindex 3dae882b2b5..046ddf8e5d9 100644--- a/resources/migrations/057_update_migrations.yaml+++ b/resources/migrations/057_update_migrations.yaml@@ -2604,6 +2604,18 @@ databaseChangeLog:path: instance_analytics_views/content/v5/h2-content.sqlrelativeToChangelogFile: true+ - changeSet:+ id: v57.2025-12-09T12:45:01+ author: dpsutton+ comment: >-+ Copy legacy MBQL queries from report_card.legacy_query to report_card.dataset_query where the parsed query is erroneously '{}'+ changes:+ - sql: >-+ UPDATE report_card+ SET dataset_query = legacy_query+ where dataset_query = '{}' and legacy_query is not null;+ rollback: # nothing to undo+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/legacy_mbql/normalize.cljc b/src/metabase/legacy_mbql/normalize.cljcindex d002495c297..c1433783b75 100644--- a/src/metabase/legacy_mbql/normalize.cljc+++ b/src/metabase/legacy_mbql/normalize.cljc@@ -2,20 +2,20 @@"Normalize MBQL 1-4 to MBQL 4."(:require#?@(:clj- ([potemkin :as p]))+ ([metabase.legacy-mbql.schema.helpers :as helpers]+ [potemkin :as p]))[metabase.legacy-mbql.schema :as mbql.s]- [metabase.legacy-mbql.schema.helpers :as helpers][metabase.lib.normalize :as lib.normalize][metabase.util.malli :as mu]))+#?(:clj (comment helpers/keep-me))+;; this is only for the sake of a few random namespaces that use this still#?(:clj(p/import-vars [helpers is-clause?]))(defn- infer-schema [x]- (or (when (sequential? x)- (when-let [tag (helpers/effective-clause-tag x)]- (keyword "metabase.legacy-mbql.schema" (name tag))))+ (or (mbql.s/infer-mbql-clause-schema x)::mbql.s/Query))(defn normalizediff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex 14934b1c23e..e3ac1af2f41 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -17,7 +17,10 @@[malli.core :as mc][metabase.legacy-mbql.schema.helpers :as helpers :refer [is-clause?]][metabase.legacy-mbql.schema.macros :refer [defclause defclause* one-of]]+ [metabase.lib.hierarchy :as lib.hierarchy]+ [metabase.lib.normalize :as lib.normalize][metabase.lib.schema.actions :as lib.schema.actions]+ [metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.schema.binning :as lib.schema.binning][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.constraints :as lib.schema.constraints]@@ -42,22 +45,23 @@[metabase.util.performance :as perf :refer [every? select-keys #?(:clj doseq) some mapv update-keys empty? not-empty]][metabase.util.time :as u.time]))-;; A NOTE ABOUT METADATA:-;;-;; Clauses below are marked with the following tags for documentation purposes:-;;-;; * Clauses marked `^:sugar` are syntactic sugar primarily intended to make generating queries easier on the-;; frontend. These clauses are automatically rewritten as simpler clauses by the `desugar` or `expand-macros`-;; middleware. Thus driver implementations do not need to handle these clauses.-;;-;; * Clauses marked `^:internal` are automatically generated by `wrap-value-literals` or other middleware from values-;; passed in. They are not intended to be used by the frontend when generating a query. These add certain-;; information that simplify driver implementations. When writing MBQL queries yourself you should pretend these-;; clauses don't exist.-;;-;; * Clauses marked ` require a certain set of features to be used. At some date in-;; the future we will likely add middleware that uses this metadata to automatically validate that a driver has the-;; features needed to run the query in question.+(defn infer-mbql-clause-schema+ "Infer the schema for something that is (probably) an MBQL clause."+ [x]+ (when (sequential? x)+ (when-let [tag (helpers/effective-clause-tag x)]+ (keyword "metabase.legacy-mbql.schema" (name tag)))))++(defn- normalize-mbql-clause [x]+ (when-let [schema (infer-mbql-clause-schema x)]+ (lib.normalize/normalize schema x)))++(mr/def ::AnyMBQLClause+ "Schema for ANY valid MBQL clause"+ [:fn+ {:error/message "Valid MBQL clause"+ :decode/normalize normalize-mbql-clause}+ helpers/mbql-clause?])(mr/def ::options-style"For convenience of converting back and forth between MBQL 5, from now on we can record info about the options style@@ -1292,16 +1296,27 @@;;;; [:+ [:sum [:field 10 nil]] [:sum [:field 20 nil]]]+(defn- aggregation-expression?+ "A clause is a valid aggregation if it is an aggregation clause, or it is an expression that transitively contains+ a single aggregation clause.++ (This is mostly copied from [[metabase.lib.schema.aggregation/aggregation-expression?]]"+ [x]+ (when (vector? x)+ (when-let [[tag & args] x]+ (or (lib.hierarchy/isa? tag ::lib.schema.aggregation/aggregation-clause-tag)+ ;; Case has the following shape [:case [[cond expr]...] default-expr?]+ (if (= :case tag)+ (or (some aggregation-expression? (ffirst args))+ (some aggregation-expression? (fnext args)))+ (some aggregation-expression? args))))))+(mr/def ::UnnamedAggregation- [:multi- {:error/message "unnamed aggregation clause or numeric expression"- :dispatch (fn [x]- (if (is-clause? numeric-functions x)- :numeric-expression- :else))}- [:numeric-expression [:ref ::NumericExpression]]- [:else (one-of aggregation avg cum-sum distinct distinct-where stddev sum min max metric share count-where- sum-where case if median percentile var cum-count count offset)]])+ [:and+ [:ref ::AnyMBQLClause]+ [:fn+ {:error/message "Aggregations should contain at least one aggregation function."}+ aggregation-expression?]])(mr/def ::AggregationOptionsOptions"Additional options for any aggregation clause when wrapping it in `:aggregation-options`."diff --git a/src/metabase/lib_be/hash.clj b/src/metabase/lib_be/hash.cljindex 279fb040b24..ac232923931 100644--- a/src/metabase/lib_be/hash.clj+++ b/src/metabase/lib_be/hash.clj@@ -37,7 +37,8 @@"Return a 256-bit SHA3 hash of `query` as a key for the cache. (This is returned as a byte array.)"^bytes [query :- :map](-> query- (cond-> (not= (keyword (:type query)) :internal) lib-be.models.transforms/normalize-query)+ (cond-> (not= (keyword (:type query)) :internal)+ (as-> $query (lib-be.models.transforms/normalize-query nil $query {:strict? true})))encodejson/encodebuddy-hash/sha3-256))diff --git a/src/metabase/lib_be/models/transforms.clj b/src/metabase/lib_be/models/transforms.cljindex 3fa165f6967..d3e5ae5674d 100644--- a/src/metabase/lib_be/models/transforms.clj+++ b/src/metabase/lib_be/models/transforms.clj@@ -29,47 +29,62 @@([query](normalize-query nil query))+ ([metadata-providerable query]+ (normalize-query metadata-providerable query nil))+([metadata-providerable :- [:maybe ::lib.metadata.protocols/metadata-providerable]- query :- [:maybe :map]]- (let [metadata-providerable (or metadata-providerable- (when-let [mp (:lib/metadata query)]- (when (lib/metadata-provider? mp)- mp))- lib.metadata.jvm/application-database-metadata-provider)]- (cond- (empty? query)- {}+ query :- [:maybe :map]+ {:keys [strict?]} :- [:maybe+ [:map+ {:closed true}+ [:strict? {:optional true, :default false} [:maybe :boolean]]]]]+ (try+ (let [metadata-providerable (or metadata-providerable+ (when-let [mp (:lib/metadata query)]+ (when (lib/metadata-provider? mp)+ mp))+ lib.metadata.jvm/application-database-metadata-provider)]+ (cond+ (empty? query)+ {}- (not (has-normalized-key? query :database))- (throw (ex-info "Query must include :database" {:query query}))+ (not (has-normalized-key? query :database))+ (throw (ex-info "Query must include :database" {:query query}))- (not (has-any-of-these-normalized-keys? query #{:lib/type :type}))- (throw (ex-info "Query must include :lib/type or :type" {:query query}))+ (not (has-any-of-these-normalized-keys? query #{:lib/type :type}))+ (throw (ex-info "Query must include :lib/type or :type" {:query query}))- (and (has-normalized-key? query :lib/type)- (has-any-of-these-normalized-keys? query #{:type :query :native}))- (throw (ex-info "MBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type"- {:query query}))+ (and (has-normalized-key? query :lib/type)+ (has-any-of-these-normalized-keys? query #{:type :query :native}))+ (throw (ex-info "MBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type"+ {:query query}))- (and (has-normalized-key? query :type)- (has-normalized-key? query :stages))- (throw (ex-info "MBQL 5 :stages is not allowed in an MBQL 4 query with :type" {:query query}))+ (and (has-normalized-key? query :type)+ (has-normalized-key? query :stages))+ (throw (ex-info "MBQL 5 :stages is not allowed in an MBQL 4 query with :type" {:query query}))- (lib/cached-metadata-provider-with-cache? (:lib/metadata query))- (lib/normalize ::lib.schema/query query)+ (lib/cached-metadata-provider-with-cache? (:lib/metadata query))+ (lib/normalize ::lib.schema/query query)- :else- (->> query- (lib-be.bootstrap/resolve-database (when (lib/metadata-provider? metadata-providerable)- metadata-providerable))- (lib/query metadata-providerable))))))+ :else+ (->> query+ (lib-be.bootstrap/resolve-database (when (lib/metadata-provider? metadata-providerable)+ metadata-providerable))+ (lib/query metadata-providerable))))+ ;; return an empty map if we are unable to normalize the query correctly to prevent breaking things downstream,+ ;; unless strict mode is on (when we are saving a query)+ (catch Throwable e+ (when strict?+ (throw e))+ (log/errorf e "Error normalizing query %s" (pr-str query))+ {}))))(defn- transform-query-in [query](when-not (map? query)(throw (ex-info (format "Query must be a map, got ^%s %s" (.getCanonicalName (class query)) (pr-str query)){:query query, :status-code 400})))(-> query- normalize-query+ (as-> $query (normalize-query nil $query {:strict? true}))lib/prepare-for-serializationmi/json-in))@@ -82,7 +97,7 @@(normalize-query query))(catch Throwable e(log/errorf e "Error deserializing dataset_query from app DB: %s" (ex-message e))- nil))))+ {}))))(def transform-query"Toucan 2 transform spec for Card `dataset_query` and other columns that store MBQL."diff --git a/test/metabase/legacy_mbql/schema_test.cljc b/test/metabase/legacy_mbql/schema_test.cljcindex e46901d048d..f28bf2546e7 100644--- a/test/metabase/legacy_mbql/schema_test.cljc+++ b/test/metabase/legacy_mbql/schema_test.cljc@@ -317,3 +317,17 @@(testing "Remove deprecated keys like :model/inner_ident automatically (GIT-8399)"(is (= {:base_type :type/*, :name "X", :display_name "X"}(lib/normalize ::mbql.s/legacy-column-metadata {:name "X", :model/inner_ident "wow"})))))++(deftest ^:parallel normalize-aggregation-inside-non-aggregation-function-test+ (is (= [:concat "$" [:round [:sum [:field "cost_per_customer" {:base-type :type/Decimal}]]]]+ (lib/normalize+ ::mbql.s/Aggregation+ ["concat" "$" ["round" ["sum" ["field" "cost_per_customer" {"base-type" "type/Decimal"}]]]])))+ (is (= [:aggregation-options+ [:concat "$" [:round [:sum [:field "cost_per_customer" {:base-type :type/Decimal}]]]]+ {:name "Sum", :display-name "Sum"}]+ (lib/normalize+ ::mbql.s/Aggregation+ ["aggregation-options"+ ["concat" "$" ["round" ["sum" ["field" "cost_per_customer" {"base-type" "type/Decimal"}]]]]+ {"name" "Sum", "display-name" "Sum"}]))))diff --git a/test/metabase/lib_be/hash_test.clj b/test/metabase/lib_be/hash_test.cljindex fc9a80aff28..2752e277ece 100644--- a/test/metabase/lib_be/hash_test.clj+++ b/test/metabase/lib_be/hash_test.clj@@ -5,8 +5,11 @@[metabase.lib-be.hash :as lib-be.hash][metabase.lib.test-metadata :as meta][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][metabase.util.malli :as mu]))+(use-fixtures :once (fixtures/initialize :db))+(defn- query-hash-hex [query](codecs/bytes->hex(mu/disable-enforcementdiff --git a/test/metabase/lib_be/models/transforms_test.clj b/test/metabase/lib_be/models/transforms_test.cljindex ce17f8ffedb..92f678afb1a 100644--- a/test/metabase/lib_be/models/transforms_test.clj+++ b/test/metabase/lib_be/models/transforms_test.clj@@ -8,7 +8,7 @@(deftest ^:parallel handle-bad-template-tags-test(testing (str "an malformed template tags map like the one below is invalid. Rather than potentially destroy an entire API ""response because of one malformed Card, dump the error to the logs and return nil.")- (is (= nil+ (is (= {}((:out lib-be/transform-query)(json/encode{:database 1@@ -16,7 +16,7 @@:native {:template-tags 1000}}))))))(deftest ^:parallel template-tag-validate-saves-test- (testing "on the other hand we should be a little more strict on the way and disallow you from saving the invalid stuff"+ (testing "on the other hand we should be a little more strict on the way in and disallow you from saving the invalid stuff";; TODO -- we should make sure this returns a good error message so we don't have to dig thru the exception chain.(is (thrown?Exception@@ -30,33 +30,27 @@((:out lib-be/transform-query) "{}"))))(deftest ^:parallel normalize-busted-query-test- ;; this validation should be done even in prod(mu/disable-enforcement- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"\QQuery must include :database\E"- (lib-be/normalize-query {:query {:source-table "card__117"}})))))+ (is (= {}+ (lib-be/normalize-query {:query {:source-table "card__117"}})))))(deftest ^:parallel normalize-busted-query-test-2- ;; this validation should be done even in prod(mu/disable-enforcement- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"\QQuery must include :lib/type or :type\E"- (lib-be/normalize-query {:database 1, :query {:source-table "card__117"}})))))+ (is (= {}+ (lib-be/normalize-query {:database 1, :query {:source-table "card__117"}})))))(deftest ^:parallel normalize-busted-query-test-3- ;; this validation should be done even in prod(mu/disable-enforcement- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"\QMBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type\E"- (lib-be/normalize-query {:database 1, :lib/type :mbql/query, :query {:source-table "card__117"}})))))+ (is (= {}+ (lib-be/normalize-query {:database 1, :lib/type :mbql/query, :query {:source-table "card__117"}})))))(deftest ^:parallel normalize-busted-query-test-4- ;; this validation should be done even in prod(mu/disable-enforcement- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"\QMBQL 5 :stages is not allowed in an MBQL 4 query with :type\E"- (lib-be/normalize-query {:database 1, :type :query, :stages []})))))+ (is (= {}+ (lib-be/normalize-query {:database 1, :type :query, :stages []})))))++(deftest ^:parallel normalize-busted-query-test-5+ (testing "A totally broken query should get normalized to a map rather than return a string or nil"+ (mu/disable-enforcement+ (is (= {}+ ((:out lib-be/transform-query) "WOW THIS IS A MESSED UP DATASET_QUERY!"))))))diff --git a/test/metabase/segments/models/segment_test.clj b/test/metabase/segments/models/segment_test.cljindex cfb5e143394..cfcd3c8d913 100644--- a/test/metabase/segments/models/segment_test.clj+++ b/test/metabase/segments/models/segment_test.clj@@ -49,7 +49,8 @@(t2/query-one {:update :segment:set {:definition (json/encode {:filter "X"})}:where [:= :id segment-id]})- (is (nil? (:definition (t2/select-one :model/Segment :id segment-id)))))))+ (is (= {}+ (:definition (t2/select-one :model/Segment :id segment-id)))))))(testing "...but should still throw them on insert"(is (thrown? Exception(t2/insert! :model/Segment {:table_id (mt/id :venues)