The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GIT-71792] Fix literal handling in legacy MBQL conversion (#83162)
Tyler Plude78e91452ece9
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.cljindex 54abb36a289..9270b97b234 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj@@ -15,19 +15,19 @@:table_id crm-id}:model/Field {nps-id :id} {:name "nps":table_id crm-id}]- (is (= {:database db-id- :type "query"- :query {:source-table crm-id- :aggregation [["cum-count"]]- :breakout [[:field created-id {"temporal-unit" "week"}]]- :filter ["<" [:field nps-id nil] 9]}}- (#'serdes/import-mbql- {:database "Metabase Store",- :type "query",- :query {:source-table ["Metabase Store" "public" "crm_survey_response"],- :aggregation [["cum-count"]],- :breakout [["field"- ["Metabase Store" "public"- "crm_survey_response" "created_at"]- {"temporal-unit" "week"}]]- :filter ["<" ["field" ["Metabase Store" "public" "crm_survey_response" "nps"] nil] 9]}}))))))+ (is (=? {:lib/type :mbql/query+ :database db-id+ :stages [{:source-table crm-id+ :aggregation [[:cum-count {}]]+ :breakout [[:field {:temporal-unit :week} created-id]]+ :filters [[:< {} [:field {} nps-id] 9]]}]}+ (#'serdes/import-mbql+ {:database "Metabase Store",+ :type "query",+ :query {:source-table ["Metabase Store" "public" "crm_survey_response"],+ :aggregation [["cum-count"]],+ :breakout [["field"+ ["Metabase Store" "public"+ "crm_survey_response" "created_at"]+ {"temporal-unit" "week"}]]+ :filter ["<" ["field" ["Metabase Store" "public" "crm_survey_response" "nps"] nil] 9]}}))))))diff --git a/src/metabase/legacy_mbql/normalize.cljc b/src/metabase/legacy_mbql/normalize.cljcindex e060f2d18f9..567ab746026 100644--- a/src/metabase/legacy_mbql/normalize.cljc+++ b/src/metabase/legacy_mbql/normalize.cljc@@ -20,14 +20,19 @@(defn normalize"Normalize the tokens in a Metabase query (i.e., make them all `lisp-case` keywords), rewrite deprecated clauses as- MBQL 4, and remove empty clauses."+ MBQL 4, and remove empty clauses.++ `options` are passed to [[metabase.lib.normalize/normalize]], e.g. `{:legacy-int-field-ids? false}`."([x](normalize (infer-schema x) x))([schema x]+ (normalize schema x nil))++ ([schema x options](when (:lib/type x)(throw (ex-info "Legacy MBQL normalization code cannot normalize MBQL >= 5" {:query x})))- (lib.normalize/normalize schema x)))+ (lib.normalize/normalize schema x options)))(mu/defn normalize-or-throw :- ::mbql.s/Query"Like [[normalize]], but checks the result against the Malli schema for a legacy query, which will cause it to throwdiff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex ba3dde4fa40..7f5ba684173 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -81,15 +81,18 @@(into (remove (comp own-keys first)) common-clause-option-entries)(into entries))))-(defn- normalize-mbql-clause [x]+(defn- normalize-mbql-clause [x options](when-let [schema (infer-mbql-clause-schema x)]- (lib.normalize/normalize schema x)))+ (lib.normalize/normalize schema x options)))+;;; The nested normalize doesn't inherit the outer coercer's options, so the `:decode/normalize` pass keeps raw+;;; integers as literals and the `:decode/legacy-int-field-ids` pass, which runs after it, coerces them.(mr/def ::AnyMBQLClause"Schema for ANY valid MBQL clause"[:fn- {:error/message "Valid MBQL clause"- :decode/normalize normalize-mbql-clause}+ {:error/message "Valid MBQL clause"+ :decode/normalize #(normalize-mbql-clause % {:legacy-int-field-ids? false})+ :decode/legacy-int-field-ids #(normalize-mbql-clause % nil)}helpers/normalized-mbql-clause?])(mr/def ::options-style@@ -450,7 +453,10 @@(defn- normalize-raw-positive-int-to-field-ref"Treats raw positive integers as Field IDs for backwards compatibility with MBQL 2, e.g.- [:= 10 20] => [:= [:field 10 nil] 20]"+ [:= 10 20] => [:= [:field 10 nil] 20]++ Runs as a `:decode/legacy-int-field-ids` decoder, which [[metabase.lib.normalize/normalize]] applies unless called+ with `{:legacy-int-field-ids? false}`."[x](if (pos-int? x)[:field x nil]@@ -458,7 +464,7 @@(mr/def ::FieldOrExpressionRef[:schema- {:decode/normalize #'normalize-raw-positive-int-to-field-ref}+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}(one-of expression field)]);; aggregate field reference refers to an aggregation, e.g.@@ -977,7 +983,7 @@(mr/def ::EqualityFilterFieldArg"Schema for the first arg to `=`, `!=`, and friends."[:schema- {:decode/normalize #'normalize-raw-positive-int-to-field-ref}+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}[:ref ::EqualityComparable]])(defclause =@@ -1034,7 +1040,7 @@(mr/def ::OrderedFilterFieldArg[:schema- {:decode/normalize #'normalize-raw-positive-int-to-field-ref}+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}[:ref ::OrderComparable]])(defclause <, field [:ref ::OrderedFilterFieldArg], value-or-field [:ref ::OrderComparable])@@ -1316,10 +1322,7 @@[:and[:ref ::FieldOrExpressionDef][:any- {:decode/normalize (fn [x]- (if (pos-int? x)- [:field x nil]- x))}]])+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}]]);; For all of the 'normal' Aggregations below (excluding Metrics) fields are implicit Field IDsdiff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljcindex 32d8c98601b..90b3b52e955 100644--- a/src/metabase/lib/convert.cljc+++ b/src/metabase/lib/convert.cljc@@ -770,11 +770,13 @@query-type (if (-> query :stages last :lib/type (= :mbql.stage/native)):native:query)]- (->> (merge (dissoc base :stages :parameters :lib.convert/converted?)- (cond-> {:type query-type}- (seq inner-query) (assoc query-type inner-query)- (seq parameters) (assoc :parameters parameters)))- (lib.normalize/normalize ::mbql.s/Query)))+ (lib.normalize/normalize ::mbql.s/Query+ (merge (dissoc base :stages :parameters :lib.convert/converted?)+ (cond-> {:type query-type}+ (seq inner-query) (assoc query-type inner-query)+ (seq parameters) (assoc :parameters parameters)))+ ;; MBQL 5 has no raw-int Field IDs, so any raw int here is a literal+ {:legacy-int-field-ids? false}))(catch #?(:clj Throwable :cljs :default) e(throw (ex-info (lib.util/format "Error converting MBQL 5 query to legacy MBQL query: %s" (ex-message e)){:query query}diff --git a/src/metabase/lib/normalize.cljc b/src/metabase/lib/normalize.cljcindex 7184de267f8..edbc1a3ae1a 100644--- a/src/metabase/lib/normalize.cljc+++ b/src/metabase/lib/normalize.cljc@@ -56,14 +56,20 @@(def ^:private ^:dynamic *error-fn*default-error-fn)-(defn- coercer [schema]- (mr/cached ::coercer+(defn- coercer [schema legacy-int-field-ids?]+ (mr/cached (if legacy-int-field-ids? ::coercer ::coercer.no-legacy-int-field-ids)schema(fn []- (let [respond identity- raise #'*error-fn*] ; capture var rather than the bound value at the time this is eval'ed+ (let [respond identity+ raise #'*error-fn* ; capture var rather than the bound value at the time this is eval'ed+ transformer (if legacy-int-field-ids?+ (mtx/transformer mtx/default-value-transformer+ {:name :normalize}+ {:name :legacy-int-field-ids})+ (mtx/transformer mtx/default-value-transformer+ {:name :normalize}))](log/debugf "Building :normalize coercer for schema %s" (pr-str schema))- (mc/coercer schema (mtx/transformer mtx/default-value-transformer {:name :normalize}) respond raise)))))+ (mc/coercer schema transformer respond raise)))))(defn normalize"Ensure some part of an MBQL query `x`, e.g. a clause or map, is in the right shape after coming in from JavaScript or@@ -80,17 +86,20 @@Pass in a `nil` schema to automatically attempt to infer the schema based on `x` itself.By default, does not throw Exceptions -- just logs them and returns what it was able to normalize, but you can pass- in the option `{:throw? true}` to have it throw exceptions when normalization fails."+ in the option `{:throw? true}` to have it throw exceptions when normalization fails.++ Legacy MBQL schemas treat a raw positive integer in a comparison's first argument as an MBQL 2 Field ID, e.g.+ `[:= 1 1]` => `[:= [:field 1 nil] 1]`. Pass `{:legacy-int-field-ids? false}` to keep it as a literal."([x](normalize nil x))([schema x](normalize schema x nil))- ([schema x {:keys [throw?], :or {throw? false}, :as _options}]+ ([schema x {:keys [throw? legacy-int-field-ids?], :or {throw? false, legacy-int-field-ids? true}, :as _options}](let [schema (or schema (infer-schema x))thunk (^:once fn* []- ((coercer schema) x))]+ ((coercer schema legacy-int-field-ids?) x))](if throw?(binding [*error-fn* (fn [error](throw (ex-info (i18n/tru "Normalization error")diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 7d62c06733b..c4277b713fe 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -64,6 +64,7 @@[metabase.lib.core :as lib][metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.models.db :as models.db]@@ -1284,9 +1285,25 @@(lib/all-template-tags x)))x))+(defn- legacy-mbql-query->mbql5+ "Converts an imported legacy MBQL query to MBQL 5, returning `x` unchanged if it isn't one or conversion fails.++ Export writes every Field ID as a portable path, so a raw integer left in an imported query is a literal. Converting+ here with `{:legacy-int-field-ids? false}` keeps it one; the legacy normalization models apply on save would treat+ it as an MBQL 2 Field ID."+ [x]+ (if (and (map? x) (= (lib/normalized-query-type x) :query))+ (try+ (binding [lib.schema.expression/*suppress-expression-type-check?* true]+ (lib/->mbql5 (lib/normalize :metabase.legacy-mbql.schema/Query x {:legacy-int-field-ids? false})))+ (catch Throwable e+ (log/warnf "Error converting imported legacy MBQL query: %s" (ex-message e))+ x))+ x))+(defn import-mbql"Given an MBQL expression (or any structure that may contain portable references) as an EDN structure with portable- IDs embedded, convert the IDs back to raw numeric IDs.+ IDs embedded, convert the IDs back to raw numeric IDs. Legacy MBQL queries are converted to MBQL 5.Throws if an MBQL 5 expression doesn't match the schema."[x]@@ -1294,7 +1311,8 @@import-mbql*normalize-imported(cond-> (not *skip-schema-validation?*) validate-imported-query!)- repair-card-template-tag-names))+ repair-card-template-tag-names+ legacy-mbql-query->mbql5))(declare ^:private mbql-deps-map)diff --git a/test/metabase/legacy_mbql/normalize_test.cljc b/test/metabase/legacy_mbql/normalize_test.cljcindex 220171779f1..c4e3dd1119e 100644--- a/test/metabase/legacy_mbql/normalize_test.cljc+++ b/test/metabase/legacy_mbql/normalize_test.cljc@@ -904,6 +904,62 @@{:query {:filter [:and [:= [:field 10 nil] 20] [:= [:field 20 nil] 30]]}}{:query {:filter [:between 10 20 30]}} {:query {:filter [:between [:field 10 nil] 20 30]}}}))+(deftest ^:parallel legacy-int-field-ids-opt-out-test+ (let [query {:database 1+ :type :query+ :query {:source-table 1+ :filter [:and+ [:= 1 1]+ [:!= 2 3]+ [:< 4 5]+ [:between 5 1 10]+ [:contains 6 "x"]+ [:= [:field 10 nil] 20]]+ :aggregation [[:sum-where [:field 10 nil] [:= 7 7]]+ [:count-where [:< 8 9]]+ [:sum [:case [[[:= 11 11] 1]]]]+ [:sum 12]]+ :joins [{:source-table 2, :alias "J", :condition ["=" 1 1]}]}}+ normalize (fn [options]+ (-> (mbql.normalize/normalize ::mbql.s/Query query options)+ :query+ (select-keys [:filter :aggregation :joins])))+ literals {:filter [:and+ [:= 1 1]+ [:!= 2 3]+ [:< 4 5]+ [:between 5 1 10]+ [:contains 6 "x"]+ [:= [:field 10 nil] 20]]+ :aggregation [[:sum-where [:field 10 nil] [:= 7 7]]+ [:count-where [:< 8 9]]+ [:sum [:case [[[:= 11 11] 1]]]]+ [:sum 12]]+ :joins [{:source-table 2, :alias "J", :condition [:= 1 1]}]}+ field-refs {:filter [:and+ [:= [:field 1 nil] 1]+ [:!= [:field 2 nil] 3]+ [:< [:field 4 nil] 5]+ [:between [:field 5 nil] 1 10]+ [:contains [:field 6 nil] "x"]+ [:= [:field 10 nil] 20]]+ :aggregation [[:sum-where [:field 10 nil] [:= [:field 7 nil] 7]]+ [:count-where [:< [:field 8 nil] 9]]+ [:sum [:case [[[:= [:field 11 nil] 11] 1]]]]+ [:sum [:field 12 nil]]]+ :joins [{:source-table 2, :alias "J", :condition [:= [:field 1 nil] 1]}]}]+ (testing "{:legacy-int-field-ids? false} keeps raw integers as literals in comparisons and aggregation arguments"+ (is (= literals+ (normalize {:legacy-int-field-ids? false}))))+ (testing "raw integers in comparisons and aggregation arguments are treated as Field IDs by default"+ (is (= field-refs+ (normalize nil))))+ (testing "the opt-out coercer is cached separately from the default one"+ (is (= literals+ (normalize {:legacy-int-field-ids? false})))+ (is (= field-refs+ (normalize {:legacy-int-field-ids? true}))))))+(deftest ^:parallel canonicalize-filter-test-2(normalize-tests"`:inside` filters should get implict Field IDs for the first two args"diff --git a/test/metabase/lib/convert_test.cljc b/test/metabase/lib/convert_test.cljcindex c7a569127c5..9ff05279144 100644--- a/test/metabase/lib/convert_test.cljc+++ b/test/metabase/lib/convert_test.cljc@@ -531,6 +531,28 @@"2020-10-20T10:20:00Z""10:20:00"))+(deftest ^:parallel ->legacy-MBQL-raw-int-literals-test+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/join (lib/join-clause (meta/table-metadata :categories) [(lib/= 1 1)]))+ (lib/filter (lib/= 1 1))+ (lib/filter (lib/< 4 5))+ (lib/filter (lib/between 5 1 10))+ (lib/aggregate (lib/count-where (lib/= 2 2)))+ (lib/aggregate (lib/sum 3)))+ legacy (lib.convert/->legacy-MBQL query)]+ (testing "raw integer literals in comparisons and aggregation arguments stay literals in legacy MBQL"+ (is (=? {:filter [:and [:= 1 1] [:< 4 5] [:between 5 1 10]]+ :aggregation [[:count-where [:= 2 2]]+ [:sum 3]]+ :joins [{:condition [:= 1 1]}]}+ (:query legacy))))+ (testing "raw integer literals survive converting back to MBQL 5"+ (is (=? {:filters [[:= {} 1 1] [:< {} 4 5] [:between {} 5 1 10]]+ :aggregation [[:count-where {} [:= {} 2 2]]+ [:sum {} 3]]+ :joins [{:conditions [[:= {} 1 1]]}]}+ (-> legacy lib.convert/->mbql5 :stages first))))))+(deftest ^:parallel round-trip-filter-expression-test(are [expressions filter-expression](test-round-trip {:database 1diff --git a/test/metabase/lib/normalize_test.cljc b/test/metabase/lib/normalize_test.cljcindex 46994b65828..1cc51193c0a 100644--- a/test/metabase/lib/normalize_test.cljc+++ b/test/metabase/lib/normalize_test.cljc@@ -222,12 +222,12 @@#?(:clj(deftest ^:synchronized normalize-error-containment-test(testing "an AssertionError from the coercer is contained (wrapped), not propagated"- (with-redefs-fn {#'lib.normalize/coercer (fn [_schema] (fn [_x] (throw (AssertionError. "boom"))))}+ (with-redefs-fn {#'lib.normalize/coercer (fn [_schema _legacy-int-field-ids?] (fn [_x] (throw (AssertionError. "boom"))))}(fn [](is (thrown-with-msg? clojure.lang.ExceptionInfo #"Uncaught normalization error"(lib/normalize {:lib/type :mbql/query}))))))(testing "a fatal Error from the coercer propagates unwrapped"- (with-redefs-fn {#'lib.normalize/coercer (fn [_schema] (fn [_x] (throw (Error. "boom"))))}+ (with-redefs-fn {#'lib.normalize/coercer (fn [_schema _legacy-int-field-ids?] (fn [_x] (throw (Error. "boom"))))}(fn [](is (thrown? Error(lib/normalize {:lib/type :mbql/query}))))))))diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.cljindex ba509a35d26..5b863aee0ff 100644--- a/test/metabase/models/serialization_test.clj+++ b/test/metabase/models/serialization_test.clj@@ -157,6 +157,38 @@(fake-uuid 0)]]]}]}(serdes/import-mbql query))))))+(deftest ^:parallel import-mbql-legacy-query-test+ (binding [serdes/*import-database-fk* (constantly 1)+ serdes/*import-table-fk* (constantly 2)+ serdes/*import-field-fk* (constantly 3)]+ (testing "a legacy MBQL query is converted to MBQL 5, keeping integer literals in comparisons and aggregations as literals"+ (is (=? {:lib/type :mbql/query+ :database 1+ :stages [{:source-table 2+ :filters [[:= {} [:field {} 3] 1]+ [:= {} 1 1]+ [:< {} 4 5]]+ :aggregation [[:sum-where {} [:field {} 3] [:= {} 6 6]]]+ :joins [{:alias "J"+ :conditions [[:= {} 1 1]]}]}]}+ (serdes/import-mbql+ {:database "DB"+ :type "query"+ :query {:source-table ["DB" "SCHEMA" "TABLE"]+ :filter ["and"+ ["=" ["field" ["DB" "SCHEMA" "TABLE" "FIELD"] nil] 1]+ ["=" 1 1]+ ["<" 4 5]]+ :aggregation [["sum-where" ["field" ["DB" "SCHEMA" "TABLE" "FIELD"] nil] ["=" 6 6]]]+ :joins [{:source-table ["DB" "SCHEMA" "TABLE"]+ :alias "J"+ :condition ["=" 1 1]}]}}))))+ (testing "a legacy native query is not converted"+ (is (=? {:database 1+ :type "native"+ :native {:query "SELECT 1"}}+ (serdes/import-mbql {:database "DB", :type "native", :native {:query "SELECT 1"}}))))))+(deftest ^:parallel hydrate-mbql-5-uuids-on-import-test-2(binding [serdes/*import-field-fk* (constantly 3)](are [x expected] (=? expecteddiff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex eb67bc391f9..7b644fb349b 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -17,6 +17,7 @@[metabase.queries.models.card :as card][metabase.queries.models.parameter-card :as parameter-card][metabase.queries.schema :as queries.schema]+ [metabase.query-processor :as qp][metabase.query-processor.card :as qp.card][metabase.query-processor.card-test :as qp.card-test][metabase.query-processor.preprocess :as qp.preprocess]@@ -821,6 +822,31 @@;; :id should be portablized to a Field FK path: [db-name schema table-name field-name](is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))))))+(deftest serdes-load-legacy-query-integer-literals-test+ (testing "a Card loaded from a legacy-MBQL export keeps integer literals in comparisons and runs"+ (let [db-name (t2/select-one-fn :name :model/Database (mt/id))+ price-id (mt/id :venues :price)]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues)}]+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))+ ;; `price-id` is the Field ID of VENUES.PRICE, so coercing it would silently filter on that column+ legacy {:database db-name+ :type "query"+ :query {:source-table [db-name "PUBLIC" "VENUES"]+ :aggregation [["count"]]+ :filter ["=" price-id price-id]+ :joins [{:source-table [db-name "PUBLIC" "CATEGORIES"]+ :alias "C"+ :strategy "left-join"+ :condition ["=" 1 1]}]}}]+ (serdes/load-one! (assoc extracted :dataset_query legacy)+ (t2/select-one :model/Card :id card-id))+ (let [query (t2/select-one-fn :dataset_query :model/Card :id card-id)]+ (is (=? {:stages [{:filters [[:= {} price-id price-id]]+ :joins [{:conditions [[:= {} 1 1]]}]}]}+ query))+ (is (= [[(* 100 75)]]+ (mt/rows (qp/process-query query))))))))))+(deftest ^:parallel upgrade-to-v2-db-test(testing ":visualization_settings v. 1 should be upgraded to v. 2 on select"(mt/with-temp [:model/Card {card-id :id} {:visualization_settings {:pie.show_legend true}}]