Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GIT-71792] Fix literal handling in legacy MBQL conversion (#83162)

Tyler Plude78e91452ece9
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj
index 54abb36a289..9270b97b234 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj
@@ -15,19 +15,19 @@
:table_id crm-id}
:model/Field {nps-id :id} {:name "nps"
:table_id crm-id}]
- (is (= {:database db-id
- :type "query"
- :query {:source-table crm-id
- :aggregation [["cum-count"]]
- :breakout [[:field created-id {"temporal-unit" "week"}]]
- :filter ["<" [:field nps-id nil] 9]}}
- (#'serdes/import-mbql
- {:database "Metabase Store",
- :type "query",
- :query {:source-table ["Metabase Store" "public" "crm_survey_response"],
- :aggregation [["cum-count"]],
- :breakout [["field"
- ["Metabase Store" "public"
- "crm_survey_response" "created_at"]
- {"temporal-unit" "week"}]]
- :filter ["<" ["field" ["Metabase Store" "public" "crm_survey_response" "nps"] nil] 9]}}))))))
+ (is (=? {:lib/type :mbql/query
+ :database db-id
+ :stages [{:source-table crm-id
+ :aggregation [[:cum-count {}]]
+ :breakout [[:field {:temporal-unit :week} created-id]]
+ :filters [[:< {} [:field {} nps-id] 9]]}]}
+ (#'serdes/import-mbql
+ {:database "Metabase Store",
+ :type "query",
+ :query {:source-table ["Metabase Store" "public" "crm_survey_response"],
+ :aggregation [["cum-count"]],
+ :breakout [["field"
+ ["Metabase Store" "public"
+ "crm_survey_response" "created_at"]
+ {"temporal-unit" "week"}]]
+ :filter ["<" ["field" ["Metabase Store" "public" "crm_survey_response" "nps"] nil] 9]}}))))))
diff --git a/src/metabase/legacy_mbql/normalize.cljc b/src/metabase/legacy_mbql/normalize.cljc
index e060f2d18f9..567ab746026 100644
--- a/src/metabase/legacy_mbql/normalize.cljc
+++ b/src/metabase/legacy_mbql/normalize.cljc
@@ -20,14 +20,19 @@
(defn normalize
"Normalize the tokens in a Metabase query (i.e., make them all `lisp-case` keywords), rewrite deprecated clauses as
- MBQL 4, and remove empty clauses."
+ MBQL 4, and remove empty clauses.
+
+ `options` are passed to [[metabase.lib.normalize/normalize]], e.g. `{:legacy-int-field-ids? false}`."
([x]
(normalize (infer-schema x) x))
([schema x]
+ (normalize schema x nil))
+
+ ([schema x options]
(when (:lib/type x)
(throw (ex-info "Legacy MBQL normalization code cannot normalize MBQL >= 5" {:query x})))
- (lib.normalize/normalize schema x)))
+ (lib.normalize/normalize schema x options)))
(mu/defn normalize-or-throw :- ::mbql.s/Query
"Like [[normalize]], but checks the result against the Malli schema for a legacy query, which will cause it to throw
diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljc
index ba3dde4fa40..7f5ba684173 100644
--- a/src/metabase/legacy_mbql/schema.cljc
+++ b/src/metabase/legacy_mbql/schema.cljc
@@ -81,15 +81,18 @@
(into (remove (comp own-keys first)) common-clause-option-entries)
(into entries))))
-(defn- normalize-mbql-clause [x]
+(defn- normalize-mbql-clause [x options]
(when-let [schema (infer-mbql-clause-schema x)]
- (lib.normalize/normalize schema x)))
+ (lib.normalize/normalize schema x options)))
+;;; The nested normalize doesn't inherit the outer coercer's options, so the `:decode/normalize` pass keeps raw
+;;; integers as literals and the `:decode/legacy-int-field-ids` pass, which runs after it, coerces them.
(mr/def ::AnyMBQLClause
"Schema for ANY valid MBQL clause"
[:fn
- {:error/message "Valid MBQL clause"
- :decode/normalize normalize-mbql-clause}
+ {:error/message "Valid MBQL clause"
+ :decode/normalize #(normalize-mbql-clause % {:legacy-int-field-ids? false})
+ :decode/legacy-int-field-ids #(normalize-mbql-clause % nil)}
helpers/normalized-mbql-clause?])
(mr/def ::options-style
@@ -450,7 +453,10 @@
(defn- normalize-raw-positive-int-to-field-ref
"Treats raw positive integers as Field IDs for backwards compatibility with MBQL 2, e.g.
- [:= 10 20] => [:= [:field 10 nil] 20]"
+ [:= 10 20] => [:= [:field 10 nil] 20]
+
+ Runs as a `:decode/legacy-int-field-ids` decoder, which [[metabase.lib.normalize/normalize]] applies unless called
+ with `{:legacy-int-field-ids? false}`."
[x]
(if (pos-int? x)
[:field x nil]
@@ -458,7 +464,7 @@
(mr/def ::FieldOrExpressionRef
[:schema
- {:decode/normalize #'normalize-raw-positive-int-to-field-ref}
+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}
(one-of expression field)])
;; aggregate field reference refers to an aggregation, e.g.
@@ -977,7 +983,7 @@
(mr/def ::EqualityFilterFieldArg
"Schema for the first arg to `=`, `!=`, and friends."
[:schema
- {:decode/normalize #'normalize-raw-positive-int-to-field-ref}
+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}
[:ref ::EqualityComparable]])
(defclause =
@@ -1034,7 +1040,7 @@
(mr/def ::OrderedFilterFieldArg
[:schema
- {:decode/normalize #'normalize-raw-positive-int-to-field-ref}
+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}
[:ref ::OrderComparable]])
(defclause <, field [:ref ::OrderedFilterFieldArg], value-or-field [:ref ::OrderComparable])
@@ -1316,10 +1322,7 @@
[:and
[:ref ::FieldOrExpressionDef]
[:any
- {:decode/normalize (fn [x]
- (if (pos-int? x)
- [:field x nil]
- x))}]])
+ {:decode/legacy-int-field-ids #'normalize-raw-positive-int-to-field-ref}]])
;; For all of the 'normal' Aggregations below (excluding Metrics) fields are implicit Field IDs
diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljc
index 32d8c98601b..90b3b52e955 100644
--- a/src/metabase/lib/convert.cljc
+++ b/src/metabase/lib/convert.cljc
@@ -770,11 +770,13 @@
query-type (if (-> query :stages last :lib/type (= :mbql.stage/native))
:native
:query)]
- (->> (merge (dissoc base :stages :parameters :lib.convert/converted?)
- (cond-> {:type query-type}
- (seq inner-query) (assoc query-type inner-query)
- (seq parameters) (assoc :parameters parameters)))
- (lib.normalize/normalize ::mbql.s/Query)))
+ (lib.normalize/normalize ::mbql.s/Query
+ (merge (dissoc base :stages :parameters :lib.convert/converted?)
+ (cond-> {:type query-type}
+ (seq inner-query) (assoc query-type inner-query)
+ (seq parameters) (assoc :parameters parameters)))
+ ;; MBQL 5 has no raw-int Field IDs, so any raw int here is a literal
+ {:legacy-int-field-ids? false}))
(catch #?(:clj Throwable :cljs :default) e
(throw (ex-info (lib.util/format "Error converting MBQL 5 query to legacy MBQL query: %s" (ex-message e))
{:query query}
diff --git a/src/metabase/lib/normalize.cljc b/src/metabase/lib/normalize.cljc
index 7184de267f8..edbc1a3ae1a 100644
--- a/src/metabase/lib/normalize.cljc
+++ b/src/metabase/lib/normalize.cljc
@@ -56,14 +56,20 @@
(def ^:private ^:dynamic *error-fn*
default-error-fn)
-(defn- coercer [schema]
- (mr/cached ::coercer
+(defn- coercer [schema legacy-int-field-ids?]
+ (mr/cached (if legacy-int-field-ids? ::coercer ::coercer.no-legacy-int-field-ids)
schema
(fn []
- (let [respond identity
- raise #'*error-fn*] ; capture var rather than the bound value at the time this is eval'ed
+ (let [respond identity
+ raise #'*error-fn* ; capture var rather than the bound value at the time this is eval'ed
+ transformer (if legacy-int-field-ids?
+ (mtx/transformer mtx/default-value-transformer
+ {:name :normalize}
+ {:name :legacy-int-field-ids})
+ (mtx/transformer mtx/default-value-transformer
+ {:name :normalize}))]
(log/debugf "Building :normalize coercer for schema %s" (pr-str schema))
- (mc/coercer schema (mtx/transformer mtx/default-value-transformer {:name :normalize}) respond raise)))))
+ (mc/coercer schema transformer respond raise)))))
(defn normalize
"Ensure some part of an MBQL query `x`, e.g. a clause or map, is in the right shape after coming in from JavaScript or
@@ -80,17 +86,20 @@
Pass in a `nil` schema to automatically attempt to infer the schema based on `x` itself.
By default, does not throw Exceptions -- just logs them and returns what it was able to normalize, but you can pass
- in the option `{:throw? true}` to have it throw exceptions when normalization fails."
+ in the option `{:throw? true}` to have it throw exceptions when normalization fails.
+
+ Legacy MBQL schemas treat a raw positive integer in a comparison's first argument as an MBQL 2 Field ID, e.g.
+ `[:= 1 1]` => `[:= [:field 1 nil] 1]`. Pass `{:legacy-int-field-ids? false}` to keep it as a literal."
([x]
(normalize nil x))
([schema x]
(normalize schema x nil))
- ([schema x {:keys [throw?], :or {throw? false}, :as _options}]
+ ([schema x {:keys [throw? legacy-int-field-ids?], :or {throw? false, legacy-int-field-ids? true}, :as _options}]
(let [schema (or schema (infer-schema x))
thunk (^:once fn* []
- ((coercer schema) x))]
+ ((coercer schema legacy-int-field-ids?) x))]
(if throw?
(binding [*error-fn* (fn [error]
(throw (ex-info (i18n/tru "Normalization error")
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 7d62c06733b..c4277b713fe 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -64,6 +64,7 @@
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.parameter :as lib.schema.parameter]
[metabase.models.db :as models.db]
@@ -1284,9 +1285,25 @@
(lib/all-template-tags x)))
x))
+(defn- legacy-mbql-query->mbql5
+ "Converts an imported legacy MBQL query to MBQL 5, returning `x` unchanged if it isn't one or conversion fails.
+
+ Export writes every Field ID as a portable path, so a raw integer left in an imported query is a literal. Converting
+ here with `{:legacy-int-field-ids? false}` keeps it one; the legacy normalization models apply on save would treat
+ it as an MBQL 2 Field ID."
+ [x]
+ (if (and (map? x) (= (lib/normalized-query-type x) :query))
+ (try
+ (binding [lib.schema.expression/*suppress-expression-type-check?* true]
+ (lib/->mbql5 (lib/normalize :metabase.legacy-mbql.schema/Query x {:legacy-int-field-ids? false})))
+ (catch Throwable e
+ (log/warnf "Error converting imported legacy MBQL query: %s" (ex-message e))
+ x))
+ x))
+
(defn import-mbql
"Given an MBQL expression (or any structure that may contain portable references) as an EDN structure with portable
- IDs embedded, convert the IDs back to raw numeric IDs.
+ IDs embedded, convert the IDs back to raw numeric IDs. Legacy MBQL queries are converted to MBQL 5.
Throws if an MBQL 5 expression doesn't match the schema."
[x]
@@ -1294,7 +1311,8 @@
import-mbql*
normalize-imported
(cond-> (not *skip-schema-validation?*) validate-imported-query!)
- repair-card-template-tag-names))
+ repair-card-template-tag-names
+ legacy-mbql-query->mbql5))
(declare ^:private mbql-deps-map)
diff --git a/test/metabase/legacy_mbql/normalize_test.cljc b/test/metabase/legacy_mbql/normalize_test.cljc
index 220171779f1..c4e3dd1119e 100644
--- a/test/metabase/legacy_mbql/normalize_test.cljc
+++ b/test/metabase/legacy_mbql/normalize_test.cljc
@@ -904,6 +904,62 @@
{:query {:filter [:and [:= [:field 10 nil] 20] [:= [:field 20 nil] 30]]}}
{:query {:filter [:between 10 20 30]}} {:query {:filter [:between [:field 10 nil] 20 30]}}}))
+(deftest ^:parallel legacy-int-field-ids-opt-out-test
+ (let [query {:database 1
+ :type :query
+ :query {:source-table 1
+ :filter [:and
+ [:= 1 1]
+ [:!= 2 3]
+ [:< 4 5]
+ [:between 5 1 10]
+ [:contains 6 "x"]
+ [:= [:field 10 nil] 20]]
+ :aggregation [[:sum-where [:field 10 nil] [:= 7 7]]
+ [:count-where [:< 8 9]]
+ [:sum [:case [[[:= 11 11] 1]]]]
+ [:sum 12]]
+ :joins [{:source-table 2, :alias "J", :condition ["=" 1 1]}]}}
+ normalize (fn [options]
+ (-> (mbql.normalize/normalize ::mbql.s/Query query options)
+ :query
+ (select-keys [:filter :aggregation :joins])))
+ literals {:filter [:and
+ [:= 1 1]
+ [:!= 2 3]
+ [:< 4 5]
+ [:between 5 1 10]
+ [:contains 6 "x"]
+ [:= [:field 10 nil] 20]]
+ :aggregation [[:sum-where [:field 10 nil] [:= 7 7]]
+ [:count-where [:< 8 9]]
+ [:sum [:case [[[:= 11 11] 1]]]]
+ [:sum 12]]
+ :joins [{:source-table 2, :alias "J", :condition [:= 1 1]}]}
+ field-refs {:filter [:and
+ [:= [:field 1 nil] 1]
+ [:!= [:field 2 nil] 3]
+ [:< [:field 4 nil] 5]
+ [:between [:field 5 nil] 1 10]
+ [:contains [:field 6 nil] "x"]
+ [:= [:field 10 nil] 20]]
+ :aggregation [[:sum-where [:field 10 nil] [:= [:field 7 nil] 7]]
+ [:count-where [:< [:field 8 nil] 9]]
+ [:sum [:case [[[:= [:field 11 nil] 11] 1]]]]
+ [:sum [:field 12 nil]]]
+ :joins [{:source-table 2, :alias "J", :condition [:= [:field 1 nil] 1]}]}]
+ (testing "{:legacy-int-field-ids? false} keeps raw integers as literals in comparisons and aggregation arguments"
+ (is (= literals
+ (normalize {:legacy-int-field-ids? false}))))
+ (testing "raw integers in comparisons and aggregation arguments are treated as Field IDs by default"
+ (is (= field-refs
+ (normalize nil))))
+ (testing "the opt-out coercer is cached separately from the default one"
+ (is (= literals
+ (normalize {:legacy-int-field-ids? false})))
+ (is (= field-refs
+ (normalize {:legacy-int-field-ids? true}))))))
+
(deftest ^:parallel canonicalize-filter-test-2
(normalize-tests
"`:inside` filters should get implict Field IDs for the first two args"
diff --git a/test/metabase/lib/convert_test.cljc b/test/metabase/lib/convert_test.cljc
index c7a569127c5..9ff05279144 100644
--- a/test/metabase/lib/convert_test.cljc
+++ b/test/metabase/lib/convert_test.cljc
@@ -531,6 +531,28 @@
"2020-10-20T10:20:00Z"
"10:20:00"))
+(deftest ^:parallel ->legacy-MBQL-raw-int-literals-test
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/join (lib/join-clause (meta/table-metadata :categories) [(lib/= 1 1)]))
+ (lib/filter (lib/= 1 1))
+ (lib/filter (lib/< 4 5))
+ (lib/filter (lib/between 5 1 10))
+ (lib/aggregate (lib/count-where (lib/= 2 2)))
+ (lib/aggregate (lib/sum 3)))
+ legacy (lib.convert/->legacy-MBQL query)]
+ (testing "raw integer literals in comparisons and aggregation arguments stay literals in legacy MBQL"
+ (is (=? {:filter [:and [:= 1 1] [:< 4 5] [:between 5 1 10]]
+ :aggregation [[:count-where [:= 2 2]]
+ [:sum 3]]
+ :joins [{:condition [:= 1 1]}]}
+ (:query legacy))))
+ (testing "raw integer literals survive converting back to MBQL 5"
+ (is (=? {:filters [[:= {} 1 1] [:< {} 4 5] [:between {} 5 1 10]]
+ :aggregation [[:count-where {} [:= {} 2 2]]
+ [:sum {} 3]]
+ :joins [{:conditions [[:= {} 1 1]]}]}
+ (-> legacy lib.convert/->mbql5 :stages first))))))
+
(deftest ^:parallel round-trip-filter-expression-test
(are [expressions filter-expression]
(test-round-trip {:database 1
diff --git a/test/metabase/lib/normalize_test.cljc b/test/metabase/lib/normalize_test.cljc
index 46994b65828..1cc51193c0a 100644
--- a/test/metabase/lib/normalize_test.cljc
+++ b/test/metabase/lib/normalize_test.cljc
@@ -222,12 +222,12 @@
#?(:clj
(deftest ^:synchronized normalize-error-containment-test
(testing "an AssertionError from the coercer is contained (wrapped), not propagated"
- (with-redefs-fn {#'lib.normalize/coercer (fn [_schema] (fn [_x] (throw (AssertionError. "boom"))))}
+ (with-redefs-fn {#'lib.normalize/coercer (fn [_schema _legacy-int-field-ids?] (fn [_x] (throw (AssertionError. "boom"))))}
(fn []
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Uncaught normalization error"
(lib/normalize {:lib/type :mbql/query}))))))
(testing "a fatal Error from the coercer propagates unwrapped"
- (with-redefs-fn {#'lib.normalize/coercer (fn [_schema] (fn [_x] (throw (Error. "boom"))))}
+ (with-redefs-fn {#'lib.normalize/coercer (fn [_schema _legacy-int-field-ids?] (fn [_x] (throw (Error. "boom"))))}
(fn []
(is (thrown? Error
(lib/normalize {:lib/type :mbql/query}))))))))
diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.clj
index ba509a35d26..5b863aee0ff 100644
--- a/test/metabase/models/serialization_test.clj
+++ b/test/metabase/models/serialization_test.clj
@@ -157,6 +157,38 @@
(fake-uuid 0)]]]}]}
(serdes/import-mbql query))))))
+(deftest ^:parallel import-mbql-legacy-query-test
+ (binding [serdes/*import-database-fk* (constantly 1)
+ serdes/*import-table-fk* (constantly 2)
+ serdes/*import-field-fk* (constantly 3)]
+ (testing "a legacy MBQL query is converted to MBQL 5, keeping integer literals in comparisons and aggregations as literals"
+ (is (=? {:lib/type :mbql/query
+ :database 1
+ :stages [{:source-table 2
+ :filters [[:= {} [:field {} 3] 1]
+ [:= {} 1 1]
+ [:< {} 4 5]]
+ :aggregation [[:sum-where {} [:field {} 3] [:= {} 6 6]]]
+ :joins [{:alias "J"
+ :conditions [[:= {} 1 1]]}]}]}
+ (serdes/import-mbql
+ {:database "DB"
+ :type "query"
+ :query {:source-table ["DB" "SCHEMA" "TABLE"]
+ :filter ["and"
+ ["=" ["field" ["DB" "SCHEMA" "TABLE" "FIELD"] nil] 1]
+ ["=" 1 1]
+ ["<" 4 5]]
+ :aggregation [["sum-where" ["field" ["DB" "SCHEMA" "TABLE" "FIELD"] nil] ["=" 6 6]]]
+ :joins [{:source-table ["DB" "SCHEMA" "TABLE"]
+ :alias "J"
+ :condition ["=" 1 1]}]}}))))
+ (testing "a legacy native query is not converted"
+ (is (=? {:database 1
+ :type "native"
+ :native {:query "SELECT 1"}}
+ (serdes/import-mbql {:database "DB", :type "native", :native {:query "SELECT 1"}}))))))
+
(deftest ^:parallel hydrate-mbql-5-uuids-on-import-test-2
(binding [serdes/*import-field-fk* (constantly 3)]
(are [x expected] (=? expected
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index eb67bc391f9..7b644fb349b 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -17,6 +17,7 @@
[metabase.queries.models.card :as card]
[metabase.queries.models.parameter-card :as parameter-card]
[metabase.queries.schema :as queries.schema]
+ [metabase.query-processor :as qp]
[metabase.query-processor.card :as qp.card]
[metabase.query-processor.card-test :as qp.card-test]
[metabase.query-processor.preprocess :as qp.preprocess]
@@ -821,6 +822,31 @@
;; :id should be portablized to a Field FK path: [db-name schema table-name field-name]
(is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))))))
+(deftest serdes-load-legacy-query-integer-literals-test
+ (testing "a Card loaded from a legacy-MBQL export keeps integer literals in comparisons and runs"
+ (let [db-name (t2/select-one-fn :name :model/Database (mt/id))
+ price-id (mt/id :venues :price)]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues)}]
+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))
+ ;; `price-id` is the Field ID of VENUES.PRICE, so coercing it would silently filter on that column
+ legacy {:database db-name
+ :type "query"
+ :query {:source-table [db-name "PUBLIC" "VENUES"]
+ :aggregation [["count"]]
+ :filter ["=" price-id price-id]
+ :joins [{:source-table [db-name "PUBLIC" "CATEGORIES"]
+ :alias "C"
+ :strategy "left-join"
+ :condition ["=" 1 1]}]}}]
+ (serdes/load-one! (assoc extracted :dataset_query legacy)
+ (t2/select-one :model/Card :id card-id))
+ (let [query (t2/select-one-fn :dataset_query :model/Card :id card-id)]
+ (is (=? {:stages [{:filters [[:= {} price-id price-id]]
+ :joins [{:conditions [[:= {} 1 1]]}]}]}
+ query))
+ (is (= [[(* 100 75)]]
+ (mt/rows (qp/process-query query))))))))))
+
(deftest ^:parallel upgrade-to-v2-db-test
(testing ":visualization_settings v. 1 should be upgraded to v. 2 on select"
(mt/with-temp [:model/Card {card-id :id} {:visualization_settings {:pie.show_legend true}}]