Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Omit UUIDs in SerDes MBQL 5 export unless required (#71862)

Cam Saul22a21845f145
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj
index 886009194d5..54abb36a289 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj
@@ -21,7 +21,7 @@
:aggregation [["cum-count"]]
:breakout [[:field created-id {"temporal-unit" "week"}]]
:filter ["<" [:field nps-id nil] 9]}}
- (#'serdes/mbql-fully-qualified-names->ids
+ (#'serdes/import-mbql
{:database "Metabase Store",
:type "query",
:query {:source-table ["Metabase Store" "public" "crm_survey_response"],
diff --git a/src/metabase/lib/normalize.cljc b/src/metabase/lib/normalize.cljc
index 000bc785aad..4ca9021a406 100644
--- a/src/metabase/lib/normalize.cljc
+++ b/src/metabase/lib/normalize.cljc
@@ -97,7 +97,12 @@
(throw (ex-info (i18n/tru "Normalization error")
{:schema schema, :x x, :error error})))]
(thunk))
- (thunk)))))
+ (try
+ (thunk)
+ (catch #?(:clj Throwable :cljs :default) e
+ (throw (ex-info (str "Uncaught normalization error: " (ex-message e))
+ {:schema schema}
+ e))))))))
(mu/defn ->normalized-stage-metadata :- ::lib.schema.metadata/stage
"Take a sequence of legacy or Lib metadata maps, convert to Lib-style if needed, then normalize them.
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index a8c138453d7..3a7aa97d404 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -65,6 +65,7 @@
;; legacy usages -- do not use in new code
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
+ [metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.parameter :as lib.schema.parameter]
[metabase.lib.util.match :as lib.util.match]
@@ -956,6 +957,7 @@
`(try
~@body
(catch clojure.lang.ExceptionInfo e#
+ (log/debugf e# "Caught error in fk-elide")
(when-not (= (::type (ex-data e#)) :target-not-found)
(throw e#))
nil)))
@@ -1149,16 +1151,24 @@
;;; ## MBQL Fields
-(mu/defn- mbql-clause-tag :- [:maybe [:enum :field :dimension :metric :segment :measure]]
+(mu/defn- mbql-ref? :- [:maybe [:enum :field :field-id :dimension :metric :segment :measure]]
"Is given form an MBQL entity reference?"
[form]
(when (and (vector? form)
- (#{:field :dimension :metric :segment :measure} (keyword (first form))))
+ (#{:field :field-id :dimension :metric :segment :measure} (keyword (first form))))
(keyword (first form))))
-(defn- normalize [mbql]
- (let [tag (mbql-clause-tag mbql)
+(mr/def ::mbql-3-field-id-ref
+ "Ultra-legacy MBQL 3 `:field-id` ref; which (allegedly) can show up in viz settings; this schema is only here so we
+ can use Lib normalization in [[normalize-mbql-ref]]."
+ [:tuple
+ [:= {:decode/normalize lib.schema.common/normalize-keyword} :field-id]
+ pos-int?])
+
+(defn- normalize-mbql-ref [mbql]
+ (let [tag (mbql-ref? mbql)
schema (case tag
+ :field-id ::mbql-3-field-id-ref
:field [:multi
{:dispatch #(and (vector? %)
(map? (second %)))}
@@ -1172,71 +1182,81 @@
(cond->> mbql
schema (lib/normalize schema mbql))))
-(defn- mbql-id->fully-qualified-name
+(def ^:private ^:dynamic *required-lib-uuids-for-export* nil)
+
+(mu/defn- collect-required-lib-uuids :- [:set ::lib.schema.common/uuid]
+ [x]
+ (set
+ (lib.util.match/match-many x
+ [:aggregation (_opts :guard map?) (uuid :guard string?)]
+ uuid)))
+
+(declare export-mbql)
+
+(defn- export-mbql-map [m]
+ (reduce-kv
+ (fn [m k v]
+ (case k
+ :lib/metadata (dissoc m k)
+ :database (update m k *export-database-fk*)
+ (:card_id :card-id) (update m k *export-fk* :model/Card) ; attributes that refer to db fields use `_`; template-tags use `-`
+ (:source_table :source-table) (cond-> m
+ (pos-int? v)
+ (update k *export-table-fk*))
+ (:source_card :source-card) (cond-> m
+ (pos-int? v)
+ (update k *export-fk* :model/Card))
+ :source-field (cond-> m
+ (pos-int? v)
+ (update k *export-field-fk*))
+ ::mb.viz/param-mapping-source (update m k *export-field-fk*)
+ :segment (update m k *export-fk* :model/Segment)
+ :snippet-id (update m k *export-fk* :model/NativeQuerySnippet)
+ :lib/uuid (cond-> m
+ (not (contains? *required-lib-uuids-for-export* v))
+ (dissoc :lib/uuid))
+ #_else (update m k export-mbql)))
+ m
+ m))
+
+(defn- export-mbql-ref
[mbql]
- (lib.util.match/replace-lite (normalize mbql)
+ (lib.util.match/replace-lite (normalize-mbql-ref mbql)
;; `pos-int?` guard is here to make the operation idempotent
[:field (opts :guard map?) (id :guard pos-int?)]
- [:field (mbql-id->fully-qualified-name opts) (*export-field-fk* id)]
+ [:field (export-mbql-map opts) (*export-field-fk* id)]
;; legacy (MBQL 4) field refs are still supported in parameter targets and in result metadata `field_ref`...
[:field (id :guard pos-int?) (opts :guard (some-fn map? nil?))]
- [:field (*export-field-fk* id) (mbql-id->fully-qualified-name opts)]
+ [:field (*export-field-fk* id) (export-mbql-map opts)]
;; MBQL 3 `:field-id` can (allegedly) still show up sometimes? Support it just in case.
[(tag :guard #{:field :field-id}) (id :guard pos-int?)]
[tag (*export-field-fk* id)]
- {:source-table (id :guard pos-int?)}
- (assoc &match :source-table (*export-table-fk* id))
-
- ;; source-field is also used within parameter mapping dimensions
- ;; example relevant clause - [:field 2 {:source-field 1}]
- {:source-field (id :guard pos-int?)}
- (assoc &match :source-field (*export-field-fk* id))
-
[:dimension (dim :guard vector?)]
- [:dimension (mbql-id->fully-qualified-name dim)]
+ [:dimension (export-mbql-ref dim)]
[:metric opts (id :guard pos-int?)]
- [:metric (mbql-id->fully-qualified-name opts) (*export-fk* id 'Card)]
+ [:metric (export-mbql-map opts) (*export-fk* id 'Card)]
[:segment opts (id :guard pos-int?)]
- [:segment (mbql-id->fully-qualified-name opts) (*export-fk* id 'Segment)]
+ [:segment (export-mbql-map opts) (*export-fk* id 'Segment)]
[:measure opts (id :guard pos-int?)]
- [:measure (mbql-id->fully-qualified-name opts) (*export-fk* id 'Measure)]))
+ [:measure (export-mbql-map opts) (*export-fk* id 'Measure)]))
(defn export-mbql
"Given an MBQL expression, convert it to an EDN structure and turn the non-portable Database, Table and Field IDs
inside it into portable references."
- [entity]
- (lib.util.match/replace-lite entity
- (_ :guard mbql-clause-tag)
- (mbql-id->fully-qualified-name &match)
-
- (_ :guard sequential?)
- (mapv export-mbql &match)
-
- (_ :guard map?)
- (reduce-kv
- (fn [entity k _v]
- (let [f (case k
- :database (fn [db-id]
- (if (= db-id lib.schema.id/saved-questions-virtual-database-id)
- "database/__virtual"
- (t2/select-one-fn :name :model/Database :id db-id)))
- (:card_id :card-id) #(*export-fk* % :model/Card) ; attributes that refer to db fields use `_`; template-tags use `-`
- (:source_table :source-table) *export-table-fk*
- (:source_card :source-card) #(*export-fk* % :model/Card)
- ::mb.viz/param-mapping-source *export-field-fk*
- :segment #(*export-fk* % :model/Segment)
- :snippet-id #(*export-fk* % :model/NativeQuerySnippet)
- :lib/metadata (constantly nil)
- #_else export-mbql)]
- (update entity k f)))
- &match
- &match)))
+ [x]
+ ;; if required UUIDs are already calculated don't recalculate when we recurse.
+ (binding [*required-lib-uuids-for-export* (or *required-lib-uuids-for-export* (collect-required-lib-uuids x))]
+ (cond
+ (mbql-ref? x) (export-mbql-ref x)
+ (sequential? x) (mapv export-mbql x)
+ (map? x) (export-mbql-map x)
+ :else x)))
(defn- portable-id?
"True if the provided string is either an Entity ID or identity-hash string."
@@ -1245,45 +1265,58 @@
(or (entity-id? s)
(identity-hash? s))))
-(defn- mbql-fully-qualified-names->ids*
+(declare import-mbql*)
+
+(defn- import-mbql-map [m]
+ (reduce-kv
+ (fn [m k v]
+ (case k
+ :database (cond-> m
+ (string? v)
+ (update k (if (= v "database/__virtual")
+ (constantly lib.schema.id/saved-questions-virtual-database-id)
+ *import-database-fk*)))
+ (:source-table :source_table) (cond-> m
+ (vector? v) (update k *import-table-fk*)
+ (portable-id? v) (assoc :source-table (str "card__" (*import-fk* v 'Card))))
+ :source-field (cond-> m
+ (vector? v)
+ (update k *import-field-fk*))
+ :source-card (cond-> m
+ (portable-id? v)
+ (update k *import-fk* 'Card))
+ :card-id (cond-> m
+ (portable-id? v)
+ (update k *import-fk* 'Card))
+ :snippet-id (cond-> m
+ (portable-id? v)
+ (update k *import-fk* 'NativeQuerySnippet))
+ #_else (update m k import-mbql*)))
+ m
+ m))
+
+(defn- import-mbql-update-refs
[entity]
(lib.util.match/replace-lite entity
[#{:field "field"} (opts :guard map?) (fully-qualified-name :guard vector?)]
- [:field (mbql-fully-qualified-names->ids* opts) (*import-field-fk* fully-qualified-name)]
+ [:field (import-mbql-map opts) (*import-field-fk* fully-qualified-name)]
;; legacy field refs, still used in parameters and result metadata `field_ref`
[#{:field "field"} (fully-qualified-name :guard vector?) (opts :guard (some-fn map? nil))]
- [:field (*import-field-fk* fully-qualified-name) (some-> opts mbql-fully-qualified-names->ids*)]
+ [:field (*import-field-fk* fully-qualified-name) (some-> opts import-mbql-update-refs)]
;; MBQL 3 `:field-id` can (allegedly) still show up sometimes? Support it just in case.
[(tag :guard #{:field :field-id "field" "field-id"}) (id :guard vector?)]
[:field (*import-field-fk* id) nil]
- ;; source-field is also used within parameter mapping dimensions
- ;; example relevant clause - [:field 2 {:source-field 1}]
- {:source-field (fully-qualified-name :guard vector?)}
- (assoc &match :source-field (*import-field-fk* fully-qualified-name))
-
- {:database (fully-qualified-name :guard string?)}
- (-> &match
- (assoc :database (if (= fully-qualified-name "database/__virtual")
- lib.schema.id/saved-questions-virtual-database-id
- (t2/select-one-pk :model/Database :name fully-qualified-name)))
- mbql-fully-qualified-names->ids*) ; Process other keys
-
- {:card-id (entity-id :guard portable-id?)}
- (-> &match
- (assoc :card-id (*import-fk* entity-id 'Card))
- mbql-fully-qualified-names->ids*) ; Process other keys
-
[#{:metric "metric"} opts (entity-id :guard portable-id?)]
- [:metric (mbql-fully-qualified-names->ids* opts) (*import-fk* entity-id 'Card)]
+ [:metric (import-mbql-map opts) (*import-fk* entity-id 'Card)]
[#{:segment "segment"} opts (entity-id :guard portable-id?)]
- [:segment (mbql-fully-qualified-names->ids* opts) (*import-fk* entity-id 'Segment)]
+ [:segment (import-mbql-map opts) (*import-fk* entity-id 'Segment)]
[#{:measure "measure"} opts (entity-id :guard portable-id?)]
- [:measure (mbql-fully-qualified-names->ids* opts) (*import-fk* entity-id 'Measure)]
+ [:measure (import-mbql-map opts) (*import-fk* entity-id 'Measure)]
;; support legacy MBQL 4 refs for things like the serialized Audit v2 queries
[#{:metric "metric"} (entity-id :guard portable-id?)]
@@ -1293,42 +1326,35 @@
[:segment (*import-fk* entity-id 'Segment)]
[#{:measure "measure"} (entity-id :guard portable-id?)]
- [:measure (*import-fk* entity-id 'Measure)]
-
- {:source-table (_ :guard vector?)}
- (-> &match
- (update :source-table *import-table-fk*)
- mbql-fully-qualified-names->ids*)
-
- {:source_table (_ :guard vector?)}
- (-> &match
- (update :source_table *import-table-fk*)
- mbql-fully-qualified-names->ids*)
-
- ;; support legacy MBQL 4 for the Audit v2 queries
- {:source-table (id :guard portable-id?)}
- (-> &match
- (assoc :source-table (str "card__" (*import-fk* id 'Card)))
- mbql-fully-qualified-names->ids*)
-
- {:source-card (id :guard portable-id?)}
- (-> &match
- (assoc :source-card (*import-fk* id 'Card))
- mbql-fully-qualified-names->ids*)
-
- {:snippet-id (id :guard portable-id?)}
- (-> &match
- (assoc :snippet-id (*import-fk* id 'NativeQuerySnippet))
- mbql-fully-qualified-names->ids*)))
-
-(defn- mbql-fully-qualified-names->ids
- [entity]
- (mbql-fully-qualified-names->ids* entity))
+ [:measure (*import-fk* entity-id 'Measure)]))
+
+(defn- import-mbql-update-maps [x]
+ (lib.util.match/replace-lite x
+ (m :guard map?)
+ (import-mbql-map m)))
+
+(defn- normalize-imported [x]
+ (when x
+ (try
+ (if (mbql-ref? x)
+ (normalize-mbql-ref x)
+ (lib/normalize x))
+ (catch Throwable e
+ (log/warn e "Error normalizing imported MBQL")
+ x))))
+
+(defn- import-mbql*
+ [x]
+ (-> x
+ import-mbql-update-refs
+ import-mbql-update-maps))
(defn import-mbql
"Given an MBQL expression as an EDN structure with portable IDs embedded, convert the IDs back to raw numeric IDs."
- [exported]
- (mbql-fully-qualified-names->ids exported))
+ [x]
+ (-> x
+ import-mbql*
+ normalize-imported))
(declare ^:private mbql-deps-map)
@@ -1404,15 +1430,15 @@
(`[db schema table field]`) back into raw IDs."
[mappings]
(->> mappings
- (map mbql-fully-qualified-names->ids)
+ (map import-mbql)
(map #(m/update-existing % :card_id *import-fk* 'Card))))
-(defn export-parameters
+(mu/defn export-parameters
"Given the :parameter field of a `Card` or `Dashboard`, as a vector of maps, converts
it to a portable form with the CardIds/FieldIds replaced with `[db schema table field]` references.
Parameters are sorted by `:id` for stable serialization output. A `:position` field is added
to preserve display order through the sort."
- [parameters]
+ [parameters :- [:maybe [:sequential :map]]]
(->> parameters
(map-indexed (fn [i p] (assoc p :position i)))
(sort-by :id)
@@ -1425,7 +1451,7 @@
[parameters]
(let [params (for [param parameters]
(-> param
- mbql-fully-qualified-names->ids
+ import-mbql
(m/update-existing-in [:values_source_config :card_id] *import-fk* 'Card)))]
(if (every? :position params)
(sort-by :position params)
@@ -1479,13 +1505,13 @@
export-mbql
json/encode))
-(defn- json-mbql-fully-qualified-names->ids
+(defn- json-import-mbql
"Converts fully qualified names to IDs in MBQL embedded inside a JSON string.
Returns a new JSON string with the IDs converted inside."
[json-str]
(-> json-str
json/decode+kw
- mbql-fully-qualified-names->ids
+ import-mbql
json/encode))
(defn- export-viz-click-behavior-link
@@ -1511,24 +1537,27 @@
(defn- import-viz-click-behavior-mapping [mapping]
(-> mapping
- (m/update-existing :id json-mbql-fully-qualified-names->ids)
- (m/update-existing-in [:target :id] json-mbql-fully-qualified-names->ids)
- (m/update-existing-in [:target :dimension] mbql-fully-qualified-names->ids)))
+ (m/update-existing :id json-import-mbql)
+ (m/update-existing-in [:target :id] json-import-mbql)
+ (m/update-existing-in [:target :dimension] import-mbql)))
(defn- export-viz-click-behavior-mappings
"The `:parameterMapping` on a `:click_behavior` viz settings is a map of... IDs turned into JSON strings which have
been keywordized. Therefore the keys must be converted to strings, parsed, exported, and JSONified. The values are
ported by [[export-viz-click-behavior-mapping]]."
[mappings]
- (into {} (for [[kw-key mapping] mappings
- ;; Mapping keyword shouldn't been a keyword in the first place, it's just how it's processed after
- ;; being selected from db. In an ideal world we'd either have different data layout for
- ;; click_behavior or not convert it's keys to a keywords. We need its full content here.
- :let [k (u/qualified-name kw-key)]]
- (if (mb.viz/dimension-param-mapping? mapping)
- [(json-export-mbql k)
- (export-viz-click-behavior-mapping mapping)]
- [k mapping]))))
+ (into {}
+ (comp (map (fn [[k v]]
+ ;; Mapping keyword shouldn't been a keyword in the first place, it's just how it's processed
+ ;; after being selected from db. In an ideal world we'd either have different data layout for
+ ;; click_behavior or not convert it's keys to a keywords. We need its full content here.
+ [(u/qualified-name k) v]))
+ (map (fn [[k mapping]]
+ (if (mb.viz/dimension-param-mapping? mapping)
+ [(json-export-mbql k)
+ (export-viz-click-behavior-mapping mapping)]
+ [k mapping]))))
+ mappings))
(defn- import-viz-click-behavior-mappings
"The exported form of `:parameterMapping` on a `:click_behavior` viz settings is a map of JSON strings which contain
@@ -1537,7 +1566,7 @@
[mappings]
(into {} (for [[json-key mapping] mappings]
(if (mb.viz/dimension-param-mapping? mapping)
- [(keyword (json-mbql-fully-qualified-names->ids json-key))
+ [(json-import-mbql json-key)
(import-viz-click-behavior-mapping mapping)]
[json-key mapping]))))
@@ -1558,23 +1587,8 @@
(defn- import-pivot-table [settings]
(some-> settings
- (m/update-existing-in [:pivot_table.column_split :rows] mbql-fully-qualified-names->ids)
- (m/update-existing-in [:pivot_table.column_split :columns] mbql-fully-qualified-names->ids)))
-
-(defn- export-visualizations [entity]
- (lib.util.match/replace-lite entity
- [#{"field" :field} (opts :guard map?) (id :guard pos-int?)] [:field (export-visualizations opts) (*export-field-fk* id)]
- ;; legacy refs (MBQL 4)
- [#{"field" :field} (id :guard pos-int?) opts] [:field (*export-field-fk* id) (export-visualizations opts)]
- ;; super-legacy refs (MBQL 3)... not sure these can even still show up IRL
- [#{"field-id" :field-id} (id :guard pos-int?)] [:field-id (*export-field-fk* id)]
- [#{"field-id" :field-id} (id :guard pos-int?) opts] [:field-id (*export-field-fk* id) (export-visualizations opts)]
-
- (_ :guard map?)
- (m/map-vals export-visualizations &match)
-
- (_ :guard vector?)
- (mapv export-visualizations &match)))
+ (m/update-existing-in [:pivot_table.column_split :rows] import-mbql)
+ (m/update-existing-in [:pivot_table.column_split :columns] import-mbql)))
(defn- export-column-settings
"Column settings use a JSON-encoded string as a map key, and it contains field numbers.
@@ -1582,7 +1596,7 @@
[settings]
(when settings
(-> settings
- (update-keys #(-> % json/decode export-visualizations json/encode))
+ (update-keys #(-> % json/decode export-mbql json/encode))
(update-vals export-viz-click-behavior))))
(defn- export-card-dimension-ref
@@ -1624,7 +1638,7 @@
[settings]
(when settings
(-> settings
- export-visualizations
+ export-mbql
export-viz-link-card
export-viz-click-behavior
export-visualizer-settings
diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.clj
new file mode 100644
index 00000000000..9709d459ef5
--- /dev/null
+++ b/test/metabase/models/serialization_test.clj
@@ -0,0 +1,196 @@
+(ns metabase.models.serialization-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.models.serialization :as serdes]))
+
+(deftest ^:parallel drop-mbql-5-uuids-on-export-test
+ (binding [serdes/*export-field-fk* (constantly ::field-id)]
+ (is (= [:field {} :metabase.models.serialization-test/field-id]
+ (#'serdes/export-mbql-ref [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1])))
+ (binding [serdes/*required-lib-uuids-for-export* #{"00000000-0000-0000-0000-000000000001"}]
+ (is (= [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} :metabase.models.serialization-test/field-id]
+ (#'serdes/export-mbql-ref [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1])))
+ (is (= [:field {} :metabase.models.serialization-test/field-id]
+ (#'serdes/export-mbql-ref [:field {:lib/uuid "00000000-0000-0000-0000-000000000002"} 1]))))))
+
+(deftest ^:parallel drop-mbql-5-uuids-on-export-test-2
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/filter (lib/= (meta/field-metadata :venues :id) 1))
+ (lib/aggregate (-> (lib/count)
+ (lib/update-options assoc :lib/uuid "00000000-0000-0000-0000-000000000000")))
+ (as-> $query (lib/order-by $query (first (lib/aggregations-metadata $query)))))]
+ (is (= #{"00000000-0000-0000-0000-000000000000"}
+ (#'serdes/collect-required-lib-uuids query)))
+ (binding [serdes/*export-database-fk* (constantly "DATABASE")
+ serdes/*export-table-fk* (constantly ["DATABASE" "SCHEMA" "TABLE"])
+ serdes/*export-field-fk* (constantly ["DATABASE" "SCHEMA" "TABLE" "FIELD"])]
+ (is (= {:lib/type :mbql/query
+ :database "DATABASE"
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table ["DATABASE" "SCHEMA" "TABLE"]
+ :filters [[:=
+ {}
+ [:field
+ {:effective-type :type/BigInteger, :base-type :type/BigInteger}
+ ["DATABASE" "SCHEMA" "TABLE" "FIELD"]]
+ 1]]
+ :aggregation [[:count {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]
+ :order-by [[:asc
+ {}
+ [:aggregation
+ {:base-type :type/Integer
+ :effective-type :type/Integer
+ :lib/source-name "count"}
+ "00000000-0000-0000-0000-000000000000"]]]}]}
+ (serdes/export-mbql query))))))
+
+(deftest ^:parallel hydrate-mbql-5-uuids-on-import-test
+ ;; when read out of the YAML, map keys should get keywordized but not other strings
+ (let [query {:lib/type "mbql/query"
+ :database "DATABASE"
+ :stages [{:lib/type "mbql.stage/mbql"
+ :source-table ["DATABASE" "SCHEMA" "TABLE"]
+ :filters [["="
+ {}
+ ["field"
+ {:effective-type "type/BigInteger", :base-type "type/BigInteger"}
+ ["DATABASE" "SCHEMA" "TABLE" "FIELD"]]
+ 1]]
+ :aggregation [["count" {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]
+ :order-by [["asc"
+ {}
+ [:aggregation {:base-type "type/Integer"
+ :effective-type "type/Integer"
+ :lib/source-name "count"}
+ "00000000-0000-0000-0000-000000000000"]]]}]}]
+ (binding [serdes/*import-database-fk* (constantly 1)
+ serdes/*import-table-fk* (constantly 2)
+ serdes/*import-field-fk* (constantly 3)]
+ (is (=? {:lib/type :mbql/query
+ :database 1
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table 2
+ :filters [[:=
+ {:lib/uuid string?}
+ [:field
+ {:lib/uuid string?
+ :effective-type :type/BigInteger
+ :base-type :type/BigInteger}
+ 3]
+ 1]]
+ :aggregation [[:count {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]
+ :order-by [[:asc
+ {:lib/uuid string?}
+ [:aggregation
+ {:base-type :type/Integer
+ :effective-type :type/Integer
+ :lib/source-name "count"}
+ "00000000-0000-0000-0000-000000000000"]]]}]}
+ (serdes/import-mbql query))))))
+
+(deftest ^:parallel hydrate-mbql-5-uuids-on-import-test-2
+ (binding [serdes/*import-field-fk* (constantly 3)]
+ (are [x expected] (=? expected
+ (serdes/import-mbql x))
+ ["field" {} ["DB" "SCHEMA" "TABLE" "FIELD"]]
+ [:field {:lib/uuid string?} 3]
+
+ ["dimension" ["field" ["DB" "SCHEMA" "TABLE" "FIELD"] {:source-field ["DB" "SCHEMA" "TABLE" "FIELD2"]}]]
+ [:dimension [:field 3 {:source-field 3}]])))
+
+(deftest ^:parallel export-mbql-3-field-id-ref-in-viz-settings-test
+ (testing "Allegedly viz settings can still contain MBQL 3 `:field-id` refs, make sure we export them properly"
+ (binding [serdes/*export-field-fk* (constantly ["A" "B" "C" "D"])]
+ (are [clause expected] (= expected
+ (#'serdes/export-mbql clause))
+
+ [:field-id 1]
+ [:field-id ["A" "B" "C" "D"]]
+
+ ["field-id" 1]
+ [:field-id ["A" "B" "C" "D"]]
+
+ [:fk-> [:field-id 1] [:field-id 2]]
+ [:fk-> [:field-id ["A" "B" "C" "D"]] [:field-id ["A" "B" "C" "D"]]]
+
+ ["fk->" ["field-id" 1] ["field-id" 2]]
+ ["fk->" [:field-id ["A" "B" "C" "D"]] [:field-id ["A" "B" "C" "D"]]]))))
+
+(deftest ^:parallel export-visualization-settings-test
+ (binding [serdes/*export-field-fk* (constantly ["A" "B" "C" "D"])
+ serdes/*export-fk* (fn [id model]
+ (format "%s___%d" (name model) id))]
+ (is (= {:column_settings
+ {"[\"ref\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],null]]"
+ {:column_title "Locus"
+ :click_behavior
+ {:type "link"
+ :linkType "question"
+ :targetId "Card___1"
+ :parameterMapping
+ {"[\"dimension\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],{\"source-field\":[\"A\",\"B\",\"C\",\"D\"]}]]"
+ {:id "[\"dimension\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],{\"source-field\":[\"A\",\"B\",\"C\",\"D\"]}]]"
+ :source {:type "column", :id "Category_ID", :name "Category ID"}
+ :target {:type "dimension"
+ :id "[\"dimension\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],{\"source-field\":[\"A\",\"B\",\"C\",\"D\"]}]]"
+ :dimension [:dimension [:field ["A" "B" "C" "D"] {:source-field ["A" "B" "C" "D"]}]]}}}}}}}
+ (serdes/export-visualization-settings
+ {:column_settings
+ {"[\"ref\",[\"field\",54,null]]"
+ {:column_title "Locus"
+ :click_behavior
+ {:type "link"
+ :linkType "question"
+ :targetId 1
+ :parameterMapping
+ {(keyword "[\"dimension\",[\"field\",54,{\"source-field\":53}]]")
+ {:id "[\"dimension\",[\"field\",54,{\"source-field\":53}]]"
+ :source {:type "column", :id "Category_ID", :name "Category ID"}
+ :target {:type "dimension"
+ :id "[\"dimension\",[\"field\",54,{\"source-field\":53}]]"
+ :dimension ["dimension" [:field 54 {:source-field 53}]]}}}}}}})))))
+
+(deftest ^:parallel import-viz-settings-test
+ (binding [serdes/*import-field-fk* (constantly 3)]
+ (is (= {:column_settings
+ {"[\"ref\",[\"field\",3,null]]"
+ {:click_behavior
+ {:parameterMapping
+ {"[\"dimension\",[\"field\",3,{\"source-field\":3}]]"
+ {:id "[\"dimension\",[\"field\",3,{\"source-field\":3}]]"
+ :target {:type "dimension"
+ :dimension [:dimension [:field 3 {:source-field 3}]]
+ :id "[\"dimension\",[\"field\",3,{\"source-field\":3}]]"}}}}}}}
+ (serdes/import-visualization-settings
+ {:column_settings
+ {"[\"ref\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],null]]"
+ {:click_behavior
+ {:parameterMapping
+ {"[\"dimension\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],{\"source-field\":[\"my-db\",null,\"orders\",\"subtotal\"]}]]"
+ {:id "[\"dimension\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],{\"source-field\":[\"my-db\",null,\"orders\",\"subtotal\"]}]]"
+ :target {:type "dimension"
+ :dimension [:dimension
+ [:field
+ ["my-db" nil "orders" "invoice"]
+ {:source-field ["my-db" nil "orders" "subtotal"]}]]
+ :id "[\"dimension\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],{\"source-field\":[\"my-db\",null,\"orders\",\"subtotal\"]}]]"}}}}}}})))))
+
+(deftest ^:parallel export-parameters-test
+ (binding [serdes/*export-fk* (fn [id model]
+ (format "%s___%d" (name model) id))
+ serdes/*export-field-fk* (constantly ["DATABASE" "SCHEMA" "TABLE" "FIELD"])]
+ (is (= [{:id "abc"
+ :name "CATEGORY"
+ :position 0
+ :type :category
+ :values_source_config {:card_id "Card___1"
+ :value_field [:field ["DATABASE" "SCHEMA" "TABLE" "FIELD"] nil]}
+ :values_source_type :card}]
+ (serdes/export-parameters [{:id "abc"
+ :type :category
+ :name "CATEGORY"
+ :values_source_type :card
+ :values_source_config {:card_id 1, :value_field [:field 53 nil]}
+ :position 0}])))))
diff --git a/test/metabase/models/visualization_settings_test.cljc b/test/metabase/models/visualization_settings_test.cljc
index 3bc1c2c6503..651879fd43b 100644
--- a/test/metabase/models/visualization_settings_test.cljc
+++ b/test/metabase/models/visualization_settings_test.cljc
@@ -12,7 +12,7 @@
(def fmt #?(:clj format :cljs gstring/format))
-(t/deftest parse-column-ref-strings-test
+(t/deftest ^:parallel parse-column-ref-strings-test
(t/testing "Column ref strings are parsed correctly"
(let [f-qual-nm "/databases/MY_DB/tables/MY_TBL/fields/COL1"
f-id 42
@@ -27,7 +27,7 @@
{::mb.viz/column-name expn-nm}]]]
(t/is (= expected (mb.viz/parse-db-column-ref input-str)))))))
-(t/deftest form-conversion-test
+(t/deftest ^:parallel form-conversion-test
(t/testing ":visualization_settings are correctly converted from DB to qualified form and back"
(let [f-id 42
target-id 19
@@ -72,21 +72,22 @@
;; for a non-table card, the :click_behavior map is directly underneath :visualization_settings
(t/is (= norm-click-bhvr-map (mb.viz/db->norm db-click-bhv-map))))))
-(t/deftest form-transformation-test
+(t/deftest ^:parallel form-transformation-test
(t/testing "The deprecated k:mm :time_style is converted to HH:mm when normalized, and kept in the new style when converted back to the DB form"
(let [db-col-settings-old {:column_settings {"[\"name\",\"Column Name\"]" {:time_style "k:mm"}}}
db-col-settings-new {:column_settings {"[\"name\",\"Column Name\"]" {:time_style "HH:mm"}}}
norm-col-settings {::mb.viz/column-settings {{::mb.viz/column-name "Column Name"} {::mb.viz/time-style "HH:mm"}}}]
(t/is (= norm-col-settings (mb.viz/db->norm db-col-settings-old)))
- (t/is (= db-col-settings-new (mb.viz/norm->db norm-col-settings)))))
+ (t/is (= db-col-settings-new (mb.viz/norm->db norm-col-settings))))))
+(t/deftest ^:parallel form-transformation-test-2
(t/testing "Invalid column refs are dropped when viz settings are normalized (#18972)"
(t/is (= {::mb.viz/column-settings {}}
(mb.viz/db->norm {:column_settings {"[\"ref\",null]" {:column_title "invalid"}}})))
(t/is (= {::mb.viz/column-settings {}}
(mb.viz/db->norm {:column_settings {"bad-column-ref" {:column_title "invalid"}}})))))
-(t/deftest virtual-card-test
+(t/deftest ^:parallel virtual-card-test
(t/testing "Virtual card in visualization settings is preserved through normalization roundtrip"
;; virtual cards have the form of a regular card, mostly
(let [db-form {:virtual_card {:archived false
@@ -105,7 +106,7 @@
mb.viz/db->norm
mb.viz/norm->db))))))
-(t/deftest parameter-mapping-test
+(t/deftest ^:parallel parameter-mapping-test
(t/testing "parameterMappings are handled correctly"
(let [from-id 101
to-id 294
@@ -149,7 +150,7 @@
(walk/postwalk (fn [v] (when (keyword? v) (swap! all-kws #(conj % v)))) m)
@all-kws))
-(t/deftest comprehensive-click-actions-test
+(t/deftest ^:parallel comprehensive-click-actions-test
(t/testing "Visualization settings for card in 'EE14566 - Click Behavior visualization_settings' should be handled"
(let [db-form {:column_settings
{"[\"name\",\"Year\"]"
diff --git a/test_resources/serialization_baseline/collections/main/source_question.yaml b/test_resources/serialization_baseline/collections/main/source_question.yaml
index afb065c7f06..a05e3b1ebad 100644
--- a/test_resources/serialization_baseline/collections/main/source_question.yaml
+++ b/test_resources/serialization_baseline/collections/main/source_question.yaml
@@ -9,7 +9,6 @@ parameters: []
parameter_mappings: []
dataset_query:
"lib/type": mbql/query
- "lib/metadata": null
database: My Database
stages:
- "lib/type": mbql.stage/native
diff --git a/test_resources/serialization_baseline/collections/transforms/test_transform.yaml b/test_resources/serialization_baseline/collections/transforms/test_transform.yaml
index 27048291495..bd685a14e57 100644
--- a/test_resources/serialization_baseline/collections/transforms/test_transform.yaml
+++ b/test_resources/serialization_baseline/collections/transforms/test_transform.yaml
@@ -6,7 +6,6 @@ name: Test Transform
source:
query:
"lib/type": mbql/query
- "lib/metadata": null
database: My Database
stages:
- "lib/type": mbql.stage/mbql
diff --git a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml
index b7ae943d817..a7f2ceec0dc 100644
--- a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml
+++ b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml
@@ -3,15 +3,13 @@ creator_id: [email protected]
definition:
database: My Database
"lib/type": mbql/query
- "lib/metadata": null
stages:
- "lib/type": mbql.stage/mbql
aggregation:
- - sum
- - "lib/uuid": 865abdef-bb71-490c-9796-fa5b01bf16a2
+ - {}
- - field
- - "lib/uuid": c115167e-2961-4ff1-b2e9-8affbff1a20b
- base-type: type/Text
+ - base-type: type/Text
- - My Database
- null
- Schemaless Table
diff --git a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml
index 28697330fd4..48f463ebace 100644
--- a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml
+++ b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml
@@ -2,15 +2,14 @@ created_at: '2024-08-28T14:28:03.451106Z'
creator_id: [email protected]
definition:
"lib/type": mbql/query
- "lib/metadata": null
database: My Database
stages:
- "lib/type": mbql.stage/mbql
filters:
- - <
- - "lib/uuid": eb938c48-cdf5-49b4-8652-cb816e281a8b
+ - {}
- - field
- - "lib/uuid": cfe7c17c-789d-4486-9592-a26f49f06992
+ - {}
- - My Database
- null
- Schemaless Table