The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Omit UUIDs in SerDes MBQL 5 export unless required (#71862)
Cam Saul22a21845f145
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.cljindex 886009194d5..54abb36a289 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/test_util_test.clj@@ -21,7 +21,7 @@:aggregation [["cum-count"]]:breakout [[:field created-id {"temporal-unit" "week"}]]:filter ["<" [:field nps-id nil] 9]}}- (#'serdes/mbql-fully-qualified-names->ids+ (#'serdes/import-mbql{:database "Metabase Store",:type "query",:query {:source-table ["Metabase Store" "public" "crm_survey_response"],diff --git a/src/metabase/lib/normalize.cljc b/src/metabase/lib/normalize.cljcindex 000bc785aad..4ca9021a406 100644--- a/src/metabase/lib/normalize.cljc+++ b/src/metabase/lib/normalize.cljc@@ -97,7 +97,12 @@(throw (ex-info (i18n/tru "Normalization error"){:schema schema, :x x, :error error})))](thunk))- (thunk)))))+ (try+ (thunk)+ (catch #?(:clj Throwable :cljs :default) e+ (throw (ex-info (str "Uncaught normalization error: " (ex-message e))+ {:schema schema}+ e))))))))(mu/defn ->normalized-stage-metadata :- ::lib.schema.metadata/stage"Take a sequence of legacy or Lib metadata maps, convert to Lib-style if needed, then normalize them.diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex a8c138453d7..3a7aa97d404 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -65,6 +65,7 @@;; legacy usages -- do not use in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.util.match :as lib.util.match]@@ -956,6 +957,7 @@`(try~@body(catch clojure.lang.ExceptionInfo e#+ (log/debugf e# "Caught error in fk-elide")(when-not (= (::type (ex-data e#)) :target-not-found)(throw e#))nil)))@@ -1149,16 +1151,24 @@;;; ## MBQL Fields-(mu/defn- mbql-clause-tag :- [:maybe [:enum :field :dimension :metric :segment :measure]]+(mu/defn- mbql-ref? :- [:maybe [:enum :field :field-id :dimension :metric :segment :measure]]"Is given form an MBQL entity reference?"[form](when (and (vector? form)- (#{:field :dimension :metric :segment :measure} (keyword (first form))))+ (#{:field :field-id :dimension :metric :segment :measure} (keyword (first form))))(keyword (first form))))-(defn- normalize [mbql]- (let [tag (mbql-clause-tag mbql)+(mr/def ::mbql-3-field-id-ref+ "Ultra-legacy MBQL 3 `:field-id` ref; which (allegedly) can show up in viz settings; this schema is only here so we+ can use Lib normalization in [[normalize-mbql-ref]]."+ [:tuple+ [:= {:decode/normalize lib.schema.common/normalize-keyword} :field-id]+ pos-int?])++(defn- normalize-mbql-ref [mbql]+ (let [tag (mbql-ref? mbql)schema (case tag+ :field-id ::mbql-3-field-id-ref:field [:multi{:dispatch #(and (vector? %)(map? (second %)))}@@ -1172,71 +1182,81 @@(cond->> mbqlschema (lib/normalize schema mbql))))-(defn- mbql-id->fully-qualified-name+(def ^:private ^:dynamic *required-lib-uuids-for-export* nil)++(mu/defn- collect-required-lib-uuids :- [:set ::lib.schema.common/uuid]+ [x]+ (set+ (lib.util.match/match-many x+ [:aggregation (_opts :guard map?) (uuid :guard string?)]+ uuid)))++(declare export-mbql)++(defn- export-mbql-map [m]+ (reduce-kv+ (fn [m k v]+ (case k+ :lib/metadata (dissoc m k)+ :database (update m k *export-database-fk*)+ (:card_id :card-id) (update m k *export-fk* :model/Card) ; attributes that refer to db fields use `_`; template-tags use `-`+ (:source_table :source-table) (cond-> m+ (pos-int? v)+ (update k *export-table-fk*))+ (:source_card :source-card) (cond-> m+ (pos-int? v)+ (update k *export-fk* :model/Card))+ :source-field (cond-> m+ (pos-int? v)+ (update k *export-field-fk*))+ ::mb.viz/param-mapping-source (update m k *export-field-fk*)+ :segment (update m k *export-fk* :model/Segment)+ :snippet-id (update m k *export-fk* :model/NativeQuerySnippet)+ :lib/uuid (cond-> m+ (not (contains? *required-lib-uuids-for-export* v))+ (dissoc :lib/uuid))+ #_else (update m k export-mbql)))+ m+ m))++(defn- export-mbql-ref[mbql]- (lib.util.match/replace-lite (normalize mbql)+ (lib.util.match/replace-lite (normalize-mbql-ref mbql);; `pos-int?` guard is here to make the operation idempotent[:field (opts :guard map?) (id :guard pos-int?)]- [:field (mbql-id->fully-qualified-name opts) (*export-field-fk* id)]+ [:field (export-mbql-map opts) (*export-field-fk* id)];; legacy (MBQL 4) field refs are still supported in parameter targets and in result metadata `field_ref`...[:field (id :guard pos-int?) (opts :guard (some-fn map? nil?))]- [:field (*export-field-fk* id) (mbql-id->fully-qualified-name opts)]+ [:field (*export-field-fk* id) (export-mbql-map opts)];; MBQL 3 `:field-id` can (allegedly) still show up sometimes? Support it just in case.[(tag :guard #{:field :field-id}) (id :guard pos-int?)][tag (*export-field-fk* id)]- {:source-table (id :guard pos-int?)}- (assoc &match :source-table (*export-table-fk* id))-- ;; source-field is also used within parameter mapping dimensions- ;; example relevant clause - [:field 2 {:source-field 1}]- {:source-field (id :guard pos-int?)}- (assoc &match :source-field (*export-field-fk* id))-[:dimension (dim :guard vector?)]- [:dimension (mbql-id->fully-qualified-name dim)]+ [:dimension (export-mbql-ref dim)][:metric opts (id :guard pos-int?)]- [:metric (mbql-id->fully-qualified-name opts) (*export-fk* id 'Card)]+ [:metric (export-mbql-map opts) (*export-fk* id 'Card)][:segment opts (id :guard pos-int?)]- [:segment (mbql-id->fully-qualified-name opts) (*export-fk* id 'Segment)]+ [:segment (export-mbql-map opts) (*export-fk* id 'Segment)][:measure opts (id :guard pos-int?)]- [:measure (mbql-id->fully-qualified-name opts) (*export-fk* id 'Measure)]))+ [:measure (export-mbql-map opts) (*export-fk* id 'Measure)]))(defn export-mbql"Given an MBQL expression, convert it to an EDN structure and turn the non-portable Database, Table and Field IDsinside it into portable references."- [entity]- (lib.util.match/replace-lite entity- (_ :guard mbql-clause-tag)- (mbql-id->fully-qualified-name &match)-- (_ :guard sequential?)- (mapv export-mbql &match)-- (_ :guard map?)- (reduce-kv- (fn [entity k _v]- (let [f (case k- :database (fn [db-id]- (if (= db-id lib.schema.id/saved-questions-virtual-database-id)- "database/__virtual"- (t2/select-one-fn :name :model/Database :id db-id)))- (:card_id :card-id) #(*export-fk* % :model/Card) ; attributes that refer to db fields use `_`; template-tags use `-`- (:source_table :source-table) *export-table-fk*- (:source_card :source-card) #(*export-fk* % :model/Card)- ::mb.viz/param-mapping-source *export-field-fk*- :segment #(*export-fk* % :model/Segment)- :snippet-id #(*export-fk* % :model/NativeQuerySnippet)- :lib/metadata (constantly nil)- #_else export-mbql)]- (update entity k f)))- &match- &match)))+ [x]+ ;; if required UUIDs are already calculated don't recalculate when we recurse.+ (binding [*required-lib-uuids-for-export* (or *required-lib-uuids-for-export* (collect-required-lib-uuids x))]+ (cond+ (mbql-ref? x) (export-mbql-ref x)+ (sequential? x) (mapv export-mbql x)+ (map? x) (export-mbql-map x)+ :else x)))(defn- portable-id?"True if the provided string is either an Entity ID or identity-hash string."@@ -1245,45 +1265,58 @@(or (entity-id? s)(identity-hash? s))))-(defn- mbql-fully-qualified-names->ids*+(declare import-mbql*)++(defn- import-mbql-map [m]+ (reduce-kv+ (fn [m k v]+ (case k+ :database (cond-> m+ (string? v)+ (update k (if (= v "database/__virtual")+ (constantly lib.schema.id/saved-questions-virtual-database-id)+ *import-database-fk*)))+ (:source-table :source_table) (cond-> m+ (vector? v) (update k *import-table-fk*)+ (portable-id? v) (assoc :source-table (str "card__" (*import-fk* v 'Card))))+ :source-field (cond-> m+ (vector? v)+ (update k *import-field-fk*))+ :source-card (cond-> m+ (portable-id? v)+ (update k *import-fk* 'Card))+ :card-id (cond-> m+ (portable-id? v)+ (update k *import-fk* 'Card))+ :snippet-id (cond-> m+ (portable-id? v)+ (update k *import-fk* 'NativeQuerySnippet))+ #_else (update m k import-mbql*)))+ m+ m))++(defn- import-mbql-update-refs[entity](lib.util.match/replace-lite entity[#{:field "field"} (opts :guard map?) (fully-qualified-name :guard vector?)]- [:field (mbql-fully-qualified-names->ids* opts) (*import-field-fk* fully-qualified-name)]+ [:field (import-mbql-map opts) (*import-field-fk* fully-qualified-name)];; legacy field refs, still used in parameters and result metadata `field_ref`[#{:field "field"} (fully-qualified-name :guard vector?) (opts :guard (some-fn map? nil))]- [:field (*import-field-fk* fully-qualified-name) (some-> opts mbql-fully-qualified-names->ids*)]+ [:field (*import-field-fk* fully-qualified-name) (some-> opts import-mbql-update-refs)];; MBQL 3 `:field-id` can (allegedly) still show up sometimes? Support it just in case.[(tag :guard #{:field :field-id "field" "field-id"}) (id :guard vector?)][:field (*import-field-fk* id) nil]- ;; source-field is also used within parameter mapping dimensions- ;; example relevant clause - [:field 2 {:source-field 1}]- {:source-field (fully-qualified-name :guard vector?)}- (assoc &match :source-field (*import-field-fk* fully-qualified-name))-- {:database (fully-qualified-name :guard string?)}- (-> &match- (assoc :database (if (= fully-qualified-name "database/__virtual")- lib.schema.id/saved-questions-virtual-database-id- (t2/select-one-pk :model/Database :name fully-qualified-name)))- mbql-fully-qualified-names->ids*) ; Process other keys-- {:card-id (entity-id :guard portable-id?)}- (-> &match- (assoc :card-id (*import-fk* entity-id 'Card))- mbql-fully-qualified-names->ids*) ; Process other keys-[#{:metric "metric"} opts (entity-id :guard portable-id?)]- [:metric (mbql-fully-qualified-names->ids* opts) (*import-fk* entity-id 'Card)]+ [:metric (import-mbql-map opts) (*import-fk* entity-id 'Card)][#{:segment "segment"} opts (entity-id :guard portable-id?)]- [:segment (mbql-fully-qualified-names->ids* opts) (*import-fk* entity-id 'Segment)]+ [:segment (import-mbql-map opts) (*import-fk* entity-id 'Segment)][#{:measure "measure"} opts (entity-id :guard portable-id?)]- [:measure (mbql-fully-qualified-names->ids* opts) (*import-fk* entity-id 'Measure)]+ [:measure (import-mbql-map opts) (*import-fk* entity-id 'Measure)];; support legacy MBQL 4 refs for things like the serialized Audit v2 queries[#{:metric "metric"} (entity-id :guard portable-id?)]@@ -1293,42 +1326,35 @@[:segment (*import-fk* entity-id 'Segment)][#{:measure "measure"} (entity-id :guard portable-id?)]- [:measure (*import-fk* entity-id 'Measure)]-- {:source-table (_ :guard vector?)}- (-> &match- (update :source-table *import-table-fk*)- mbql-fully-qualified-names->ids*)-- {:source_table (_ :guard vector?)}- (-> &match- (update :source_table *import-table-fk*)- mbql-fully-qualified-names->ids*)-- ;; support legacy MBQL 4 for the Audit v2 queries- {:source-table (id :guard portable-id?)}- (-> &match- (assoc :source-table (str "card__" (*import-fk* id 'Card)))- mbql-fully-qualified-names->ids*)-- {:source-card (id :guard portable-id?)}- (-> &match- (assoc :source-card (*import-fk* id 'Card))- mbql-fully-qualified-names->ids*)-- {:snippet-id (id :guard portable-id?)}- (-> &match- (assoc :snippet-id (*import-fk* id 'NativeQuerySnippet))- mbql-fully-qualified-names->ids*)))--(defn- mbql-fully-qualified-names->ids- [entity]- (mbql-fully-qualified-names->ids* entity))+ [:measure (*import-fk* entity-id 'Measure)]))++(defn- import-mbql-update-maps [x]+ (lib.util.match/replace-lite x+ (m :guard map?)+ (import-mbql-map m)))++(defn- normalize-imported [x]+ (when x+ (try+ (if (mbql-ref? x)+ (normalize-mbql-ref x)+ (lib/normalize x))+ (catch Throwable e+ (log/warn e "Error normalizing imported MBQL")+ x))))++(defn- import-mbql*+ [x]+ (-> x+ import-mbql-update-refs+ import-mbql-update-maps))(defn import-mbql"Given an MBQL expression as an EDN structure with portable IDs embedded, convert the IDs back to raw numeric IDs."- [exported]- (mbql-fully-qualified-names->ids exported))+ [x]+ (-> x+ import-mbql*+ normalize-imported))(declare ^:private mbql-deps-map)@@ -1404,15 +1430,15 @@(`[db schema table field]`) back into raw IDs."[mappings](->> mappings- (map mbql-fully-qualified-names->ids)+ (map import-mbql)(map #(m/update-existing % :card_id *import-fk* 'Card))))-(defn export-parameters+(mu/defn export-parameters"Given the :parameter field of a `Card` or `Dashboard`, as a vector of maps, convertsit to a portable form with the CardIds/FieldIds replaced with `[db schema table field]` references.Parameters are sorted by `:id` for stable serialization output. A `:position` field is addedto preserve display order through the sort."- [parameters]+ [parameters :- [:maybe [:sequential :map]]](->> parameters(map-indexed (fn [i p] (assoc p :position i)))(sort-by :id)@@ -1425,7 +1451,7 @@[parameters](let [params (for [param parameters](-> param- mbql-fully-qualified-names->ids+ import-mbql(m/update-existing-in [:values_source_config :card_id] *import-fk* 'Card)))](if (every? :position params)(sort-by :position params)@@ -1479,13 +1505,13 @@export-mbqljson/encode))-(defn- json-mbql-fully-qualified-names->ids+(defn- json-import-mbql"Converts fully qualified names to IDs in MBQL embedded inside a JSON string.Returns a new JSON string with the IDs converted inside."[json-str](-> json-strjson/decode+kw- mbql-fully-qualified-names->ids+ import-mbqljson/encode))(defn- export-viz-click-behavior-link@@ -1511,24 +1537,27 @@(defn- import-viz-click-behavior-mapping [mapping](-> mapping- (m/update-existing :id json-mbql-fully-qualified-names->ids)- (m/update-existing-in [:target :id] json-mbql-fully-qualified-names->ids)- (m/update-existing-in [:target :dimension] mbql-fully-qualified-names->ids)))+ (m/update-existing :id json-import-mbql)+ (m/update-existing-in [:target :id] json-import-mbql)+ (m/update-existing-in [:target :dimension] import-mbql)))(defn- export-viz-click-behavior-mappings"The `:parameterMapping` on a `:click_behavior` viz settings is a map of... IDs turned into JSON strings which havebeen keywordized. Therefore the keys must be converted to strings, parsed, exported, and JSONified. The values areported by [[export-viz-click-behavior-mapping]]."[mappings]- (into {} (for [[kw-key mapping] mappings- ;; Mapping keyword shouldn't been a keyword in the first place, it's just how it's processed after- ;; being selected from db. In an ideal world we'd either have different data layout for- ;; click_behavior or not convert it's keys to a keywords. We need its full content here.- :let [k (u/qualified-name kw-key)]]- (if (mb.viz/dimension-param-mapping? mapping)- [(json-export-mbql k)- (export-viz-click-behavior-mapping mapping)]- [k mapping]))))+ (into {}+ (comp (map (fn [[k v]]+ ;; Mapping keyword shouldn't been a keyword in the first place, it's just how it's processed+ ;; after being selected from db. In an ideal world we'd either have different data layout for+ ;; click_behavior or not convert it's keys to a keywords. We need its full content here.+ [(u/qualified-name k) v]))+ (map (fn [[k mapping]]+ (if (mb.viz/dimension-param-mapping? mapping)+ [(json-export-mbql k)+ (export-viz-click-behavior-mapping mapping)]+ [k mapping]))))+ mappings))(defn- import-viz-click-behavior-mappings"The exported form of `:parameterMapping` on a `:click_behavior` viz settings is a map of JSON strings which contain@@ -1537,7 +1566,7 @@[mappings](into {} (for [[json-key mapping] mappings](if (mb.viz/dimension-param-mapping? mapping)- [(keyword (json-mbql-fully-qualified-names->ids json-key))+ [(json-import-mbql json-key)(import-viz-click-behavior-mapping mapping)][json-key mapping]))))@@ -1558,23 +1587,8 @@(defn- import-pivot-table [settings](some-> settings- (m/update-existing-in [:pivot_table.column_split :rows] mbql-fully-qualified-names->ids)- (m/update-existing-in [:pivot_table.column_split :columns] mbql-fully-qualified-names->ids)))--(defn- export-visualizations [entity]- (lib.util.match/replace-lite entity- [#{"field" :field} (opts :guard map?) (id :guard pos-int?)] [:field (export-visualizations opts) (*export-field-fk* id)]- ;; legacy refs (MBQL 4)- [#{"field" :field} (id :guard pos-int?) opts] [:field (*export-field-fk* id) (export-visualizations opts)]- ;; super-legacy refs (MBQL 3)... not sure these can even still show up IRL- [#{"field-id" :field-id} (id :guard pos-int?)] [:field-id (*export-field-fk* id)]- [#{"field-id" :field-id} (id :guard pos-int?) opts] [:field-id (*export-field-fk* id) (export-visualizations opts)]-- (_ :guard map?)- (m/map-vals export-visualizations &match)-- (_ :guard vector?)- (mapv export-visualizations &match)))+ (m/update-existing-in [:pivot_table.column_split :rows] import-mbql)+ (m/update-existing-in [:pivot_table.column_split :columns] import-mbql)))(defn- export-column-settings"Column settings use a JSON-encoded string as a map key, and it contains field numbers.@@ -1582,7 +1596,7 @@[settings](when settings(-> settings- (update-keys #(-> % json/decode export-visualizations json/encode))+ (update-keys #(-> % json/decode export-mbql json/encode))(update-vals export-viz-click-behavior))))(defn- export-card-dimension-ref@@ -1624,7 +1638,7 @@[settings](when settings(-> settings- export-visualizations+ export-mbqlexport-viz-link-cardexport-viz-click-behaviorexport-visualizer-settingsdiff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.cljnew file mode 100644index 00000000000..9709d459ef5--- /dev/null+++ b/test/metabase/models/serialization_test.clj@@ -0,0 +1,196 @@+(ns metabase.models.serialization-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta]+ [metabase.models.serialization :as serdes]))++(deftest ^:parallel drop-mbql-5-uuids-on-export-test+ (binding [serdes/*export-field-fk* (constantly ::field-id)]+ (is (= [:field {} :metabase.models.serialization-test/field-id]+ (#'serdes/export-mbql-ref [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1])))+ (binding [serdes/*required-lib-uuids-for-export* #{"00000000-0000-0000-0000-000000000001"}]+ (is (= [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} :metabase.models.serialization-test/field-id]+ (#'serdes/export-mbql-ref [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1])))+ (is (= [:field {} :metabase.models.serialization-test/field-id]+ (#'serdes/export-mbql-ref [:field {:lib/uuid "00000000-0000-0000-0000-000000000002"} 1]))))))++(deftest ^:parallel drop-mbql-5-uuids-on-export-test-2+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/filter (lib/= (meta/field-metadata :venues :id) 1))+ (lib/aggregate (-> (lib/count)+ (lib/update-options assoc :lib/uuid "00000000-0000-0000-0000-000000000000")))+ (as-> $query (lib/order-by $query (first (lib/aggregations-metadata $query)))))]+ (is (= #{"00000000-0000-0000-0000-000000000000"}+ (#'serdes/collect-required-lib-uuids query)))+ (binding [serdes/*export-database-fk* (constantly "DATABASE")+ serdes/*export-table-fk* (constantly ["DATABASE" "SCHEMA" "TABLE"])+ serdes/*export-field-fk* (constantly ["DATABASE" "SCHEMA" "TABLE" "FIELD"])]+ (is (= {:lib/type :mbql/query+ :database "DATABASE"+ :stages [{:lib/type :mbql.stage/mbql+ :source-table ["DATABASE" "SCHEMA" "TABLE"]+ :filters [[:=+ {}+ [:field+ {:effective-type :type/BigInteger, :base-type :type/BigInteger}+ ["DATABASE" "SCHEMA" "TABLE" "FIELD"]]+ 1]]+ :aggregation [[:count {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]+ :order-by [[:asc+ {}+ [:aggregation+ {:base-type :type/Integer+ :effective-type :type/Integer+ :lib/source-name "count"}+ "00000000-0000-0000-0000-000000000000"]]]}]}+ (serdes/export-mbql query))))))++(deftest ^:parallel hydrate-mbql-5-uuids-on-import-test+ ;; when read out of the YAML, map keys should get keywordized but not other strings+ (let [query {:lib/type "mbql/query"+ :database "DATABASE"+ :stages [{:lib/type "mbql.stage/mbql"+ :source-table ["DATABASE" "SCHEMA" "TABLE"]+ :filters [["="+ {}+ ["field"+ {:effective-type "type/BigInteger", :base-type "type/BigInteger"}+ ["DATABASE" "SCHEMA" "TABLE" "FIELD"]]+ 1]]+ :aggregation [["count" {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]+ :order-by [["asc"+ {}+ [:aggregation {:base-type "type/Integer"+ :effective-type "type/Integer"+ :lib/source-name "count"}+ "00000000-0000-0000-0000-000000000000"]]]}]}]+ (binding [serdes/*import-database-fk* (constantly 1)+ serdes/*import-table-fk* (constantly 2)+ serdes/*import-field-fk* (constantly 3)]+ (is (=? {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 2+ :filters [[:=+ {:lib/uuid string?}+ [:field+ {:lib/uuid string?+ :effective-type :type/BigInteger+ :base-type :type/BigInteger}+ 3]+ 1]]+ :aggregation [[:count {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]+ :order-by [[:asc+ {:lib/uuid string?}+ [:aggregation+ {:base-type :type/Integer+ :effective-type :type/Integer+ :lib/source-name "count"}+ "00000000-0000-0000-0000-000000000000"]]]}]}+ (serdes/import-mbql query))))))++(deftest ^:parallel hydrate-mbql-5-uuids-on-import-test-2+ (binding [serdes/*import-field-fk* (constantly 3)]+ (are [x expected] (=? expected+ (serdes/import-mbql x))+ ["field" {} ["DB" "SCHEMA" "TABLE" "FIELD"]]+ [:field {:lib/uuid string?} 3]++ ["dimension" ["field" ["DB" "SCHEMA" "TABLE" "FIELD"] {:source-field ["DB" "SCHEMA" "TABLE" "FIELD2"]}]]+ [:dimension [:field 3 {:source-field 3}]])))++(deftest ^:parallel export-mbql-3-field-id-ref-in-viz-settings-test+ (testing "Allegedly viz settings can still contain MBQL 3 `:field-id` refs, make sure we export them properly"+ (binding [serdes/*export-field-fk* (constantly ["A" "B" "C" "D"])]+ (are [clause expected] (= expected+ (#'serdes/export-mbql clause))++ [:field-id 1]+ [:field-id ["A" "B" "C" "D"]]++ ["field-id" 1]+ [:field-id ["A" "B" "C" "D"]]++ [:fk-> [:field-id 1] [:field-id 2]]+ [:fk-> [:field-id ["A" "B" "C" "D"]] [:field-id ["A" "B" "C" "D"]]]++ ["fk->" ["field-id" 1] ["field-id" 2]]+ ["fk->" [:field-id ["A" "B" "C" "D"]] [:field-id ["A" "B" "C" "D"]]]))))++(deftest ^:parallel export-visualization-settings-test+ (binding [serdes/*export-field-fk* (constantly ["A" "B" "C" "D"])+ serdes/*export-fk* (fn [id model]+ (format "%s___%d" (name model) id))]+ (is (= {:column_settings+ {"[\"ref\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],null]]"+ {:column_title "Locus"+ :click_behavior+ {:type "link"+ :linkType "question"+ :targetId "Card___1"+ :parameterMapping+ {"[\"dimension\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],{\"source-field\":[\"A\",\"B\",\"C\",\"D\"]}]]"+ {:id "[\"dimension\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],{\"source-field\":[\"A\",\"B\",\"C\",\"D\"]}]]"+ :source {:type "column", :id "Category_ID", :name "Category ID"}+ :target {:type "dimension"+ :id "[\"dimension\",[\"field\",[\"A\",\"B\",\"C\",\"D\"],{\"source-field\":[\"A\",\"B\",\"C\",\"D\"]}]]"+ :dimension [:dimension [:field ["A" "B" "C" "D"] {:source-field ["A" "B" "C" "D"]}]]}}}}}}}+ (serdes/export-visualization-settings+ {:column_settings+ {"[\"ref\",[\"field\",54,null]]"+ {:column_title "Locus"+ :click_behavior+ {:type "link"+ :linkType "question"+ :targetId 1+ :parameterMapping+ {(keyword "[\"dimension\",[\"field\",54,{\"source-field\":53}]]")+ {:id "[\"dimension\",[\"field\",54,{\"source-field\":53}]]"+ :source {:type "column", :id "Category_ID", :name "Category ID"}+ :target {:type "dimension"+ :id "[\"dimension\",[\"field\",54,{\"source-field\":53}]]"+ :dimension ["dimension" [:field 54 {:source-field 53}]]}}}}}}})))))++(deftest ^:parallel import-viz-settings-test+ (binding [serdes/*import-field-fk* (constantly 3)]+ (is (= {:column_settings+ {"[\"ref\",[\"field\",3,null]]"+ {:click_behavior+ {:parameterMapping+ {"[\"dimension\",[\"field\",3,{\"source-field\":3}]]"+ {:id "[\"dimension\",[\"field\",3,{\"source-field\":3}]]"+ :target {:type "dimension"+ :dimension [:dimension [:field 3 {:source-field 3}]]+ :id "[\"dimension\",[\"field\",3,{\"source-field\":3}]]"}}}}}}}+ (serdes/import-visualization-settings+ {:column_settings+ {"[\"ref\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],null]]"+ {:click_behavior+ {:parameterMapping+ {"[\"dimension\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],{\"source-field\":[\"my-db\",null,\"orders\",\"subtotal\"]}]]"+ {:id "[\"dimension\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],{\"source-field\":[\"my-db\",null,\"orders\",\"subtotal\"]}]]"+ :target {:type "dimension"+ :dimension [:dimension+ [:field+ ["my-db" nil "orders" "invoice"]+ {:source-field ["my-db" nil "orders" "subtotal"]}]]+ :id "[\"dimension\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],{\"source-field\":[\"my-db\",null,\"orders\",\"subtotal\"]}]]"}}}}}}})))))++(deftest ^:parallel export-parameters-test+ (binding [serdes/*export-fk* (fn [id model]+ (format "%s___%d" (name model) id))+ serdes/*export-field-fk* (constantly ["DATABASE" "SCHEMA" "TABLE" "FIELD"])]+ (is (= [{:id "abc"+ :name "CATEGORY"+ :position 0+ :type :category+ :values_source_config {:card_id "Card___1"+ :value_field [:field ["DATABASE" "SCHEMA" "TABLE" "FIELD"] nil]}+ :values_source_type :card}]+ (serdes/export-parameters [{:id "abc"+ :type :category+ :name "CATEGORY"+ :values_source_type :card+ :values_source_config {:card_id 1, :value_field [:field 53 nil]}+ :position 0}])))))diff --git a/test/metabase/models/visualization_settings_test.cljc b/test/metabase/models/visualization_settings_test.cljcindex 3bc1c2c6503..651879fd43b 100644--- a/test/metabase/models/visualization_settings_test.cljc+++ b/test/metabase/models/visualization_settings_test.cljc@@ -12,7 +12,7 @@(def fmt #?(:clj format :cljs gstring/format))-(t/deftest parse-column-ref-strings-test+(t/deftest ^:parallel parse-column-ref-strings-test(t/testing "Column ref strings are parsed correctly"(let [f-qual-nm "/databases/MY_DB/tables/MY_TBL/fields/COL1"f-id 42@@ -27,7 +27,7 @@{::mb.viz/column-name expn-nm}]]](t/is (= expected (mb.viz/parse-db-column-ref input-str)))))))-(t/deftest form-conversion-test+(t/deftest ^:parallel form-conversion-test(t/testing ":visualization_settings are correctly converted from DB to qualified form and back"(let [f-id 42target-id 19@@ -72,21 +72,22 @@;; for a non-table card, the :click_behavior map is directly underneath :visualization_settings(t/is (= norm-click-bhvr-map (mb.viz/db->norm db-click-bhv-map))))))-(t/deftest form-transformation-test+(t/deftest ^:parallel form-transformation-test(t/testing "The deprecated k:mm :time_style is converted to HH:mm when normalized, and kept in the new style when converted back to the DB form"(let [db-col-settings-old {:column_settings {"[\"name\",\"Column Name\"]" {:time_style "k:mm"}}}db-col-settings-new {:column_settings {"[\"name\",\"Column Name\"]" {:time_style "HH:mm"}}}norm-col-settings {::mb.viz/column-settings {{::mb.viz/column-name "Column Name"} {::mb.viz/time-style "HH:mm"}}}](t/is (= norm-col-settings (mb.viz/db->norm db-col-settings-old)))- (t/is (= db-col-settings-new (mb.viz/norm->db norm-col-settings)))))+ (t/is (= db-col-settings-new (mb.viz/norm->db norm-col-settings))))))+(t/deftest ^:parallel form-transformation-test-2(t/testing "Invalid column refs are dropped when viz settings are normalized (#18972)"(t/is (= {::mb.viz/column-settings {}}(mb.viz/db->norm {:column_settings {"[\"ref\",null]" {:column_title "invalid"}}})))(t/is (= {::mb.viz/column-settings {}}(mb.viz/db->norm {:column_settings {"bad-column-ref" {:column_title "invalid"}}})))))-(t/deftest virtual-card-test+(t/deftest ^:parallel virtual-card-test(t/testing "Virtual card in visualization settings is preserved through normalization roundtrip";; virtual cards have the form of a regular card, mostly(let [db-form {:virtual_card {:archived false@@ -105,7 +106,7 @@mb.viz/db->normmb.viz/norm->db))))))-(t/deftest parameter-mapping-test+(t/deftest ^:parallel parameter-mapping-test(t/testing "parameterMappings are handled correctly"(let [from-id 101to-id 294@@ -149,7 +150,7 @@(walk/postwalk (fn [v] (when (keyword? v) (swap! all-kws #(conj % v)))) m)@all-kws))-(t/deftest comprehensive-click-actions-test+(t/deftest ^:parallel comprehensive-click-actions-test(t/testing "Visualization settings for card in 'EE14566 - Click Behavior visualization_settings' should be handled"(let [db-form {:column_settings{"[\"name\",\"Year\"]"diff --git a/test_resources/serialization_baseline/collections/main/source_question.yaml b/test_resources/serialization_baseline/collections/main/source_question.yamlindex afb065c7f06..a05e3b1ebad 100644--- a/test_resources/serialization_baseline/collections/main/source_question.yaml+++ b/test_resources/serialization_baseline/collections/main/source_question.yaml@@ -9,7 +9,6 @@ parameters: []parameter_mappings: []dataset_query:"lib/type": mbql/query- "lib/metadata": nulldatabase: My Databasestages:- "lib/type": mbql.stage/nativediff --git a/test_resources/serialization_baseline/collections/transforms/test_transform.yaml b/test_resources/serialization_baseline/collections/transforms/test_transform.yamlindex 27048291495..bd685a14e57 100644--- a/test_resources/serialization_baseline/collections/transforms/test_transform.yaml+++ b/test_resources/serialization_baseline/collections/transforms/test_transform.yaml@@ -6,7 +6,6 @@ name: Test Transformsource:query:"lib/type": mbql/query- "lib/metadata": nulldatabase: My Databasestages:- "lib/type": mbql.stage/mbqldiff --git a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yamlindex b7ae943d817..a7f2ceec0dc 100644--- a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml+++ b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml@@ -3,15 +3,13 @@ creator_id: [email protected]definition:database: My Database"lib/type": mbql/query- "lib/metadata": nullstages:- "lib/type": mbql.stage/mbqlaggregation:- - sum- - "lib/uuid": 865abdef-bb71-490c-9796-fa5b01bf16a2+ - {}- - field- - "lib/uuid": c115167e-2961-4ff1-b2e9-8affbff1a20b- base-type: type/Text+ - base-type: type/Text- - My Database- null- Schemaless Tablediff --git a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yamlindex 28697330fd4..48f463ebace 100644--- a/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml+++ b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml@@ -2,15 +2,14 @@ created_at: '2024-08-28T14:28:03.451106Z'creator_id: [email protected]definition:"lib/type": mbql/query- "lib/metadata": nulldatabase: My Databasestages:- "lib/type": mbql.stage/mbqlfilters:- - <- - "lib/uuid": eb938c48-cdf5-49b4-8652-cb816e281a8b+ - {}- - field- - "lib/uuid": cfe7c17c-789d-4486-9592-a26f49f06992+ - {}- - My Database- null- Schemaless Table