The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Revert "Fix serdes perf on export" (#72416)
Alexander Polyankin011bbd3f981c
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.cljindex 4bd7a0dc436..3e476c235b4 100644--- a/src/metabase/actions/models.clj+++ b/src/metabase/actions/models.clj@@ -426,7 +426,7 @@:skip [;; this is a temporary column to power v57 => v56 rollbacks, and we can remove it in v58.:legacy_query]:transform {:action_id (serdes/parent-ref)- :database_id (serdes/fk :model/Database)+ :database_id (serdes/fk :model/Database :name):dataset_query {:export serdes/export-mbql :import serdes/import-mbql}}})(defmethod serdes/generate-path "HTTPAction" [_ _] nil)diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 02268d8778c..26413f32c7a 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -35,7 +35,7 @@## Existing transformations- - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database)` - export foreign key in a portable way+ - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database :name)` - export foreign key in a portable way- `(serdes/nested :model/DashboardCard :dashboard_id opts)` - include some entities in your entity export- `(serdes/parent-ref)` - symmetrical call for `serdes/nested` to handle parent ids (you'd use it on `:dashboard_id`in that case)@@ -1028,59 +1028,11 @@;;; ## Databases-(def ^:dynamic *batch-cache-max-size*- "Maximum number of entries in a batch cache before old entries are dropped."- 50000)--(defn- make-batch-cache- "Returns a fn `(f id) -> entity`. On miss, calls `(load-fn id max-batch)` which must- return a map of `{id entity, ...}`. `max-batch` is the maximum number of entries to return.- All returned entries are merged into the cache so sibling lookups become hits.- When old entries plus new batch exceed `*batch-cache-max-size*`, old entries are dropped.- New batch entries are never evicted, so the cache may temporarily exceed the limit- (e.g. a field parent_id chain longer than the limit)."- [load-fn]- (let [cache (atom {})- order (atom clojure.lang.PersistentQueue/EMPTY)]- (fn [id]- (let [v (get @cache id ::not-found)]- (if-not (identical? v ::not-found)- v- (let [new-batch (load-fn id *batch-cache-max-size*)- new-keys (set (keys new-batch))- ;; drop old entries to make room, but never drop new-batch entries- old-count (- (count @cache) (count (filter new-keys (keys @cache))))- keep-old (max 0 (- *batch-cache-max-size* (count new-batch)))]- (when (> old-count keep-old)- (let [to-drop (- old-count keep-old)]- (swap! order #(into clojure.lang.PersistentQueue/EMPTY (drop to-drop) %))- (swap! cache #(select-keys % (into new-keys @order)))))- (swap! cache merge new-batch)- (swap! order into (keys new-batch))- (get new-batch id)))))))--(defn- batch-load-databases- "Loads the requested database plus other databases up to `max-batch`."- [id max-batch]- (let [target (t2/select-pk->fn identity [:model/Database :id :name] :id id)- others (when (> max-batch 1)- (t2/select-pk->fn identity [:model/Database :id :name]- :id [:not= id]- {:limit (dec max-batch)}))]- (merge target others)))--(defn- export-database-fk- "Given a numeric database ID and a fn that resolves it to a database entity,- return its name as a portable reference."- [id db-fn]- (when-let [db (and id (db-fn id))]- (:name db)))--(defn ^:dynamic *export-database-fk*+(defn ^:dynamic ^::cache *export-database-fk*"Given a numeric database ID, return its name as a portable reference.[[*import-database-fk*]] is the inverse."[id]- (export-database-fk id #(t2/select-one [:model/Database :id :name] :id %)))+ (*export-fk-keyed* id :model/Database :name))(defn ^:dynamic ^::cache *import-database-fk*"Given a portable database name, resolve it back to a numeric ID.@@ -1090,40 +1042,16 @@;;; ## Tables-(defn- batch-load-tables- "Loads the table with `id` plus FK-connected tables, up to `max-batch` total.- A database may have millions of tables due to partitioning, so we must respect the limit."- [id max-batch]- (let [target (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema] :id id)- remaining (- max-batch (count target))- fk-field-ids (when (pos? remaining)- (t2/select-fn-set :fk_target_field_id :model/Field- :table_id id- :fk_target_field_id [:not= nil]))- fk-table-ids (when (seq fk-field-ids)- (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))- fk-tables (when (seq fk-table-ids)- (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema]- :id [:in fk-table-ids]- {:limit remaining}))]- (merge target fk-tables)))--(defn- export-table-fk- "Given a numeric table ID, a fn that resolves it to a table entity, and a fn- that resolves a database ID to a database entity, return `[db-name schema table-name]`."- [id table-fn db-fn]- (when-let [table (and id (table-fn id))]- [(export-database-fk (:db_id table) db-fn) (:schema table) (:name table)]))--(mu/defn ^:dynamic *export-table-fk*+(mu/defn ^:dynamic ^::cache *export-table-fk*"Given a numeric `table_id`, return a portable table reference.If the `table_id` is `nil`, return `nil`. This is legal for a native question.That has the form `[db-name schema table-name]`, where the `schema` might be nil.[[import-table-fk]] is the inverse."[table-id :- [:maybe ::lib.schema.id/table]]- (export-table-fk table-id- #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)- #(t2/select-one [:model/Database :id :name] :id %)))+ (when table-id+ (let [{:keys [db_id name schema]} (t2/select-one :model/Table :id table-id)+ db-name (t2/select-one-fn :name :model/Database :id db_id)]+ [db-name schema name])))(mu/defn ^:dynamic ^::cache *import-table-fk*"Given a `table_id` as exported by [[export-table-fk]], resolve it back into a numeric `table_id`.@@ -1168,64 +1096,18 @@;;; ## Fields-(defn- field-name-chain- "Walks parent_id chain in memory using `field-fn` to resolve each field.- Returns a seq of field names from root parent to the given field."- [field-id field-fn]- (loop [id field-id- names ()]- (let [field (field-fn id)]- (if-let [pid (:parent_id field)]- (recur pid (cons (:name field) names))- (cons (:name field) names)))))--(defn- export-field-fk- "Given a numeric field ID and lookup fns for fields, tables, and databases,- return `[db-name schema table-name & field-names]`."- [field-id field-fn table-fn db-fn]- (when-let [field (and field-id (field-fn field-id))]- (let [table-ref (export-table-fk (:table_id field) table-fn db-fn)- field-names (field-name-chain field-id field-fn)]- (into table-ref field-names))))--(defn- load-field-hierarchy- "Loads the field with `id` and all its ancestors (via parent_id) in a single- recursive CTE query. Returns a map of `{id entity}`."- [id]- (into {}- (map (juxt :id identity))- (t2/select :model/Field- {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]- {:union-all [{:from [[:metabase_field :mf]]- :select [:mf.id :mf.name :mf.parent_id :mf.table_id]- :where [:= :id id]}- {:from [[:metabase_field :pf]]- :select [:pf.id :pf.name :pf.parent_id :pf.table_id]- :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]- :from [:parents]- :select [:id :name :parent_id :table_id]})))--(defn- batch-load-fields- "Loads the requested field, its parent_id chain (via recursive CTE), and additional- fields from the same table and FK-connected tables, up to `max-batch` total.- A table may have millions of fields due to JSON unfolding, so we must respect the limit."- [field-id max-batch]- (let [required (load-field-hierarchy field-id)- remaining (- max-batch (count required))]- (if (or (empty? required) (not (pos? remaining)))- required- (let [table-id (:table_id (get required field-id))- fk-field-ids (t2/select-fn-set :fk_target_field_id :model/Field- :table_id table-id- :fk_target_field_id [:not= nil])- fk-table-ids (when (seq fk-field-ids)- (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))- all-table-ids (into #{table-id} fk-table-ids)- extras (t2/select-pk->fn identity [:model/Field :id :name :table_id :parent_id]- :id [:not-in (set (keys required))]- :table_id [:in all-table-ids]- {:limit remaining})]- (merge required extras)))))+(defn- field-hierarchy [id]+ (reverse+ (t2/select :model/Field+ {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]+ {:union-all [{:from [[:metabase_field :mf]]+ :select [:mf.id :mf.name :mf.parent_id :mf.table_id]+ :where [:= :id id]}+ {:from [[:metabase_field :pf]]+ :select [:pf.id :pf.name :pf.parent_id :pf.table_id]+ :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]+ :from [:parents]+ :select [:name :table_id]})))(defn recursively-find-field-q"Build a query to find a field among parents (should start with bottom-most field first), i.e.:@@ -1240,15 +1122,15 @@[:= :name field][:= :parent_id (recursively-find-field-q table-id rest)]]}))-(mu/defn ^:dynamic *export-field-fk*+(mu/defn ^:dynamic ^::cache *export-field-fk*"Given a numeric `field_id`, return a portable field reference.That has the form `[db-name schema table-name field-name]`, where the `schema` might be nil.[[*import-field-fk*]] is the inverse."[field-id :- [:maybe ::lib.schema.id/field]]- (export-field-fk field-id- #(t2/select-one [:model/Field :id :name :table_id :parent_id] :id %)- #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)- #(t2/select-one [:model/Database :id :name] :id %)))+ (when field-id+ (let [fields (field-hierarchy field-id)+ [db-name schema field-name] (*export-table-fk* (:table_id (first fields)))]+ (into [db-name schema field-name] (map :name fields)))))(mu/defn ^:dynamic ^::cache *import-field-fk* :- [:maybe pos-int?]"Given a `field_id` as exported by [[*export-field-fk*]], resolve it back into a numeric `field_id`."@@ -2037,7 +1919,7 @@;;; ## Memoizing appdb lookups-(defmacro with-memoize-cache+(defmacro with-cache"Runs body with all functions marked with ::cache re-bound to memoized versions for performance."[& body](let [ns* 'metabase.models.serialization]@@ -2047,20 +1929,3 @@:let [fq-sym (symbol (name ns*) (name var-sym))]][fq-sym `(memoize ~fq-sym)]))~@body)))--(defn with-batch-cache- "Runs `thunk` with the batch cache bound to the export functions."- [thunk]- (let [db-cache (make-batch-cache batch-load-databases)- table-cache (make-batch-cache batch-load-tables)- field-cache (make-batch-cache batch-load-fields)]- (binding [*export-database-fk* #(export-database-fk % db-cache)- *export-table-fk* #(export-table-fk % table-cache db-cache)- *export-field-fk* #(export-field-fk % field-cache table-cache db-cache)]- (thunk))))--(defmacro with-cache- "Runs body with all functions marked with ::cache re-bound to memoized versions for performance,- and with database/table lookups backed by batch-loading caches."- [& body]- `(with-batch-cache (fn [] (with-memoize-cache ~@body))))diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 8e1122c8408..520ab07ceb3 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1326,7 +1326,7 @@:legacy_query]:transform{:created_at (serdes/date)- :database_id (serdes/fk :model/Database)+ :database_id (serdes/fk :model/Database :name):table_id (serdes/fk :model/Table):source_card_id (serdes/fk :model/Card):collection_id (serdes/fk :model/Collection)diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex 60f6501d2c4..3a747d21218 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -364,7 +364,7 @@:creator_id (serdes/fk :model/User):owner_user_id (serdes/fk :model/User):collection_id (serdes/fk :model/Collection)- :source_database_id (serdes/fk :model/Database)+ :source_database_id (serdes/fk :model/Database :name):source {:export (fn [source](-> source(m/update-existing :query serdes/export-mbql)diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 553ddabcb50..13d314792d6 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -420,8 +420,8 @@;; In order to retrieve the dependencies for a field its table_id needs to be serialized as [database schema table],;; a trio of strings with schema maybe nil.-(defmethod serdes/generate-path "Field" [_ {:keys [id]}]- (let [[db schema table & fields] (serdes/*export-field-fk* id)]+(defmethod serdes/generate-path "Field" [_ field]+ (let [[db schema table & fields] (serdes/*export-field-fk* (:id field))](->> (into (serdes/table->path [db schema table])(map (fn [n] {:model "Field" :id n}) fields))(filterv some?))))diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.cljindex bf47518bd88..67c444ab9eb 100644--- a/src/metabase/warehouse_schema/models/field_user_settings.clj+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj@@ -41,8 +41,9 @@(defmethod serdes/entity-id "FieldUserSettings" [_ _] nil)(defmethod serdes/generate-path "FieldUserSettings" [_ {:keys [field_id]}]- (conj (serdes/generate-path "Field" {:id field_id})- {:model "FieldUserSettings" :id "1"}))+ (let [field (t2/select-one 'Field :id field_id)]+ (conj (serdes/generate-path "Field" field)+ {:model "FieldUserSettings" :id "1"})))(defmethod serdes/dependencies "FieldUserSettings" [fv];; Take the path, but drop the FieldUserSettings section at the end, to get the parent Field's path instead.diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 60445e60727..e7a52abe298 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -571,8 +571,9 @@(defmethod serdes/entity-id "FieldValues" [_ _] nil)(defmethod serdes/generate-path "FieldValues" [_ {:keys [field_id]}]- (conj (serdes/generate-path "Field" {:id field_id})- {:model "FieldValues" :id "0"}))+ (let [field (t2/select-one 'Field :id field_id)]+ (conj (serdes/generate-path "Field" field)+ {:model "FieldValues" :id "0"})))(defmethod serdes/dependencies "FieldValues" [fv];; Take the path, but drop the FieldValues section at the end, to get the parent Field's path instead.diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex abfd0841bd7..3a9322a9318 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -680,7 +680,7 @@(merge fields segments measures)))(defmethod serdes/generate-path "Table" [_ table]- (let [db-name (serdes/*export-database-fk* (:db_id table))]+ (let [db-name (t2/select-one-fn :name :model/Database :id (:db_id table))](filterv some? [{:model "Database" :id db-name}(when (:schema table){:model "Schema" :id (:schema table)})@@ -708,7 +708,7 @@:archived_at (serdes/date):deactivated_at (serdes/date):data_layer (serdes/optional-kw)- :db_id (serdes/fk :model/Database)+ :db_id (serdes/fk :model/Database :name):collection_id (serdes/fk :model/Collection):transform_id (serdes/fk :model/Transform)}:defaults {:is_defective_duplicate falsediff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 22368aeac15..3584a9eec7a 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -648,7 +648,7 @@::serdes/skip)):import identity}:creator_id (serdes/fk :model/User)- :router_database_id (serdes/fk :model/Database)+ :router_database_id (serdes/fk :model/Database :name):initial_sync_status {:export identity :import (constantly "complete")}}:defaults {:auto_run_queries true:is_attached_dwh falsediff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.cljindex 882a19897e9..9709d459ef5 100644--- a/test/metabase/models/serialization_test.clj+++ b/test/metabase/models/serialization_test.clj@@ -3,8 +3,7 @@[clojure.test :refer :all][metabase.lib.core :as lib][metabase.lib.test-metadata :as meta]- [metabase.models.serialization :as serdes]- [metabase.test :as mt]))+ [metabase.models.serialization :as serdes]))(deftest ^:parallel drop-mbql-5-uuids-on-export-test(binding [serdes/*export-field-fk* (constantly ::field-id)]@@ -195,55 +194,3 @@:values_source_type :card:values_source_config {:card_id 1, :value_field [:field 53 nil]}:position 0}])))))--(deftest export-database-fk-with-cache-test- (testing "3 databases with cache limit 2: evicts and re-loads on miss"- (mt/with-temp [:model/Database {db1-id :id} {:name "db1" :engine :h2}- :model/Database {db2-id :id} {:name "db2" :engine :h2}- :model/Database {db3-id :id} {:name "db3" :engine :h2}]- (binding [serdes/*batch-cache-max-size* 2]- (serdes/with-cache- (is (= "db1" (serdes/*export-database-fk* db1-id)))- (is (= "db2" (serdes/*export-database-fk* db2-id)))- (is (= "db3" (serdes/*export-database-fk* db3-id)))- (is (= "db1" (serdes/*export-database-fk* db1-id)))))))- (testing "non-existent database ID returns nil"- (serdes/with-cache- (is (nil? (serdes/*export-database-fk* Integer/MAX_VALUE))))))--(deftest export-table-fk-with-cache-test- (testing "3 tables with FK and cache limit 2: batch-loads FK target, evicts and re-loads"- (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}- :model/Table {t1-id :id} {:name "t1" :schema "public" :db_id db-id}- :model/Table {t2-id :id} {:name "t2" :schema "public" :db_id db-id}- :model/Table {t3-id :id} {:name "t3" :schema nil :db_id db-id}- :model/Field {f1-id :id} {:name "f1" :table_id t2-id :base_type :type/Integer}- :model/Field _f2 {:name "f2" :table_id t1-id :base_type :type/Integer- :fk_target_field_id f1-id}]- (binding [serdes/*batch-cache-max-size* 2]- (serdes/with-cache- (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id)))- (is (= ["db" "public" "t2"] (serdes/*export-table-fk* t2-id)))- (is (= ["db" nil "t3"] (serdes/*export-table-fk* t3-id)))- (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id)))))))- (testing "non-existent table ID returns nil"- (serdes/with-cache- (is (nil? (serdes/*export-table-fk* Integer/MAX_VALUE))))))--(deftest export-field-fk-with-cache-test- (testing "3-level parent chain with cache limit 2: all ancestors kept even when exceeding limit"- (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}- :model/Table {t-id :id} {:name "t" :schema nil :db_id db-id}- :model/Field {f1-id :id} {:name "f1" :table_id t-id :base_type :type/JSON}- :model/Field {f2-id :id} {:name "f2" :table_id t-id :base_type :type/JSON- :parent_id f1-id}- :model/Field {f3-id :id} {:name "f3" :table_id t-id :base_type :type/Text- :parent_id f2-id}]- (binding [serdes/*batch-cache-max-size* 2]- (serdes/with-cache- (is (= ["db" nil "t" "f1" "f2" "f3"] (serdes/*export-field-fk* f3-id)))- (is (= ["db" nil "t" "f1" "f2"] (serdes/*export-field-fk* f2-id)))- (is (= ["db" nil "t" "f1"] (serdes/*export-field-fk* f1-id)))))))- (testing "non-existent field ID returns nil"- (serdes/with-cache- (is (nil? (serdes/*export-field-fk* Integer/MAX_VALUE))))))