The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix serdes export perf with database metadata - id lookup (#72326)
Alexander Polyankine90c0ea24e55
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.cljindex 3e476c235b4..4bd7a0dc436 100644--- a/src/metabase/actions/models.clj+++ b/src/metabase/actions/models.clj@@ -426,7 +426,7 @@:skip [;; this is a temporary column to power v57 => v56 rollbacks, and we can remove it in v58.:legacy_query]:transform {:action_id (serdes/parent-ref)- :database_id (serdes/fk :model/Database :name)+ :database_id (serdes/fk :model/Database):dataset_query {:export serdes/export-mbql :import serdes/import-mbql}}})(defmethod serdes/generate-path "HTTPAction" [_ _] nil)diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 26413f32c7a..5d2ff45ee9a 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -35,7 +35,7 @@## Existing transformations- - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database :name)` - export foreign key in a portable way+ - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database)` - export foreign key in a portable way- `(serdes/nested :model/DashboardCard :dashboard_id opts)` - include some entities in your entity export- `(serdes/parent-ref)` - symmetrical call for `serdes/nested` to handle parent ids (you'd use it on `:dashboard_id`in that case)@@ -1028,11 +1028,59 @@;;; ## Databases-(defn ^:dynamic ^::cache *export-database-fk*+(def ^:dynamic *batch-cache-max-size*+ "Maximum number of entries in a batch cache before old entries are dropped."+ 50000)++(defn- make-batch-cache+ "Returns a fn `(f id) -> entity`. On miss, calls `(load-fn id max-batch)` which must+ return a map of `{id entity, ...}`. `max-batch` is the maximum number of entries to return.+ All returned entries are merged into the cache so sibling lookups become hits.+ When old entries plus new batch exceed `*batch-cache-max-size*`, old entries are dropped.+ New batch entries are never evicted, so the cache may temporarily exceed the limit+ (e.g. a field parent_id chain longer than the limit)."+ [load-fn]+ (let [cache (atom {})+ order (atom clojure.lang.PersistentQueue/EMPTY)]+ (fn [id]+ (let [v (get @cache id ::not-found)]+ (if-not (identical? v ::not-found)+ v+ (let [new-batch (load-fn id *batch-cache-max-size*)+ new-keys (set (keys new-batch))+ ;; drop old entries to make room, but never drop new-batch entries+ old-count (- (count @cache) (count (filter new-keys (keys @cache))))+ keep-old (max 0 (- *batch-cache-max-size* (count new-batch)))]+ (when (> old-count keep-old)+ (let [to-drop (- old-count keep-old)]+ (swap! order #(into clojure.lang.PersistentQueue/EMPTY (drop to-drop) %))+ (swap! cache #(select-keys % (into new-keys @order)))))+ (swap! cache merge new-batch)+ (swap! order into (keys new-batch))+ (get new-batch id)))))))++(defn- batch-load-databases+ "Loads the requested database plus other databases up to `max-batch`."+ [id max-batch]+ (let [target (t2/select-pk->fn identity [:model/Database :id :name] :id id)+ others (when (> max-batch 1)+ (t2/select-pk->fn identity [:model/Database :id :name]+ :id [:not= id]+ {:limit (dec max-batch)}))]+ (merge target others)))++(defn- export-database-fk+ "Given a numeric database ID and a fn that resolves it to a database entity,+ return its name as a portable reference."+ [id db-fn]+ (when id+ (:name (db-fn id))))++(defn ^:dynamic *export-database-fk*"Given a numeric database ID, return its name as a portable reference.[[*import-database-fk*]] is the inverse."[id]- (*export-fk-keyed* id :model/Database :name))+ (export-database-fk id #(t2/select-one [:model/Database :id :name] :id %)))(defn ^:dynamic ^::cache *import-database-fk*"Given a portable database name, resolve it back to a numeric ID.@@ -1042,16 +1090,41 @@;;; ## Tables-(mu/defn ^:dynamic ^::cache *export-table-fk*+(defn- batch-load-tables+ "Loads the table with `id` plus FK-connected tables, up to `max-batch` total.+ A database may have millions of tables due to partitioning, so we must respect the limit."+ [id max-batch]+ (let [target (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema] :id id)+ remaining (- max-batch (count target))+ fk-field-ids (when (pos? remaining)+ (t2/select-fn-set :fk_target_field_id :model/Field+ :table_id id+ :fk_target_field_id [:not= nil]))+ fk-table-ids (when (seq fk-field-ids)+ (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))+ fk-tables (when (seq fk-table-ids)+ (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema]+ :id [:in fk-table-ids]+ {:limit remaining}))]+ (merge target fk-tables)))++(defn- export-table-fk+ "Given a numeric table ID, a fn that resolves it to a table entity, and a fn+ that resolves a database ID to a database entity, return `[db-name schema table-name]`."+ [id table-fn db-fn]+ (when id+ (let [table (table-fn id)]+ [(export-database-fk (:db_id table) db-fn) (:schema table) (:name table)])))++(mu/defn ^:dynamic *export-table-fk*"Given a numeric `table_id`, return a portable table reference.If the `table_id` is `nil`, return `nil`. This is legal for a native question.That has the form `[db-name schema table-name]`, where the `schema` might be nil.[[import-table-fk]] is the inverse."[table-id :- [:maybe ::lib.schema.id/table]]- (when table-id- (let [{:keys [db_id name schema]} (t2/select-one :model/Table :id table-id)- db-name (t2/select-one-fn :name :model/Database :id db_id)]- [db-name schema name])))+ (export-table-fk table-id+ #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)+ #(t2/select-one [:model/Database :id :name] :id %)))(mu/defn ^:dynamic ^::cache *import-table-fk*"Given a `table_id` as exported by [[export-table-fk]], resolve it back into a numeric `table_id`.@@ -1096,18 +1169,65 @@;;; ## Fields-(defn- field-hierarchy [id]- (reverse- (t2/select :model/Field- {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]- {:union-all [{:from [[:metabase_field :mf]]- :select [:mf.id :mf.name :mf.parent_id :mf.table_id]- :where [:= :id id]}- {:from [[:metabase_field :pf]]- :select [:pf.id :pf.name :pf.parent_id :pf.table_id]- :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]- :from [:parents]- :select [:name :table_id]})))+(defn- field-name-chain+ "Walks parent_id chain in memory using `field-fn` to resolve each field.+ Returns a seq of field names from root parent to the given field."+ [field-id field-fn]+ (loop [id field-id+ names ()]+ (let [field (field-fn id)]+ (if-let [pid (:parent_id field)]+ (recur pid (cons (:name field) names))+ (cons (:name field) names)))))++(defn- export-field-fk+ "Given a numeric field ID and lookup fns for fields, tables, and databases,+ return `[db-name schema table-name & field-names]`."+ [field-id field-fn table-fn db-fn]+ (when field-id+ (let [field (field-fn field-id)+ table-ref (export-table-fk (:table_id field) table-fn db-fn)+ field-names (field-name-chain field-id field-fn)]+ (into table-ref field-names))))++(defn- load-field-hierarchy+ "Loads the field with `id` and all its ancestors (via parent_id) in a single+ recursive CTE query. Returns a map of `{id entity}`."+ [id]+ (into {}+ (map (juxt :id identity))+ (t2/select :model/Field+ {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]+ {:union-all [{:from [[:metabase_field :mf]]+ :select [:mf.id :mf.name :mf.parent_id :mf.table_id]+ :where [:= :id id]}+ {:from [[:metabase_field :pf]]+ :select [:pf.id :pf.name :pf.parent_id :pf.table_id]+ :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]+ :from [:parents]+ :select [:id :name :parent_id :table_id]})))++(defn- batch-load-fields+ "Loads the requested field, its parent_id chain (via recursive CTE), and additional+ fields from the same table and FK-connected tables, up to `max-batch` total.+ A table may have millions of fields due to JSON unfolding, so we must respect the limit."+ [field-id max-batch]+ (let [required (load-field-hierarchy field-id)+ remaining (- max-batch (count required))]+ (if-not (pos? remaining)+ required+ (let [table-id (:table_id (get required field-id))+ fk-field-ids (t2/select-fn-set :fk_target_field_id :model/Field+ :table_id table-id+ :fk_target_field_id [:not= nil])+ fk-table-ids (when (seq fk-field-ids)+ (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))+ all-table-ids (cond-> #{table-id} (seq fk-table-ids) (into fk-table-ids))+ extras (t2/select-pk->fn identity [:model/Field :id :name :table_id :parent_id]+ :id [:not-in (set (keys required))]+ :table_id [:in all-table-ids]+ {:limit remaining})]+ (merge required extras)))))(defn recursively-find-field-q"Build a query to find a field among parents (should start with bottom-most field first), i.e.:@@ -1122,15 +1242,15 @@[:= :name field][:= :parent_id (recursively-find-field-q table-id rest)]]}))-(mu/defn ^:dynamic ^::cache *export-field-fk*+(mu/defn ^:dynamic *export-field-fk*"Given a numeric `field_id`, return a portable field reference.That has the form `[db-name schema table-name field-name]`, where the `schema` might be nil.[[*import-field-fk*]] is the inverse."[field-id :- [:maybe ::lib.schema.id/field]]- (when field-id- (let [fields (field-hierarchy field-id)- [db-name schema field-name] (*export-table-fk* (:table_id (first fields)))]- (into [db-name schema field-name] (map :name fields)))))+ (export-field-fk field-id+ #(t2/select-one [:model/Field :id :name :table_id :parent_id] :id %)+ #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)+ #(t2/select-one [:model/Database :id :name] :id %)))(mu/defn ^:dynamic ^::cache *import-field-fk* :- [:maybe pos-int?]"Given a `field_id` as exported by [[*export-field-fk*]], resolve it back into a numeric `field_id`."@@ -1919,7 +2039,7 @@;;; ## Memoizing appdb lookups-(defmacro with-cache+(defmacro with-memoize-cache"Runs body with all functions marked with ::cache re-bound to memoized versions for performance."[& body](let [ns* 'metabase.models.serialization]@@ -1929,3 +2049,20 @@:let [fq-sym (symbol (name ns*) (name var-sym))]][fq-sym `(memoize ~fq-sym)]))~@body)))++(defn with-batch-cache+ "Runs `thunk` with the batch cache bound to the export functions."+ [thunk]+ (let [db-cache (make-batch-cache batch-load-databases)+ table-cache (make-batch-cache batch-load-tables)+ field-cache (make-batch-cache batch-load-fields)]+ (binding [*export-database-fk* #(export-database-fk % db-cache)+ *export-table-fk* #(export-table-fk % table-cache db-cache)+ *export-field-fk* #(export-field-fk % field-cache table-cache db-cache)]+ (thunk))))++(defmacro with-cache+ "Runs body with all functions marked with ::cache re-bound to memoized versions for performance,+ and with database/table lookups backed by batch-loading caches."+ [& body]+ `(with-batch-cache (fn [] (with-memoize-cache ~@body))))diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 520ab07ceb3..8e1122c8408 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1326,7 +1326,7 @@:legacy_query]:transform{:created_at (serdes/date)- :database_id (serdes/fk :model/Database :name)+ :database_id (serdes/fk :model/Database):table_id (serdes/fk :model/Table):source_card_id (serdes/fk :model/Card):collection_id (serdes/fk :model/Collection)diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex 3a747d21218..60f6501d2c4 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -364,7 +364,7 @@:creator_id (serdes/fk :model/User):owner_user_id (serdes/fk :model/User):collection_id (serdes/fk :model/Collection)- :source_database_id (serdes/fk :model/Database :name)+ :source_database_id (serdes/fk :model/Database):source {:export (fn [source](-> source(m/update-existing :query serdes/export-mbql)diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 13d314792d6..553ddabcb50 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -420,8 +420,8 @@;; In order to retrieve the dependencies for a field its table_id needs to be serialized as [database schema table],;; a trio of strings with schema maybe nil.-(defmethod serdes/generate-path "Field" [_ field]- (let [[db schema table & fields] (serdes/*export-field-fk* (:id field))]+(defmethod serdes/generate-path "Field" [_ {:keys [id]}]+ (let [[db schema table & fields] (serdes/*export-field-fk* id)](->> (into (serdes/table->path [db schema table])(map (fn [n] {:model "Field" :id n}) fields))(filterv some?))))diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.cljindex 67c444ab9eb..bf47518bd88 100644--- a/src/metabase/warehouse_schema/models/field_user_settings.clj+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj@@ -41,9 +41,8 @@(defmethod serdes/entity-id "FieldUserSettings" [_ _] nil)(defmethod serdes/generate-path "FieldUserSettings" [_ {:keys [field_id]}]- (let [field (t2/select-one 'Field :id field_id)]- (conj (serdes/generate-path "Field" field)- {:model "FieldUserSettings" :id "1"})))+ (conj (serdes/generate-path "Field" {:id field_id})+ {:model "FieldUserSettings" :id "1"}))(defmethod serdes/dependencies "FieldUserSettings" [fv];; Take the path, but drop the FieldUserSettings section at the end, to get the parent Field's path instead.diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex e7a52abe298..60445e60727 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -571,9 +571,8 @@(defmethod serdes/entity-id "FieldValues" [_ _] nil)(defmethod serdes/generate-path "FieldValues" [_ {:keys [field_id]}]- (let [field (t2/select-one 'Field :id field_id)]- (conj (serdes/generate-path "Field" field)- {:model "FieldValues" :id "0"})))+ (conj (serdes/generate-path "Field" {:id field_id})+ {:model "FieldValues" :id "0"}))(defmethod serdes/dependencies "FieldValues" [fv];; Take the path, but drop the FieldValues section at the end, to get the parent Field's path instead.diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 3a9322a9318..abfd0841bd7 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -680,7 +680,7 @@(merge fields segments measures)))(defmethod serdes/generate-path "Table" [_ table]- (let [db-name (t2/select-one-fn :name :model/Database :id (:db_id table))]+ (let [db-name (serdes/*export-database-fk* (:db_id table))](filterv some? [{:model "Database" :id db-name}(when (:schema table){:model "Schema" :id (:schema table)})@@ -708,7 +708,7 @@:archived_at (serdes/date):deactivated_at (serdes/date):data_layer (serdes/optional-kw)- :db_id (serdes/fk :model/Database :name)+ :db_id (serdes/fk :model/Database):collection_id (serdes/fk :model/Collection):transform_id (serdes/fk :model/Transform)}:defaults {:is_defective_duplicate falsediff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 3584a9eec7a..22368aeac15 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -648,7 +648,7 @@::serdes/skip)):import identity}:creator_id (serdes/fk :model/User)- :router_database_id (serdes/fk :model/Database :name)+ :router_database_id (serdes/fk :model/Database):initial_sync_status {:export identity :import (constantly "complete")}}:defaults {:auto_run_queries true:is_attached_dwh falsediff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.cljindex 9709d459ef5..eafa8d41003 100644--- a/test/metabase/models/serialization_test.clj+++ b/test/metabase/models/serialization_test.clj@@ -3,7 +3,8 @@[clojure.test :refer :all][metabase.lib.core :as lib][metabase.lib.test-metadata :as meta]- [metabase.models.serialization :as serdes]))+ [metabase.models.serialization :as serdes]+ [metabase.test :as mt]))(deftest ^:parallel drop-mbql-5-uuids-on-export-test(binding [serdes/*export-field-fk* (constantly ::field-id)]@@ -194,3 +195,46 @@:values_source_type :card:values_source_config {:card_id 1, :value_field [:field 53 nil]}:position 0}])))))++(deftest export-database-fk-with-cache-test+ (testing "3 databases with cache limit 2: evicts and re-loads on miss"+ (mt/with-temp [:model/Database {db1-id :id} {:name "db1" :engine :h2}+ :model/Database {db2-id :id} {:name "db2" :engine :h2}+ :model/Database {db3-id :id} {:name "db3" :engine :h2}]+ (binding [serdes/*batch-cache-max-size* 2]+ (serdes/with-cache+ (is (= "db1" (serdes/*export-database-fk* db1-id)))+ (is (= "db2" (serdes/*export-database-fk* db2-id)))+ (is (= "db3" (serdes/*export-database-fk* db3-id)))+ (is (= "db1" (serdes/*export-database-fk* db1-id))))))))++(deftest export-table-fk-with-cache-test+ (testing "3 tables with FK and cache limit 2: batch-loads FK target, evicts and re-loads"+ (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}+ :model/Table {t1-id :id} {:name "t1" :schema "public" :db_id db-id}+ :model/Table {t2-id :id} {:name "t2" :schema "public" :db_id db-id}+ :model/Table {t3-id :id} {:name "t3" :schema nil :db_id db-id}+ :model/Field {f1-id :id} {:name "f1" :table_id t2-id :base_type :type/Integer}+ :model/Field _f2 {:name "f2" :table_id t1-id :base_type :type/Integer+ :fk_target_field_id f1-id}]+ (binding [serdes/*batch-cache-max-size* 2]+ (serdes/with-cache+ (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id)))+ (is (= ["db" "public" "t2"] (serdes/*export-table-fk* t2-id)))+ (is (= ["db" nil "t3"] (serdes/*export-table-fk* t3-id)))+ (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id))))))))++(deftest export-field-fk-with-cache-test+ (testing "3-level parent chain with cache limit 2: all ancestors kept even when exceeding limit"+ (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}+ :model/Table {t-id :id} {:name "t" :schema nil :db_id db-id}+ :model/Field {f1-id :id} {:name "f1" :table_id t-id :base_type :type/JSON}+ :model/Field {f2-id :id} {:name "f2" :table_id t-id :base_type :type/JSON+ :parent_id f1-id}+ :model/Field {f3-id :id} {:name "f3" :table_id t-id :base_type :type/Text+ :parent_id f2-id}]+ (binding [serdes/*batch-cache-max-size* 2]+ (serdes/with-cache+ (is (= ["db" nil "t" "f1" "f2" "f3"] (serdes/*export-field-fk* f3-id)))+ (is (= ["db" nil "t" "f1" "f2"] (serdes/*export-field-fk* f2-id)))+ (is (= ["db" nil "t" "f1"] (serdes/*export-field-fk* f1-id))))))))