Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix serdes export perf with database metadata - id lookup (#72326)

Alexander Polyankine90c0ea24e55
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.clj
index 3e476c235b4..4bd7a0dc436 100644
--- a/src/metabase/actions/models.clj
+++ b/src/metabase/actions/models.clj
@@ -426,7 +426,7 @@
:skip [;; this is a temporary column to power v57 => v56 rollbacks, and we can remove it in v58.
:legacy_query]
:transform {:action_id (serdes/parent-ref)
- :database_id (serdes/fk :model/Database :name)
+ :database_id (serdes/fk :model/Database)
:dataset_query {:export serdes/export-mbql :import serdes/import-mbql}}})
(defmethod serdes/generate-path "HTTPAction" [_ _] nil)
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 26413f32c7a..5d2ff45ee9a 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -35,7 +35,7 @@
## Existing transformations
- - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database :name)` - export foreign key in a portable way
+ - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database)` - export foreign key in a portable way
- `(serdes/nested :model/DashboardCard :dashboard_id opts)` - include some entities in your entity export
- `(serdes/parent-ref)` - symmetrical call for `serdes/nested` to handle parent ids (you'd use it on `:dashboard_id`
in that case)
@@ -1028,11 +1028,59 @@
;;; ## Databases
-(defn ^:dynamic ^::cache *export-database-fk*
+(def ^:dynamic *batch-cache-max-size*
+ "Maximum number of entries in a batch cache before old entries are dropped."
+ 50000)
+
+(defn- make-batch-cache
+ "Returns a fn `(f id) -> entity`. On miss, calls `(load-fn id max-batch)` which must
+ return a map of `{id entity, ...}`. `max-batch` is the maximum number of entries to return.
+ All returned entries are merged into the cache so sibling lookups become hits.
+ When old entries plus new batch exceed `*batch-cache-max-size*`, old entries are dropped.
+ New batch entries are never evicted, so the cache may temporarily exceed the limit
+ (e.g. a field parent_id chain longer than the limit)."
+ [load-fn]
+ (let [cache (atom {})
+ order (atom clojure.lang.PersistentQueue/EMPTY)]
+ (fn [id]
+ (let [v (get @cache id ::not-found)]
+ (if-not (identical? v ::not-found)
+ v
+ (let [new-batch (load-fn id *batch-cache-max-size*)
+ new-keys (set (keys new-batch))
+ ;; drop old entries to make room, but never drop new-batch entries
+ old-count (- (count @cache) (count (filter new-keys (keys @cache))))
+ keep-old (max 0 (- *batch-cache-max-size* (count new-batch)))]
+ (when (> old-count keep-old)
+ (let [to-drop (- old-count keep-old)]
+ (swap! order #(into clojure.lang.PersistentQueue/EMPTY (drop to-drop) %))
+ (swap! cache #(select-keys % (into new-keys @order)))))
+ (swap! cache merge new-batch)
+ (swap! order into (keys new-batch))
+ (get new-batch id)))))))
+
+(defn- batch-load-databases
+ "Loads the requested database plus other databases up to `max-batch`."
+ [id max-batch]
+ (let [target (t2/select-pk->fn identity [:model/Database :id :name] :id id)
+ others (when (> max-batch 1)
+ (t2/select-pk->fn identity [:model/Database :id :name]
+ :id [:not= id]
+ {:limit (dec max-batch)}))]
+ (merge target others)))
+
+(defn- export-database-fk
+ "Given a numeric database ID and a fn that resolves it to a database entity,
+ return its name as a portable reference."
+ [id db-fn]
+ (when id
+ (:name (db-fn id))))
+
+(defn ^:dynamic *export-database-fk*
"Given a numeric database ID, return its name as a portable reference.
[[*import-database-fk*]] is the inverse."
[id]
- (*export-fk-keyed* id :model/Database :name))
+ (export-database-fk id #(t2/select-one [:model/Database :id :name] :id %)))
(defn ^:dynamic ^::cache *import-database-fk*
"Given a portable database name, resolve it back to a numeric ID.
@@ -1042,16 +1090,41 @@
;;; ## Tables
-(mu/defn ^:dynamic ^::cache *export-table-fk*
+(defn- batch-load-tables
+ "Loads the table with `id` plus FK-connected tables, up to `max-batch` total.
+ A database may have millions of tables due to partitioning, so we must respect the limit."
+ [id max-batch]
+ (let [target (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema] :id id)
+ remaining (- max-batch (count target))
+ fk-field-ids (when (pos? remaining)
+ (t2/select-fn-set :fk_target_field_id :model/Field
+ :table_id id
+ :fk_target_field_id [:not= nil]))
+ fk-table-ids (when (seq fk-field-ids)
+ (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))
+ fk-tables (when (seq fk-table-ids)
+ (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema]
+ :id [:in fk-table-ids]
+ {:limit remaining}))]
+ (merge target fk-tables)))
+
+(defn- export-table-fk
+ "Given a numeric table ID, a fn that resolves it to a table entity, and a fn
+ that resolves a database ID to a database entity, return `[db-name schema table-name]`."
+ [id table-fn db-fn]
+ (when id
+ (let [table (table-fn id)]
+ [(export-database-fk (:db_id table) db-fn) (:schema table) (:name table)])))
+
+(mu/defn ^:dynamic *export-table-fk*
"Given a numeric `table_id`, return a portable table reference.
If the `table_id` is `nil`, return `nil`. This is legal for a native question.
That has the form `[db-name schema table-name]`, where the `schema` might be nil.
[[import-table-fk]] is the inverse."
[table-id :- [:maybe ::lib.schema.id/table]]
- (when table-id
- (let [{:keys [db_id name schema]} (t2/select-one :model/Table :id table-id)
- db-name (t2/select-one-fn :name :model/Database :id db_id)]
- [db-name schema name])))
+ (export-table-fk table-id
+ #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)
+ #(t2/select-one [:model/Database :id :name] :id %)))
(mu/defn ^:dynamic ^::cache *import-table-fk*
"Given a `table_id` as exported by [[export-table-fk]], resolve it back into a numeric `table_id`.
@@ -1096,18 +1169,65 @@
;;; ## Fields
-(defn- field-hierarchy [id]
- (reverse
- (t2/select :model/Field
- {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]
- {:union-all [{:from [[:metabase_field :mf]]
- :select [:mf.id :mf.name :mf.parent_id :mf.table_id]
- :where [:= :id id]}
- {:from [[:metabase_field :pf]]
- :select [:pf.id :pf.name :pf.parent_id :pf.table_id]
- :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]
- :from [:parents]
- :select [:name :table_id]})))
+(defn- field-name-chain
+ "Walks parent_id chain in memory using `field-fn` to resolve each field.
+ Returns a seq of field names from root parent to the given field."
+ [field-id field-fn]
+ (loop [id field-id
+ names ()]
+ (let [field (field-fn id)]
+ (if-let [pid (:parent_id field)]
+ (recur pid (cons (:name field) names))
+ (cons (:name field) names)))))
+
+(defn- export-field-fk
+ "Given a numeric field ID and lookup fns for fields, tables, and databases,
+ return `[db-name schema table-name & field-names]`."
+ [field-id field-fn table-fn db-fn]
+ (when field-id
+ (let [field (field-fn field-id)
+ table-ref (export-table-fk (:table_id field) table-fn db-fn)
+ field-names (field-name-chain field-id field-fn)]
+ (into table-ref field-names))))
+
+(defn- load-field-hierarchy
+ "Loads the field with `id` and all its ancestors (via parent_id) in a single
+ recursive CTE query. Returns a map of `{id entity}`."
+ [id]
+ (into {}
+ (map (juxt :id identity))
+ (t2/select :model/Field
+ {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]
+ {:union-all [{:from [[:metabase_field :mf]]
+ :select [:mf.id :mf.name :mf.parent_id :mf.table_id]
+ :where [:= :id id]}
+ {:from [[:metabase_field :pf]]
+ :select [:pf.id :pf.name :pf.parent_id :pf.table_id]
+ :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]
+ :from [:parents]
+ :select [:id :name :parent_id :table_id]})))
+
+(defn- batch-load-fields
+ "Loads the requested field, its parent_id chain (via recursive CTE), and additional
+ fields from the same table and FK-connected tables, up to `max-batch` total.
+ A table may have millions of fields due to JSON unfolding, so we must respect the limit."
+ [field-id max-batch]
+ (let [required (load-field-hierarchy field-id)
+ remaining (- max-batch (count required))]
+ (if-not (pos? remaining)
+ required
+ (let [table-id (:table_id (get required field-id))
+ fk-field-ids (t2/select-fn-set :fk_target_field_id :model/Field
+ :table_id table-id
+ :fk_target_field_id [:not= nil])
+ fk-table-ids (when (seq fk-field-ids)
+ (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))
+ all-table-ids (cond-> #{table-id} (seq fk-table-ids) (into fk-table-ids))
+ extras (t2/select-pk->fn identity [:model/Field :id :name :table_id :parent_id]
+ :id [:not-in (set (keys required))]
+ :table_id [:in all-table-ids]
+ {:limit remaining})]
+ (merge required extras)))))
(defn recursively-find-field-q
"Build a query to find a field among parents (should start with bottom-most field first), i.e.:
@@ -1122,15 +1242,15 @@
[:= :name field]
[:= :parent_id (recursively-find-field-q table-id rest)]]}))
-(mu/defn ^:dynamic ^::cache *export-field-fk*
+(mu/defn ^:dynamic *export-field-fk*
"Given a numeric `field_id`, return a portable field reference.
That has the form `[db-name schema table-name field-name]`, where the `schema` might be nil.
[[*import-field-fk*]] is the inverse."
[field-id :- [:maybe ::lib.schema.id/field]]
- (when field-id
- (let [fields (field-hierarchy field-id)
- [db-name schema field-name] (*export-table-fk* (:table_id (first fields)))]
- (into [db-name schema field-name] (map :name fields)))))
+ (export-field-fk field-id
+ #(t2/select-one [:model/Field :id :name :table_id :parent_id] :id %)
+ #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)
+ #(t2/select-one [:model/Database :id :name] :id %)))
(mu/defn ^:dynamic ^::cache *import-field-fk* :- [:maybe pos-int?]
"Given a `field_id` as exported by [[*export-field-fk*]], resolve it back into a numeric `field_id`."
@@ -1919,7 +2039,7 @@
;;; ## Memoizing appdb lookups
-(defmacro with-cache
+(defmacro with-memoize-cache
"Runs body with all functions marked with ::cache re-bound to memoized versions for performance."
[& body]
(let [ns* 'metabase.models.serialization]
@@ -1929,3 +2049,20 @@
:let [fq-sym (symbol (name ns*) (name var-sym))]]
[fq-sym `(memoize ~fq-sym)]))
~@body)))
+
+(defn with-batch-cache
+ "Runs `thunk` with the batch cache bound to the export functions."
+ [thunk]
+ (let [db-cache (make-batch-cache batch-load-databases)
+ table-cache (make-batch-cache batch-load-tables)
+ field-cache (make-batch-cache batch-load-fields)]
+ (binding [*export-database-fk* #(export-database-fk % db-cache)
+ *export-table-fk* #(export-table-fk % table-cache db-cache)
+ *export-field-fk* #(export-field-fk % field-cache table-cache db-cache)]
+ (thunk))))
+
+(defmacro with-cache
+ "Runs body with all functions marked with ::cache re-bound to memoized versions for performance,
+ and with database/table lookups backed by batch-loading caches."
+ [& body]
+ `(with-batch-cache (fn [] (with-memoize-cache ~@body))))
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 520ab07ceb3..8e1122c8408 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1326,7 +1326,7 @@
:legacy_query]
:transform
{:created_at (serdes/date)
- :database_id (serdes/fk :model/Database :name)
+ :database_id (serdes/fk :model/Database)
:table_id (serdes/fk :model/Table)
:source_card_id (serdes/fk :model/Card)
:collection_id (serdes/fk :model/Collection)
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 3a747d21218..60f6501d2c4 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -364,7 +364,7 @@
:creator_id (serdes/fk :model/User)
:owner_user_id (serdes/fk :model/User)
:collection_id (serdes/fk :model/Collection)
- :source_database_id (serdes/fk :model/Database :name)
+ :source_database_id (serdes/fk :model/Database)
:source {:export (fn [source]
(-> source
(m/update-existing :query serdes/export-mbql)
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 13d314792d6..553ddabcb50 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -420,8 +420,8 @@
;; In order to retrieve the dependencies for a field its table_id needs to be serialized as [database schema table],
;; a trio of strings with schema maybe nil.
-(defmethod serdes/generate-path "Field" [_ field]
- (let [[db schema table & fields] (serdes/*export-field-fk* (:id field))]
+(defmethod serdes/generate-path "Field" [_ {:keys [id]}]
+ (let [[db schema table & fields] (serdes/*export-field-fk* id)]
(->> (into (serdes/table->path [db schema table])
(map (fn [n] {:model "Field" :id n}) fields))
(filterv some?))))
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index 67c444ab9eb..bf47518bd88 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -41,9 +41,8 @@
(defmethod serdes/entity-id "FieldUserSettings" [_ _] nil)
(defmethod serdes/generate-path "FieldUserSettings" [_ {:keys [field_id]}]
- (let [field (t2/select-one 'Field :id field_id)]
- (conj (serdes/generate-path "Field" field)
- {:model "FieldUserSettings" :id "1"})))
+ (conj (serdes/generate-path "Field" {:id field_id})
+ {:model "FieldUserSettings" :id "1"}))
(defmethod serdes/dependencies "FieldUserSettings" [fv]
;; Take the path, but drop the FieldUserSettings section at the end, to get the parent Field's path instead.
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index e7a52abe298..60445e60727 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -571,9 +571,8 @@
(defmethod serdes/entity-id "FieldValues" [_ _] nil)
(defmethod serdes/generate-path "FieldValues" [_ {:keys [field_id]}]
- (let [field (t2/select-one 'Field :id field_id)]
- (conj (serdes/generate-path "Field" field)
- {:model "FieldValues" :id "0"})))
+ (conj (serdes/generate-path "Field" {:id field_id})
+ {:model "FieldValues" :id "0"}))
(defmethod serdes/dependencies "FieldValues" [fv]
;; Take the path, but drop the FieldValues section at the end, to get the parent Field's path instead.
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 3a9322a9318..abfd0841bd7 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -680,7 +680,7 @@
(merge fields segments measures)))
(defmethod serdes/generate-path "Table" [_ table]
- (let [db-name (t2/select-one-fn :name :model/Database :id (:db_id table))]
+ (let [db-name (serdes/*export-database-fk* (:db_id table))]
(filterv some? [{:model "Database" :id db-name}
(when (:schema table)
{:model "Schema" :id (:schema table)})
@@ -708,7 +708,7 @@
:archived_at (serdes/date)
:deactivated_at (serdes/date)
:data_layer (serdes/optional-kw)
- :db_id (serdes/fk :model/Database :name)
+ :db_id (serdes/fk :model/Database)
:collection_id (serdes/fk :model/Collection)
:transform_id (serdes/fk :model/Transform)}
:defaults {:is_defective_duplicate false
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 3584a9eec7a..22368aeac15 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -648,7 +648,7 @@
::serdes/skip))
:import identity}
:creator_id (serdes/fk :model/User)
- :router_database_id (serdes/fk :model/Database :name)
+ :router_database_id (serdes/fk :model/Database)
:initial_sync_status {:export identity :import (constantly "complete")}}
:defaults {:auto_run_queries true
:is_attached_dwh false
diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.clj
index 9709d459ef5..eafa8d41003 100644
--- a/test/metabase/models/serialization_test.clj
+++ b/test/metabase/models/serialization_test.clj
@@ -3,7 +3,8 @@
[clojure.test :refer :all]
[metabase.lib.core :as lib]
[metabase.lib.test-metadata :as meta]
- [metabase.models.serialization :as serdes]))
+ [metabase.models.serialization :as serdes]
+ [metabase.test :as mt]))
(deftest ^:parallel drop-mbql-5-uuids-on-export-test
(binding [serdes/*export-field-fk* (constantly ::field-id)]
@@ -194,3 +195,46 @@
:values_source_type :card
:values_source_config {:card_id 1, :value_field [:field 53 nil]}
:position 0}])))))
+
+(deftest export-database-fk-with-cache-test
+ (testing "3 databases with cache limit 2: evicts and re-loads on miss"
+ (mt/with-temp [:model/Database {db1-id :id} {:name "db1" :engine :h2}
+ :model/Database {db2-id :id} {:name "db2" :engine :h2}
+ :model/Database {db3-id :id} {:name "db3" :engine :h2}]
+ (binding [serdes/*batch-cache-max-size* 2]
+ (serdes/with-cache
+ (is (= "db1" (serdes/*export-database-fk* db1-id)))
+ (is (= "db2" (serdes/*export-database-fk* db2-id)))
+ (is (= "db3" (serdes/*export-database-fk* db3-id)))
+ (is (= "db1" (serdes/*export-database-fk* db1-id))))))))
+
+(deftest export-table-fk-with-cache-test
+ (testing "3 tables with FK and cache limit 2: batch-loads FK target, evicts and re-loads"
+ (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}
+ :model/Table {t1-id :id} {:name "t1" :schema "public" :db_id db-id}
+ :model/Table {t2-id :id} {:name "t2" :schema "public" :db_id db-id}
+ :model/Table {t3-id :id} {:name "t3" :schema nil :db_id db-id}
+ :model/Field {f1-id :id} {:name "f1" :table_id t2-id :base_type :type/Integer}
+ :model/Field _f2 {:name "f2" :table_id t1-id :base_type :type/Integer
+ :fk_target_field_id f1-id}]
+ (binding [serdes/*batch-cache-max-size* 2]
+ (serdes/with-cache
+ (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id)))
+ (is (= ["db" "public" "t2"] (serdes/*export-table-fk* t2-id)))
+ (is (= ["db" nil "t3"] (serdes/*export-table-fk* t3-id)))
+ (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id))))))))
+
+(deftest export-field-fk-with-cache-test
+ (testing "3-level parent chain with cache limit 2: all ancestors kept even when exceeding limit"
+ (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}
+ :model/Table {t-id :id} {:name "t" :schema nil :db_id db-id}
+ :model/Field {f1-id :id} {:name "f1" :table_id t-id :base_type :type/JSON}
+ :model/Field {f2-id :id} {:name "f2" :table_id t-id :base_type :type/JSON
+ :parent_id f1-id}
+ :model/Field {f3-id :id} {:name "f3" :table_id t-id :base_type :type/Text
+ :parent_id f2-id}]
+ (binding [serdes/*batch-cache-max-size* 2]
+ (serdes/with-cache
+ (is (= ["db" nil "t" "f1" "f2" "f3"] (serdes/*export-field-fk* f3-id)))
+ (is (= ["db" nil "t" "f1" "f2"] (serdes/*export-field-fk* f2-id)))
+ (is (= ["db" nil "t" "f1"] (serdes/*export-field-fk* f1-id))))))))