Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Revert "Fix serdes perf on export" (#72416)

Alexander Polyankin011bbd3f981c
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.clj
index 4bd7a0dc436..3e476c235b4 100644
--- a/src/metabase/actions/models.clj
+++ b/src/metabase/actions/models.clj
@@ -426,7 +426,7 @@
:skip [;; this is a temporary column to power v57 => v56 rollbacks, and we can remove it in v58.
:legacy_query]
:transform {:action_id (serdes/parent-ref)
- :database_id (serdes/fk :model/Database)
+ :database_id (serdes/fk :model/Database :name)
:dataset_query {:export serdes/export-mbql :import serdes/import-mbql}}})
(defmethod serdes/generate-path "HTTPAction" [_ _] nil)
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 02268d8778c..26413f32c7a 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -35,7 +35,7 @@
## Existing transformations
- - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database)` - export foreign key in a portable way
+ - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database :name)` - export foreign key in a portable way
- `(serdes/nested :model/DashboardCard :dashboard_id opts)` - include some entities in your entity export
- `(serdes/parent-ref)` - symmetrical call for `serdes/nested` to handle parent ids (you'd use it on `:dashboard_id`
in that case)
@@ -1028,59 +1028,11 @@
;;; ## Databases
-(def ^:dynamic *batch-cache-max-size*
- "Maximum number of entries in a batch cache before old entries are dropped."
- 50000)
-
-(defn- make-batch-cache
- "Returns a fn `(f id) -> entity`. On miss, calls `(load-fn id max-batch)` which must
- return a map of `{id entity, ...}`. `max-batch` is the maximum number of entries to return.
- All returned entries are merged into the cache so sibling lookups become hits.
- When old entries plus new batch exceed `*batch-cache-max-size*`, old entries are dropped.
- New batch entries are never evicted, so the cache may temporarily exceed the limit
- (e.g. a field parent_id chain longer than the limit)."
- [load-fn]
- (let [cache (atom {})
- order (atom clojure.lang.PersistentQueue/EMPTY)]
- (fn [id]
- (let [v (get @cache id ::not-found)]
- (if-not (identical? v ::not-found)
- v
- (let [new-batch (load-fn id *batch-cache-max-size*)
- new-keys (set (keys new-batch))
- ;; drop old entries to make room, but never drop new-batch entries
- old-count (- (count @cache) (count (filter new-keys (keys @cache))))
- keep-old (max 0 (- *batch-cache-max-size* (count new-batch)))]
- (when (> old-count keep-old)
- (let [to-drop (- old-count keep-old)]
- (swap! order #(into clojure.lang.PersistentQueue/EMPTY (drop to-drop) %))
- (swap! cache #(select-keys % (into new-keys @order)))))
- (swap! cache merge new-batch)
- (swap! order into (keys new-batch))
- (get new-batch id)))))))
-
-(defn- batch-load-databases
- "Loads the requested database plus other databases up to `max-batch`."
- [id max-batch]
- (let [target (t2/select-pk->fn identity [:model/Database :id :name] :id id)
- others (when (> max-batch 1)
- (t2/select-pk->fn identity [:model/Database :id :name]
- :id [:not= id]
- {:limit (dec max-batch)}))]
- (merge target others)))
-
-(defn- export-database-fk
- "Given a numeric database ID and a fn that resolves it to a database entity,
- return its name as a portable reference."
- [id db-fn]
- (when-let [db (and id (db-fn id))]
- (:name db)))
-
-(defn ^:dynamic *export-database-fk*
+(defn ^:dynamic ^::cache *export-database-fk*
"Given a numeric database ID, return its name as a portable reference.
[[*import-database-fk*]] is the inverse."
[id]
- (export-database-fk id #(t2/select-one [:model/Database :id :name] :id %)))
+ (*export-fk-keyed* id :model/Database :name))
(defn ^:dynamic ^::cache *import-database-fk*
"Given a portable database name, resolve it back to a numeric ID.
@@ -1090,40 +1042,16 @@
;;; ## Tables
-(defn- batch-load-tables
- "Loads the table with `id` plus FK-connected tables, up to `max-batch` total.
- A database may have millions of tables due to partitioning, so we must respect the limit."
- [id max-batch]
- (let [target (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema] :id id)
- remaining (- max-batch (count target))
- fk-field-ids (when (pos? remaining)
- (t2/select-fn-set :fk_target_field_id :model/Field
- :table_id id
- :fk_target_field_id [:not= nil]))
- fk-table-ids (when (seq fk-field-ids)
- (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))
- fk-tables (when (seq fk-table-ids)
- (t2/select-pk->fn identity [:model/Table :id :db_id :name :schema]
- :id [:in fk-table-ids]
- {:limit remaining}))]
- (merge target fk-tables)))
-
-(defn- export-table-fk
- "Given a numeric table ID, a fn that resolves it to a table entity, and a fn
- that resolves a database ID to a database entity, return `[db-name schema table-name]`."
- [id table-fn db-fn]
- (when-let [table (and id (table-fn id))]
- [(export-database-fk (:db_id table) db-fn) (:schema table) (:name table)]))
-
-(mu/defn ^:dynamic *export-table-fk*
+(mu/defn ^:dynamic ^::cache *export-table-fk*
"Given a numeric `table_id`, return a portable table reference.
If the `table_id` is `nil`, return `nil`. This is legal for a native question.
That has the form `[db-name schema table-name]`, where the `schema` might be nil.
[[import-table-fk]] is the inverse."
[table-id :- [:maybe ::lib.schema.id/table]]
- (export-table-fk table-id
- #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)
- #(t2/select-one [:model/Database :id :name] :id %)))
+ (when table-id
+ (let [{:keys [db_id name schema]} (t2/select-one :model/Table :id table-id)
+ db-name (t2/select-one-fn :name :model/Database :id db_id)]
+ [db-name schema name])))
(mu/defn ^:dynamic ^::cache *import-table-fk*
"Given a `table_id` as exported by [[export-table-fk]], resolve it back into a numeric `table_id`.
@@ -1168,64 +1096,18 @@
;;; ## Fields
-(defn- field-name-chain
- "Walks parent_id chain in memory using `field-fn` to resolve each field.
- Returns a seq of field names from root parent to the given field."
- [field-id field-fn]
- (loop [id field-id
- names ()]
- (let [field (field-fn id)]
- (if-let [pid (:parent_id field)]
- (recur pid (cons (:name field) names))
- (cons (:name field) names)))))
-
-(defn- export-field-fk
- "Given a numeric field ID and lookup fns for fields, tables, and databases,
- return `[db-name schema table-name & field-names]`."
- [field-id field-fn table-fn db-fn]
- (when-let [field (and field-id (field-fn field-id))]
- (let [table-ref (export-table-fk (:table_id field) table-fn db-fn)
- field-names (field-name-chain field-id field-fn)]
- (into table-ref field-names))))
-
-(defn- load-field-hierarchy
- "Loads the field with `id` and all its ancestors (via parent_id) in a single
- recursive CTE query. Returns a map of `{id entity}`."
- [id]
- (into {}
- (map (juxt :id identity))
- (t2/select :model/Field
- {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]
- {:union-all [{:from [[:metabase_field :mf]]
- :select [:mf.id :mf.name :mf.parent_id :mf.table_id]
- :where [:= :id id]}
- {:from [[:metabase_field :pf]]
- :select [:pf.id :pf.name :pf.parent_id :pf.table_id]
- :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]
- :from [:parents]
- :select [:id :name :parent_id :table_id]})))
-
-(defn- batch-load-fields
- "Loads the requested field, its parent_id chain (via recursive CTE), and additional
- fields from the same table and FK-connected tables, up to `max-batch` total.
- A table may have millions of fields due to JSON unfolding, so we must respect the limit."
- [field-id max-batch]
- (let [required (load-field-hierarchy field-id)
- remaining (- max-batch (count required))]
- (if (or (empty? required) (not (pos? remaining)))
- required
- (let [table-id (:table_id (get required field-id))
- fk-field-ids (t2/select-fn-set :fk_target_field_id :model/Field
- :table_id table-id
- :fk_target_field_id [:not= nil])
- fk-table-ids (when (seq fk-field-ids)
- (t2/select-fn-set :table_id :model/Field :id [:in fk-field-ids]))
- all-table-ids (into #{table-id} fk-table-ids)
- extras (t2/select-pk->fn identity [:model/Field :id :name :table_id :parent_id]
- :id [:not-in (set (keys required))]
- :table_id [:in all-table-ids]
- {:limit remaining})]
- (merge required extras)))))
+(defn- field-hierarchy [id]
+ (reverse
+ (t2/select :model/Field
+ {:with-recursive [[[:parents {:columns [:id :name :parent_id :table_id]}]
+ {:union-all [{:from [[:metabase_field :mf]]
+ :select [:mf.id :mf.name :mf.parent_id :mf.table_id]
+ :where [:= :id id]}
+ {:from [[:metabase_field :pf]]
+ :select [:pf.id :pf.name :pf.parent_id :pf.table_id]
+ :join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]
+ :from [:parents]
+ :select [:name :table_id]})))
(defn recursively-find-field-q
"Build a query to find a field among parents (should start with bottom-most field first), i.e.:
@@ -1240,15 +1122,15 @@
[:= :name field]
[:= :parent_id (recursively-find-field-q table-id rest)]]}))
-(mu/defn ^:dynamic *export-field-fk*
+(mu/defn ^:dynamic ^::cache *export-field-fk*
"Given a numeric `field_id`, return a portable field reference.
That has the form `[db-name schema table-name field-name]`, where the `schema` might be nil.
[[*import-field-fk*]] is the inverse."
[field-id :- [:maybe ::lib.schema.id/field]]
- (export-field-fk field-id
- #(t2/select-one [:model/Field :id :name :table_id :parent_id] :id %)
- #(t2/select-one [:model/Table :id :db_id :name :schema] :id %)
- #(t2/select-one [:model/Database :id :name] :id %)))
+ (when field-id
+ (let [fields (field-hierarchy field-id)
+ [db-name schema field-name] (*export-table-fk* (:table_id (first fields)))]
+ (into [db-name schema field-name] (map :name fields)))))
(mu/defn ^:dynamic ^::cache *import-field-fk* :- [:maybe pos-int?]
"Given a `field_id` as exported by [[*export-field-fk*]], resolve it back into a numeric `field_id`."
@@ -2037,7 +1919,7 @@
;;; ## Memoizing appdb lookups
-(defmacro with-memoize-cache
+(defmacro with-cache
"Runs body with all functions marked with ::cache re-bound to memoized versions for performance."
[& body]
(let [ns* 'metabase.models.serialization]
@@ -2047,20 +1929,3 @@
:let [fq-sym (symbol (name ns*) (name var-sym))]]
[fq-sym `(memoize ~fq-sym)]))
~@body)))
-
-(defn with-batch-cache
- "Runs `thunk` with the batch cache bound to the export functions."
- [thunk]
- (let [db-cache (make-batch-cache batch-load-databases)
- table-cache (make-batch-cache batch-load-tables)
- field-cache (make-batch-cache batch-load-fields)]
- (binding [*export-database-fk* #(export-database-fk % db-cache)
- *export-table-fk* #(export-table-fk % table-cache db-cache)
- *export-field-fk* #(export-field-fk % field-cache table-cache db-cache)]
- (thunk))))
-
-(defmacro with-cache
- "Runs body with all functions marked with ::cache re-bound to memoized versions for performance,
- and with database/table lookups backed by batch-loading caches."
- [& body]
- `(with-batch-cache (fn [] (with-memoize-cache ~@body))))
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 8e1122c8408..520ab07ceb3 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1326,7 +1326,7 @@
:legacy_query]
:transform
{:created_at (serdes/date)
- :database_id (serdes/fk :model/Database)
+ :database_id (serdes/fk :model/Database :name)
:table_id (serdes/fk :model/Table)
:source_card_id (serdes/fk :model/Card)
:collection_id (serdes/fk :model/Collection)
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 60f6501d2c4..3a747d21218 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -364,7 +364,7 @@
:creator_id (serdes/fk :model/User)
:owner_user_id (serdes/fk :model/User)
:collection_id (serdes/fk :model/Collection)
- :source_database_id (serdes/fk :model/Database)
+ :source_database_id (serdes/fk :model/Database :name)
:source {:export (fn [source]
(-> source
(m/update-existing :query serdes/export-mbql)
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 553ddabcb50..13d314792d6 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -420,8 +420,8 @@
;; In order to retrieve the dependencies for a field its table_id needs to be serialized as [database schema table],
;; a trio of strings with schema maybe nil.
-(defmethod serdes/generate-path "Field" [_ {:keys [id]}]
- (let [[db schema table & fields] (serdes/*export-field-fk* id)]
+(defmethod serdes/generate-path "Field" [_ field]
+ (let [[db schema table & fields] (serdes/*export-field-fk* (:id field))]
(->> (into (serdes/table->path [db schema table])
(map (fn [n] {:model "Field" :id n}) fields))
(filterv some?))))
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index bf47518bd88..67c444ab9eb 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -41,8 +41,9 @@
(defmethod serdes/entity-id "FieldUserSettings" [_ _] nil)
(defmethod serdes/generate-path "FieldUserSettings" [_ {:keys [field_id]}]
- (conj (serdes/generate-path "Field" {:id field_id})
- {:model "FieldUserSettings" :id "1"}))
+ (let [field (t2/select-one 'Field :id field_id)]
+ (conj (serdes/generate-path "Field" field)
+ {:model "FieldUserSettings" :id "1"})))
(defmethod serdes/dependencies "FieldUserSettings" [fv]
;; Take the path, but drop the FieldUserSettings section at the end, to get the parent Field's path instead.
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index 60445e60727..e7a52abe298 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -571,8 +571,9 @@
(defmethod serdes/entity-id "FieldValues" [_ _] nil)
(defmethod serdes/generate-path "FieldValues" [_ {:keys [field_id]}]
- (conj (serdes/generate-path "Field" {:id field_id})
- {:model "FieldValues" :id "0"}))
+ (let [field (t2/select-one 'Field :id field_id)]
+ (conj (serdes/generate-path "Field" field)
+ {:model "FieldValues" :id "0"})))
(defmethod serdes/dependencies "FieldValues" [fv]
;; Take the path, but drop the FieldValues section at the end, to get the parent Field's path instead.
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index abfd0841bd7..3a9322a9318 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -680,7 +680,7 @@
(merge fields segments measures)))
(defmethod serdes/generate-path "Table" [_ table]
- (let [db-name (serdes/*export-database-fk* (:db_id table))]
+ (let [db-name (t2/select-one-fn :name :model/Database :id (:db_id table))]
(filterv some? [{:model "Database" :id db-name}
(when (:schema table)
{:model "Schema" :id (:schema table)})
@@ -708,7 +708,7 @@
:archived_at (serdes/date)
:deactivated_at (serdes/date)
:data_layer (serdes/optional-kw)
- :db_id (serdes/fk :model/Database)
+ :db_id (serdes/fk :model/Database :name)
:collection_id (serdes/fk :model/Collection)
:transform_id (serdes/fk :model/Transform)}
:defaults {:is_defective_duplicate false
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 22368aeac15..3584a9eec7a 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -648,7 +648,7 @@
::serdes/skip))
:import identity}
:creator_id (serdes/fk :model/User)
- :router_database_id (serdes/fk :model/Database)
+ :router_database_id (serdes/fk :model/Database :name)
:initial_sync_status {:export identity :import (constantly "complete")}}
:defaults {:auto_run_queries true
:is_attached_dwh false
diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.clj
index 882a19897e9..9709d459ef5 100644
--- a/test/metabase/models/serialization_test.clj
+++ b/test/metabase/models/serialization_test.clj
@@ -3,8 +3,7 @@
[clojure.test :refer :all]
[metabase.lib.core :as lib]
[metabase.lib.test-metadata :as meta]
- [metabase.models.serialization :as serdes]
- [metabase.test :as mt]))
+ [metabase.models.serialization :as serdes]))
(deftest ^:parallel drop-mbql-5-uuids-on-export-test
(binding [serdes/*export-field-fk* (constantly ::field-id)]
@@ -195,55 +194,3 @@
:values_source_type :card
:values_source_config {:card_id 1, :value_field [:field 53 nil]}
:position 0}])))))
-
-(deftest export-database-fk-with-cache-test
- (testing "3 databases with cache limit 2: evicts and re-loads on miss"
- (mt/with-temp [:model/Database {db1-id :id} {:name "db1" :engine :h2}
- :model/Database {db2-id :id} {:name "db2" :engine :h2}
- :model/Database {db3-id :id} {:name "db3" :engine :h2}]
- (binding [serdes/*batch-cache-max-size* 2]
- (serdes/with-cache
- (is (= "db1" (serdes/*export-database-fk* db1-id)))
- (is (= "db2" (serdes/*export-database-fk* db2-id)))
- (is (= "db3" (serdes/*export-database-fk* db3-id)))
- (is (= "db1" (serdes/*export-database-fk* db1-id)))))))
- (testing "non-existent database ID returns nil"
- (serdes/with-cache
- (is (nil? (serdes/*export-database-fk* Integer/MAX_VALUE))))))
-
-(deftest export-table-fk-with-cache-test
- (testing "3 tables with FK and cache limit 2: batch-loads FK target, evicts and re-loads"
- (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}
- :model/Table {t1-id :id} {:name "t1" :schema "public" :db_id db-id}
- :model/Table {t2-id :id} {:name "t2" :schema "public" :db_id db-id}
- :model/Table {t3-id :id} {:name "t3" :schema nil :db_id db-id}
- :model/Field {f1-id :id} {:name "f1" :table_id t2-id :base_type :type/Integer}
- :model/Field _f2 {:name "f2" :table_id t1-id :base_type :type/Integer
- :fk_target_field_id f1-id}]
- (binding [serdes/*batch-cache-max-size* 2]
- (serdes/with-cache
- (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id)))
- (is (= ["db" "public" "t2"] (serdes/*export-table-fk* t2-id)))
- (is (= ["db" nil "t3"] (serdes/*export-table-fk* t3-id)))
- (is (= ["db" "public" "t1"] (serdes/*export-table-fk* t1-id)))))))
- (testing "non-existent table ID returns nil"
- (serdes/with-cache
- (is (nil? (serdes/*export-table-fk* Integer/MAX_VALUE))))))
-
-(deftest export-field-fk-with-cache-test
- (testing "3-level parent chain with cache limit 2: all ancestors kept even when exceeding limit"
- (mt/with-temp [:model/Database {db-id :id} {:name "db" :engine :h2}
- :model/Table {t-id :id} {:name "t" :schema nil :db_id db-id}
- :model/Field {f1-id :id} {:name "f1" :table_id t-id :base_type :type/JSON}
- :model/Field {f2-id :id} {:name "f2" :table_id t-id :base_type :type/JSON
- :parent_id f1-id}
- :model/Field {f3-id :id} {:name "f3" :table_id t-id :base_type :type/Text
- :parent_id f2-id}]
- (binding [serdes/*batch-cache-max-size* 2]
- (serdes/with-cache
- (is (= ["db" nil "t" "f1" "f2" "f3"] (serdes/*export-field-fk* f3-id)))
- (is (= ["db" nil "t" "f1" "f2"] (serdes/*export-field-fk* f2-id)))
- (is (= ["db" nil "t" "f1"] (serdes/*export-field-fk* f1-id)))))))
- (testing "non-existent field ID returns nil"
- (serdes/with-cache
- (is (nil? (serdes/*export-field-fk* Integer/MAX_VALUE))))))