Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GDGT-376] [GDGT-2624] fix serdes import oom (#75786)

YB403107f13fda
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
index c90393d4394..70c74b87efe 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
@@ -10,6 +10,7 @@
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.log :as log]
+ [metabase.util.memoize :as u.memo]
[metabase.util.yaml :as yaml]
[potemkin.types :as p])
(:import (java.io File)))
@@ -84,27 +85,59 @@
We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."
#{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "curated_search_entries" "snippets" "transforms"})
+(defn- path-interner
+ "Returns a function that interns `:serdes/meta` path vectors.
+
+ Every parsed file allocates fresh copies of its path maps and strings, but the
+ `[db schema table]` prefix segments recur once per sibling file (a million-field instance
+ has ~#tables distinct prefixes). Interning each segment (and its strings) collapses those
+ duplicates to shared objects, so index keys share structure — value equality is unchanged,
+ so lookups and `:seen`-set comparisons behave identically. The interner caches live only as
+ long as the returned function is reachable; do not hold it past index construction."
+ []
+ (let [intern-str (u.memo/fast-interner)
+ intern-seg (u.memo/fast-interner
+ (fn [seg]
+ (cond-> (update seg :model intern-str)
+ (string? (:id seg)) (update :id intern-str))))]
+ (fn intern-path [hierarchy]
+ (mapv intern-seg hierarchy))))
+
+(defn- ingestible-file?
+ "Whether `file` is a regular `.yaml` file under one of the [[legal-top-level-paths]].
+ Dotfiles are excluded (editor temp files, see #41567)."
+ [^File root-dir ^File file]
+ (boolean (and (.isFile file)
+ (not (str/starts-with? (.getName file) "."))
+ (str/ends-with? (.getName file) ".yaml")
+ (let [rel (.relativize (.toPath root-dir) (.toPath file))]
+ (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))))
+
+(defn- file-hierarchy!
+ "Parses `file` and returns its `:serdes/meta` abstract path, or nil on parse failure.
+ On failure the exception is recorded in the `errors` atom."
+ [^File file errors]
+ (try
+ (serdes/path (ingest-file file))
+ (catch Exception e
+ (log/warn (u/strip-error e "Error reading file during ingestion"))
+ (let [file-name (.getName file)]
+ (swap! errors conj (ex-info (format "Failed to parse file: %s" file-name)
+ {:file file-name} e)))
+ nil)))
+
(defn- ingest-all
- "Returns {:entities {unlabeled-hierarchy [hierarchy File]}, :errors [Exception...]}.
+ "Returns {:entities {unlabeled-hierarchy File}, :errors [Exception...]}.
Dotfiles are silently skipped (editor temp files, see #41567).
Non-dotfile YAML parse failures are collected in :errors."
[^File root-dir]
- (let [errors (atom [])]
+ (let [errors (atom [])
+ intern-path (path-interner)]
{:entities (into {} (for [^File file (file-seq root-dir)
- :when (and (.isFile file)
- (not (str/starts-with? (.getName file) "."))
- (str/ends-with? (.getName file) ".yaml")
- (let [rel (.relativize (.toPath root-dir) (.toPath file))]
- (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))
- :let [file-name (.getName file)
- hierarchy (try
- (serdes/path (ingest-file file))
- (catch Exception e
- (log/warn (u/strip-error e "Error reading file during ingestion"))
- (swap! errors conj (ex-info (format "Failed to parse file: %s" file-name) {:file file-name} e))
- nil))]
+ :when (ingestible-file? root-dir file)
+ :let [hierarchy (file-hierarchy! file errors)]
:when hierarchy]
- [(strip-labels hierarchy) [hierarchy file]]))
+ [(intern-path (strip-labels hierarchy)) file]))
:errors @errors}))
(defn- populate-cache! [cache errors-atom ingest-fn]
@@ -129,11 +162,11 @@
(if (= ["Setting"] (mapv :model serdes-meta))
(when (contains? settings kw-id)
{:serdes/meta serdes-meta :key kw-id :value (get settings kw-id)})
- (when-let [target (get @cache (strip-labels serdes-meta))]
+ (when-let [file (get @cache (strip-labels serdes-meta))]
(try
- (ingest-file (second target))
+ (ingest-file file)
(catch Exception e
- (throw (ex-info "Unable to ingest file" {:file (.getName ^File (second target))
+ (throw (ex-info "Unable to ingest file" {:file (.getName ^File file)
:abs-path serdes-meta} e))))))))
(ingest-errors [_]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index 32092244269..294f761f4ae 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -4,7 +4,6 @@
(:require
[clojure.string :as str]
[diehard.core :as dh]
- [medley.core :as m]
[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase-enterprise.serialization.v2.models :as serdes.models]
@@ -248,7 +247,6 @@
:seen #{}
:circular #{}
:ingestion ingestion
- :from-ids (->> ingestion serdes.ingest/ingest-list (m/index-by :id))
:errors []})
(defn load-metabase!
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index 46c1767b23e..191fd68fbba 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -841,7 +841,7 @@
(testing "Hidden YAML files are still silently skipped"
(let [{:keys [entities]} (#'ingest/ingest-all (io/file dump-dir))
files (->> entities
- (map (comp second second))
+ (map val)
(map #(.getName ^File %))
set)]
(is (not (contains? files ".hidden.yaml")))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.clj
index 5837ef28e22..1b9188127bf 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.clj
@@ -6,6 +6,8 @@
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase.util.yaml :as yaml]))
+(set! *warn-on-reflection* true)
+
(deftest basic-ingest-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(io/make-parents dump-dir "collections" "1234567890abcdefABCDE_the_label" "fake") ; Prepare the right directories.
@@ -65,6 +67,67 @@
(is (= exp
(ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE"}])))))))
+(deftest path-interner-test
+ (let [intern-path (#'ingest/path-interner)
+ ;; rebuild every map and string so equal segments are distinct objects, like segments
+ ;; parsed from separate YAML files
+ fresh-seg (fn [seg] (update-vals seg #(if (string? %) (String. ^String %) %)))
+ prefix [{:model "Database" :id "mydb"} {:model "Table" :id "T"}]
+ path-1 (mapv fresh-seg (conj prefix {:model "Field" :id "F1"}))
+ path-2 (mapv fresh-seg (conj prefix {:model "Field" :id "F2"}))
+ interned-1 (intern-path path-1)
+ interned-2 (intern-path path-2)]
+ (testing "value equality is preserved"
+ (is (= path-1 interned-1))
+ (is (= path-2 interned-2)))
+ (testing "equal prefix segments from different paths intern to the identical object"
+ (is (identical? (interned-1 0) (interned-2 0)))
+ (is (identical? (interned-1 1) (interned-2 1))))
+ (testing "re-interning a fresh copy returns the same canonical objects"
+ (let [reinterned (intern-path (mapv fresh-seg path-1))]
+ (is (identical? (interned-1 0) (reinterned 0)))
+ (is (identical? (interned-1 2) (reinterned 2)))))
+ (testing "strings are interned even across unequal segments"
+ (is (identical? (:model (interned-1 2)) (:model (interned-2 2)))))
+ (testing "non-string :id segments pass through"
+ (is (= [{:model "Setting" :id :some-key}]
+ (intern-path [(fresh-seg {:model "Setting" :id :some-key})]))))))
+
+(deftest ingest-all-interns-index-keys-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (io/make-parents dump-dir "databases" "fake")
+ (doseq [field-id ["F1" "F2"]]
+ (spit (io/file dump-dir "databases" (str field-id ".yaml"))
+ (yaml/generate-string {:some "data"
+ :serdes/meta [{:model "Database" :id "mydb"}
+ {:model "Table" :id "T"}
+ {:model "Field" :id field-id}]})))
+ (let [{:keys [entities errors]} (#'ingest/ingest-all (io/file dump-dir))
+ [key-1 key-2] (sort-by (comp :id last) (keys entities))]
+ (is (empty? errors))
+ (is (= [{:model "Database" :id "mydb"} {:model "Table" :id "T"} {:model "Field" :id "F1"}]
+ key-1))
+ (testing "sibling index keys share their prefix segments as identical objects"
+ (is (identical? (key-1 0) (key-2 0)))
+ (is (identical? (key-1 1) (key-2 1)))))))
+
+(deftest ingest-all-skips-ineligible-files-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (io/make-parents dump-dir "collections" "fake")
+ (io/make-parents dump-dir "not-a-legal-dir" "fake")
+ (let [entity-yaml (yaml/generate-string {:some "data"
+ :serdes/meta [{:model "Collection" :id "1234567890abcdefABCDE"}]})]
+ (spit (io/file dump-dir "collections" "in.yaml") entity-yaml)
+ (spit (io/file dump-dir "not-a-legal-dir" "out.yaml") entity-yaml)
+ (spit (io/file dump-dir "top-level.yaml") entity-yaml)
+ (spit (io/file dump-dir "collections" "not-yaml.txt") entity-yaml))
+ (let [{:keys [entities errors]} (#'ingest/ingest-all (io/file dump-dir))]
+ (testing "only yaml files under legal top-level paths are indexed, without errors"
+ (is (= #{[{:model "Collection" :id "1234567890abcdefABCDE"}]}
+ (set (keys entities))))
+ (is (= "in.yaml" (.getName ^java.io.File (val (first entities)))))
+ (is (empty? errors))))))
+
(deftest keyword-reconstruction-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(io/make-parents dump-dir "collections" "cards" "fake")
diff --git a/src/metabase/util/memoize.cljc b/src/metabase/util/memoize.cljc
index 916f95b65d6..389c430f462 100644
--- a/src/metabase/util/memoize.cljc
+++ b/src/metabase/util/memoize.cljc
@@ -4,7 +4,9 @@
In CLJ this re-exports some of the public functions from `clojure.core.memoize`.
In CLJS it implements the same design in CLJS.
- There are also some extra memoization tools here, like [[fast-memo]] and [[fast-bounded]]."
+ There are also extra caching primitives here: [[fast-memo]] and [[fast-bounded]] for
+ memoization, and [[fast-interner]] for interning (deduping equal values to one shared
+ instance, rather than caching a function's outputs)."
(:require
#?@(:clj ([clojure.core.memoize :as memoize])
:cljs ([metabase.util.memoize.impl.js :as memoize]))
@@ -59,6 +61,40 @@
(.computeIfAbsent cache k mapping-fn)))
:cljs (memoize/memo f)))
+(defn fast-interner
+ "Interner: collapses `=`-equal values to a single shared (`identical?`) instance, optionally
+ normalizing each value through `canon-fn` first. Like [[fast-memo]] but with two differences
+ that matter for interning:
+
+ - It stores the *canonical* value `(canon-fn x)` as the cache key, not the lookup argument `x`,
+ so the object retained forever is the canonical one (e.g. a copy whose strings are pooled),
+ and the un-normalized input is discarded. `canon-fn` therefore **must return a value `=` to
+ its argument** — it normalizes *within* an equivalence class (pool the contents, reorder a
+ map) without changing equality. The zero-arg arity uses `identity` (intern values as-is).
+ - Concurrent callers converge on a single canonical instance even on a simultaneous miss.
+
+ In CLJ this is backed by a `ConcurrentHashMap`, so it is thread-safe; `canon-fn` may run more
+ than once for the same value under contention (extra results are discarded), so it must be pure.
+ Keys and values must be non-nil with a good `Object.hashCode` (Clojure values qualify). Like
+ [[fast-memo]] it doesn't support the memoization API (`memo-swap!`, `memoized?`, etc.).
+
+ In CLJS (single-threaded) this is a plain atom-backed map with the same contract."
+ ([] (fast-interner identity))
+ ([canon-fn]
+ #?(:clj (let [cache (java.util.concurrent.ConcurrentHashMap.)]
+ (fn [x]
+ (or (.get cache x)
+ (let [canon (canon-fn x)]
+ ;; key by `canon`, not `x`: `canon` = `x`, so this lookup/insert hits the
+ ;; same bin a later equal input would, while retaining the canonical object.
+ (or (.putIfAbsent cache canon canon) canon)))))
+ :cljs (let [cache (atom {})]
+ (fn [x]
+ (or (get @cache x)
+ (let [canon (canon-fn x)]
+ (swap! cache assoc canon canon)
+ canon)))))))
+
(defn fast-bounded
"Variant of [[bounded]] that optimizes a common special case: a function with only a single argument, where that
argument makes a good map key.
diff --git a/test/metabase/util/memoize_test.cljc b/test/metabase/util/memoize_test.cljc
index 2c1b9faec66..7dbf6044e1b 100644
--- a/test/metabase/util/memoize_test.cljc
+++ b/test/metabase/util/memoize_test.cljc
@@ -57,6 +57,21 @@
(deftest ^:parallel fast-bounded-never-evicts-if-large-enough-test
(never-evicts #(memo/fast-bounded % :bounded/threshold 101)))
+(deftest ^:parallel fast-interner-test
+ (testing "equal-but-distinct inputs collapse to one identical canonical object,
+ which is the canonicalized form (sorted-map), not the lookup argument"
+ (let [intern (memo/fast-interner #(into (sorted-map) %))
+ in1 {:b 2 :a 1} ; hash-map
+ in2 {:a 1 :b 2}] ; = in1, distinct object
+ (is (not (identical? in1 in2))) ; genuinely distinct inputs
+ (is (= in1 (intern in1))) ; value equality preserved
+ (is (sorted? (intern in1))) ; canonical form returned, not the input hash-map
+ (is (identical? (intern in1) (intern in2))))) ; equal inputs share the one canonical object
+ (testing "the zero-arg arity interns values as-is"
+ (let [intern (memo/fast-interner)]
+ (is (= :kw (intern :kw)))
+ (is (identical? (intern :kw) (intern :kw))))))
+
(deftest ^:parallel bounded-evicts-when-keyspace-overflows-test
(let [keyspace (gen-keyspace 100)
{:keys [f calls misses reset]} (call-count-harness #(memo/bounded % :bounded/threshold 50) str/reverse)]