The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-376] [GDGT-2624] fix serdes import oom (#75786)
YB403107f13fda
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.cljindex c90393d4394..70c74b87efe 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj@@ -10,6 +10,7 @@[metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.log :as log]+ [metabase.util.memoize :as u.memo][metabase.util.yaml :as yaml][potemkin.types :as p])(:import (java.io File)))@@ -84,27 +85,59 @@We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."#{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "curated_search_entries" "snippets" "transforms"})+(defn- path-interner+ "Returns a function that interns `:serdes/meta` path vectors.++ Every parsed file allocates fresh copies of its path maps and strings, but the+ `[db schema table]` prefix segments recur once per sibling file (a million-field instance+ has ~#tables distinct prefixes). Interning each segment (and its strings) collapses those+ duplicates to shared objects, so index keys share structure — value equality is unchanged,+ so lookups and `:seen`-set comparisons behave identically. The interner caches live only as+ long as the returned function is reachable; do not hold it past index construction."+ []+ (let [intern-str (u.memo/fast-interner)+ intern-seg (u.memo/fast-interner+ (fn [seg]+ (cond-> (update seg :model intern-str)+ (string? (:id seg)) (update :id intern-str))))]+ (fn intern-path [hierarchy]+ (mapv intern-seg hierarchy))))++(defn- ingestible-file?+ "Whether `file` is a regular `.yaml` file under one of the [[legal-top-level-paths]].+ Dotfiles are excluded (editor temp files, see #41567)."+ [^File root-dir ^File file]+ (boolean (and (.isFile file)+ (not (str/starts-with? (.getName file) "."))+ (str/ends-with? (.getName file) ".yaml")+ (let [rel (.relativize (.toPath root-dir) (.toPath file))]+ (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))))++(defn- file-hierarchy!+ "Parses `file` and returns its `:serdes/meta` abstract path, or nil on parse failure.+ On failure the exception is recorded in the `errors` atom."+ [^File file errors]+ (try+ (serdes/path (ingest-file file))+ (catch Exception e+ (log/warn (u/strip-error e "Error reading file during ingestion"))+ (let [file-name (.getName file)]+ (swap! errors conj (ex-info (format "Failed to parse file: %s" file-name)+ {:file file-name} e)))+ nil)))+(defn- ingest-all- "Returns {:entities {unlabeled-hierarchy [hierarchy File]}, :errors [Exception...]}.+ "Returns {:entities {unlabeled-hierarchy File}, :errors [Exception...]}.Dotfiles are silently skipped (editor temp files, see #41567).Non-dotfile YAML parse failures are collected in :errors."[^File root-dir]- (let [errors (atom [])]+ (let [errors (atom [])+ intern-path (path-interner)]{:entities (into {} (for [^File file (file-seq root-dir)- :when (and (.isFile file)- (not (str/starts-with? (.getName file) "."))- (str/ends-with? (.getName file) ".yaml")- (let [rel (.relativize (.toPath root-dir) (.toPath file))]- (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))- :let [file-name (.getName file)- hierarchy (try- (serdes/path (ingest-file file))- (catch Exception e- (log/warn (u/strip-error e "Error reading file during ingestion"))- (swap! errors conj (ex-info (format "Failed to parse file: %s" file-name) {:file file-name} e))- nil))]+ :when (ingestible-file? root-dir file)+ :let [hierarchy (file-hierarchy! file errors)]:when hierarchy]- [(strip-labels hierarchy) [hierarchy file]]))+ [(intern-path (strip-labels hierarchy)) file])):errors @errors}))(defn- populate-cache! [cache errors-atom ingest-fn]@@ -129,11 +162,11 @@(if (= ["Setting"] (mapv :model serdes-meta))(when (contains? settings kw-id){:serdes/meta serdes-meta :key kw-id :value (get settings kw-id)})- (when-let [target (get @cache (strip-labels serdes-meta))]+ (when-let [file (get @cache (strip-labels serdes-meta))](try- (ingest-file (second target))+ (ingest-file file)(catch Exception e- (throw (ex-info "Unable to ingest file" {:file (.getName ^File (second target))+ (throw (ex-info "Unable to ingest file" {:file (.getName ^File file):abs-path serdes-meta} e))))))))(ingest-errors [_]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex 32092244269..294f761f4ae 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -4,7 +4,6 @@(:require[clojure.string :as str][diehard.core :as dh]- [medley.core :as m][metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill][metabase-enterprise.serialization.v2.ingest :as serdes.ingest][metabase-enterprise.serialization.v2.models :as serdes.models]@@ -248,7 +247,6 @@:seen #{}:circular #{}:ingestion ingestion- :from-ids (->> ingestion serdes.ingest/ingest-list (m/index-by :id)):errors []})(defn load-metabase!diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex 46c1767b23e..191fd68fbba 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -841,7 +841,7 @@(testing "Hidden YAML files are still silently skipped"(let [{:keys [entities]} (#'ingest/ingest-all (io/file dump-dir))files (->> entities- (map (comp second second))+ (map val)(map #(.getName ^File %))set)](is (not (contains? files ".hidden.yaml")))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.cljindex 5837ef28e22..1b9188127bf 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest_test.clj@@ -6,6 +6,8 @@[metabase-enterprise.serialization.v2.ingest :as ingest][metabase.util.yaml :as yaml]))+(set! *warn-on-reflection* true)+(deftest basic-ingest-test(ts/with-random-dump-dir [dump-dir "serdesv2-"](io/make-parents dump-dir "collections" "1234567890abcdefABCDE_the_label" "fake") ; Prepare the right directories.@@ -65,6 +67,67 @@(is (= exp(ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE"}])))))))+(deftest path-interner-test+ (let [intern-path (#'ingest/path-interner)+ ;; rebuild every map and string so equal segments are distinct objects, like segments+ ;; parsed from separate YAML files+ fresh-seg (fn [seg] (update-vals seg #(if (string? %) (String. ^String %) %)))+ prefix [{:model "Database" :id "mydb"} {:model "Table" :id "T"}]+ path-1 (mapv fresh-seg (conj prefix {:model "Field" :id "F1"}))+ path-2 (mapv fresh-seg (conj prefix {:model "Field" :id "F2"}))+ interned-1 (intern-path path-1)+ interned-2 (intern-path path-2)]+ (testing "value equality is preserved"+ (is (= path-1 interned-1))+ (is (= path-2 interned-2)))+ (testing "equal prefix segments from different paths intern to the identical object"+ (is (identical? (interned-1 0) (interned-2 0)))+ (is (identical? (interned-1 1) (interned-2 1))))+ (testing "re-interning a fresh copy returns the same canonical objects"+ (let [reinterned (intern-path (mapv fresh-seg path-1))]+ (is (identical? (interned-1 0) (reinterned 0)))+ (is (identical? (interned-1 2) (reinterned 2)))))+ (testing "strings are interned even across unequal segments"+ (is (identical? (:model (interned-1 2)) (:model (interned-2 2)))))+ (testing "non-string :id segments pass through"+ (is (= [{:model "Setting" :id :some-key}]+ (intern-path [(fresh-seg {:model "Setting" :id :some-key})]))))))++(deftest ingest-all-interns-index-keys-test+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]+ (io/make-parents dump-dir "databases" "fake")+ (doseq [field-id ["F1" "F2"]]+ (spit (io/file dump-dir "databases" (str field-id ".yaml"))+ (yaml/generate-string {:some "data"+ :serdes/meta [{:model "Database" :id "mydb"}+ {:model "Table" :id "T"}+ {:model "Field" :id field-id}]})))+ (let [{:keys [entities errors]} (#'ingest/ingest-all (io/file dump-dir))+ [key-1 key-2] (sort-by (comp :id last) (keys entities))]+ (is (empty? errors))+ (is (= [{:model "Database" :id "mydb"} {:model "Table" :id "T"} {:model "Field" :id "F1"}]+ key-1))+ (testing "sibling index keys share their prefix segments as identical objects"+ (is (identical? (key-1 0) (key-2 0)))+ (is (identical? (key-1 1) (key-2 1)))))))++(deftest ingest-all-skips-ineligible-files-test+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]+ (io/make-parents dump-dir "collections" "fake")+ (io/make-parents dump-dir "not-a-legal-dir" "fake")+ (let [entity-yaml (yaml/generate-string {:some "data"+ :serdes/meta [{:model "Collection" :id "1234567890abcdefABCDE"}]})]+ (spit (io/file dump-dir "collections" "in.yaml") entity-yaml)+ (spit (io/file dump-dir "not-a-legal-dir" "out.yaml") entity-yaml)+ (spit (io/file dump-dir "top-level.yaml") entity-yaml)+ (spit (io/file dump-dir "collections" "not-yaml.txt") entity-yaml))+ (let [{:keys [entities errors]} (#'ingest/ingest-all (io/file dump-dir))]+ (testing "only yaml files under legal top-level paths are indexed, without errors"+ (is (= #{[{:model "Collection" :id "1234567890abcdefABCDE"}]}+ (set (keys entities))))+ (is (= "in.yaml" (.getName ^java.io.File (val (first entities)))))+ (is (empty? errors))))))+(deftest keyword-reconstruction-test(ts/with-random-dump-dir [dump-dir "serdesv2-"](io/make-parents dump-dir "collections" "cards" "fake")diff --git a/src/metabase/util/memoize.cljc b/src/metabase/util/memoize.cljcindex 916f95b65d6..389c430f462 100644--- a/src/metabase/util/memoize.cljc+++ b/src/metabase/util/memoize.cljc@@ -4,7 +4,9 @@In CLJ this re-exports some of the public functions from `clojure.core.memoize`.In CLJS it implements the same design in CLJS.- There are also some extra memoization tools here, like [[fast-memo]] and [[fast-bounded]]."+ There are also extra caching primitives here: [[fast-memo]] and [[fast-bounded]] for+ memoization, and [[fast-interner]] for interning (deduping equal values to one shared+ instance, rather than caching a function's outputs)."(:require#?@(:clj ([clojure.core.memoize :as memoize]):cljs ([metabase.util.memoize.impl.js :as memoize]))@@ -59,6 +61,40 @@(.computeIfAbsent cache k mapping-fn))):cljs (memoize/memo f)))+(defn fast-interner+ "Interner: collapses `=`-equal values to a single shared (`identical?`) instance, optionally+ normalizing each value through `canon-fn` first. Like [[fast-memo]] but with two differences+ that matter for interning:++ - It stores the *canonical* value `(canon-fn x)` as the cache key, not the lookup argument `x`,+ so the object retained forever is the canonical one (e.g. a copy whose strings are pooled),+ and the un-normalized input is discarded. `canon-fn` therefore **must return a value `=` to+ its argument** — it normalizes *within* an equivalence class (pool the contents, reorder a+ map) without changing equality. The zero-arg arity uses `identity` (intern values as-is).+ - Concurrent callers converge on a single canonical instance even on a simultaneous miss.++ In CLJ this is backed by a `ConcurrentHashMap`, so it is thread-safe; `canon-fn` may run more+ than once for the same value under contention (extra results are discarded), so it must be pure.+ Keys and values must be non-nil with a good `Object.hashCode` (Clojure values qualify). Like+ [[fast-memo]] it doesn't support the memoization API (`memo-swap!`, `memoized?`, etc.).++ In CLJS (single-threaded) this is a plain atom-backed map with the same contract."+ ([] (fast-interner identity))+ ([canon-fn]+ #?(:clj (let [cache (java.util.concurrent.ConcurrentHashMap.)]+ (fn [x]+ (or (.get cache x)+ (let [canon (canon-fn x)]+ ;; key by `canon`, not `x`: `canon` = `x`, so this lookup/insert hits the+ ;; same bin a later equal input would, while retaining the canonical object.+ (or (.putIfAbsent cache canon canon) canon)))))+ :cljs (let [cache (atom {})]+ (fn [x]+ (or (get @cache x)+ (let [canon (canon-fn x)]+ (swap! cache assoc canon canon)+ canon)))))))+(defn fast-bounded"Variant of [[bounded]] that optimizes a common special case: a function with only a single argument, where thatargument makes a good map key.diff --git a/test/metabase/util/memoize_test.cljc b/test/metabase/util/memoize_test.cljcindex 2c1b9faec66..7dbf6044e1b 100644--- a/test/metabase/util/memoize_test.cljc+++ b/test/metabase/util/memoize_test.cljc@@ -57,6 +57,21 @@(deftest ^:parallel fast-bounded-never-evicts-if-large-enough-test(never-evicts #(memo/fast-bounded % :bounded/threshold 101)))+(deftest ^:parallel fast-interner-test+ (testing "equal-but-distinct inputs collapse to one identical canonical object,+ which is the canonicalized form (sorted-map), not the lookup argument"+ (let [intern (memo/fast-interner #(into (sorted-map) %))+ in1 {:b 2 :a 1} ; hash-map+ in2 {:a 1 :b 2}] ; = in1, distinct object+ (is (not (identical? in1 in2))) ; genuinely distinct inputs+ (is (= in1 (intern in1))) ; value equality preserved+ (is (sorted? (intern in1))) ; canonical form returned, not the input hash-map+ (is (identical? (intern in1) (intern in2))))) ; equal inputs share the one canonical object+ (testing "the zero-arg arity interns values as-is"+ (let [intern (memo/fast-interner)]+ (is (= :kw (intern :kw)))+ (is (identical? (intern :kw) (intern :kw))))))+(deftest ^:parallel bounded-evicts-when-keyspace-overflows-test(let [keyspace (gen-keyspace 100){:keys [f calls misses reset]} (call-count-harness #(memo/bounded % :bounded/threshold 50) str/reverse)]