The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix memory leak with GraalVM (#75982)
Alexander Polyankinbc6bc90b7e1c
diff --git a/src/metabase/channel/render/js/engine.clj b/src/metabase/channel/render/js/engine.cljindex daa169ad574..9cb4536c684 100644--- a/src/metabase/channel/render/js/engine.clj+++ b/src/metabase/channel/render/js/engine.clj@@ -11,10 +11,51 @@[clojure.java.io :as io][metabase.util.i18n :refer [trs]])(:import- (org.graalvm.polyglot Context HostAccess Source Value)))+ (org.graalvm.polyglot Context Engine HostAccess Source Value)))(set! *warn-on-reflection* true)+;; GraalVM requires every context that shares an `Engine` to use the *identical* host-access configuration, so these+;; are singletons (one shared instance, not rebuilt per context) and there is a separate engine per access level.++(def ^:private no-host-class-lookup+ "Predicate blocking all host class lookup. A singleton so all contexts sharing an engine have identical config."+ (reify java.util.function.Predicate+ (test [_ _] false)))++(def ^:private ^HostAccess trusted-host-access+ "Host access for trusted first-party JS: read Clojure collections (list/array/iterable), but no class lookup,+ method invocation, or filesystem I/O. A singleton (see above)."+ (.. (HostAccess/newBuilder)+ (allowListAccess true)+ (allowArrayAccess true)+ (allowIterableAccess true)+ (build)))++(defn- new-js-engine+ "Build a JS `Engine` to be shared across contexts. We run JS interpreted (no Graal compiler on a stock JDK), so+ `engine.WarnInterpreterOnly` is silenced here on the engine (it's an engine-level option)."+ ^Engine []+ ;; https://github.com/oracle/graaljs/blob/master/docs/user/RunOnJDK.md+ (.. (Engine/newBuilder)+ (option "engine.WarnInterpreterOnly" "false")+ (build)))++(defonce ^:private+ ^{:doc "GraalVM `Engine` shared by every *sandboxed* JS context (e.g. static-viz, untrusted custom-viz plugins). The+ engine owns the Truffle runtime + parsed-source cache, so contexts (and pool recycles) reuse one engine+ instead of each standing up its own — the per-context engine native memory is a source of the leak. Sharing+ requires identical host access across its contexts, hence a dedicated engine for the sandboxed config. The+ engine is a process-lifetime singleton (intentionally never closed)."}+ shared-sandboxed-js-engine+ (delay (new-js-engine)))++(defonce ^:private+ ^{:doc "GraalVM `Engine` shared by every *trusted* JS context (see [[trusted-context]]). Separate from+ [[shared-sandboxed-js-engine]] because contexts sharing an engine must use identical host access."}+ shared-trusted-js-engine+ (delay (new-js-engine)))+(defn threadlocal-fifo-memoizer"Returns a memoizer that is unique to each thread."[thunk threshold]@@ -29,12 +70,10 @@All data must be passed as JSON strings and parsed in JS."^Context [](.. (Context/newBuilder (into-array String ["js"]))- ;; https://github.com/oracle/graaljs/blob/master/docs/user/RunOnJDK.md- (option "engine.WarnInterpreterOnly" "false")+ (engine @shared-sandboxed-js-engine)(option "js.intl-402" "true")(allowHostAccess HostAccess/NONE)- (allowHostClassLookup (reify java.util.function.Predicate- (test [_ _] false)))+ (allowHostClassLookup no-host-class-lookup)(out System/out)(err System/err)(allowIO false)@@ -46,15 +85,10 @@Java class lookup, method invocation, and filesystem I/O."^Context [](.. (Context/newBuilder (into-array String ["js"]))- (option "engine.WarnInterpreterOnly" "false")+ (engine @shared-trusted-js-engine)(option "js.intl-402" "true")- (allowHostAccess (.. (HostAccess/newBuilder)- (allowListAccess true)- (allowArrayAccess true)- (allowIterableAccess true)- (build)))- (allowHostClassLookup (reify java.util.function.Predicate- (test [_ _] false)))+ (allowHostAccess trusted-host-access)+ (allowHostClassLookup no-host-class-lookup)(out System/out)(err System/err)(allowIO false)diff --git a/src/metabase/channel/render/js/svg.clj b/src/metabase/channel/render/js/svg.cljindex 2587f2af85d..20080a8c24b 100644--- a/src/metabase/channel/render/js/svg.clj+++ b/src/metabase/channel/render/js/svg.clj@@ -58,10 +58,15 @@(let [base-controller (Pools/utilizationController 1.0 3 3)](Pool. (reify IPool$Generator(generate [_ _]- ;; Generate a tuple of the engine and the expiry timestamp.+ ;; Generate a tuple of the context and the expiry timestamp.[(load-viz-bundle (js.engine/context))(+ (System/nanoTime) (.toNanos TimeUnit/MINUTES 10))])- (destroy [_ _ _v]))+ (destroy [_ _ [^Context ctx _expiry]]+ ;; Close the context when it's disposed from the pool (expiry/shutdown). Without this, each disposed+ ;; static-viz context (~130 MB) leaks its native memory: GraalVM only releases it on `close`, not on GC.+ (try+ (.close ctx true) ;; force close - can't wait for running code+ (catch Exception _))));; Wrap the utilization controller with a modification that doesn't allow the pool to go below 1 instance.(reify IPool$Controller(shouldIncrement [_ k a b] (.shouldIncrement base-controller k a b))diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.cljindex 26cc98063d2..d9790504509 100644--- a/src/metabase/sql_parsing/pool.clj+++ b/src/metabase/sql_parsing/pool.clj@@ -23,7 +23,7 @@(java.time Duration)(java.util Map Set)(java.util.concurrent TimeUnit TimeoutException)- (org.graalvm.polyglot Context HostAccess)+ (org.graalvm.polyglot Context Engine HostAccess)(org.graalvm.polyglot.io FileSystem IOAccess)))(set! *warn-on-reflection* true)@@ -224,10 +224,24 @@(.close raw-ctx true)false))))+(defonce ^:private+ ^{:doc "A single GraalVM `Engine` shared by every pooled Python context. The engine owns the Truffle runtime and the+ (parsed/compiled) code cache; sharing it across contexts means recycling a context on its TTL no longer+ recreates an engine or re-loads sqlglot into a fresh code cache. That per-context engine state is the dominant+ native-memory cost and, because contexts churn on a 10-minute TTL and GraalPy doesn't reliably return all of+ it to the OS on close, a major source of the native-memory growth. The engine is a process-lifetime singleton+ (it is intentionally never closed)."}+ shared-python-engine+ (delay+ (.. (Engine/newBuilder)+ (option "engine.WarnInterpreterOnly" "false")+ (build))))+(defn- create-graalvm-context"Create a new GraalVM Python context configured for sqlglot.The context is configured with:+ - The shared [[shared-python-engine]] (so contexts share one Truffle runtime + code cache)- PythonPath pointing to python-sources (contains sql_tools.py and sqlglot)- Full host access (needed for JSON serialization)- Read-only filesystem (sources loaded directly from classpath/jar, no disk extraction in prod)"@@ -237,7 +251,7 @@(fileSystem fs)(build))builder (.. (Context/newBuilder (into-array String ["python"]))- (option "engine.WarnInterpreterOnly" "false")+ (engine @shared-python-engine)(option "python.PythonPath" python-path)(allowHostAccess HostAccess/ALL)(allowIO io-access))