Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Read Python directly from uberjar (#73527)

Cam Sauld618a0622f04
diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.clj
index 5904898a255..b9feb786cca 100644
--- a/src/metabase/sql_parsing/pool.clj
+++ b/src/metabase/sql_parsing/pool.clj
@@ -5,7 +5,6 @@
(:require
[clojure.java.io :as io]
[clojure.java.shell :as shell]
- [clojure.string :as str]
[metabase.analytics-interface.core :as analytics]
[metabase.sql-parsing.common :as common]
[metabase.util.files :as u.files]
@@ -18,10 +17,12 @@
Pool
Pools)
(java.io Closeable File)
- (java.nio.file Files Path)
+ (java.nio.file FileSystems)
(java.time Duration)
+ (java.util Collections)
(java.util.concurrent TimeUnit TimeoutException)
- (org.graalvm.polyglot Context HostAccess)))
+ (org.graalvm.polyglot Context HostAccess)
+ (org.graalvm.polyglot.io FileSystem IOAccess)))
(set! *warn-on-reflection* true)
@@ -31,16 +32,6 @@
"Resource path for Python sources (sql_tools.py and sqlglot)."
"python-sources")
-(defn- plugins-dir-path
- "Get path to plugins directory. Lightweight alternative to metabase.plugins.core/plugins-dir
- that avoids loading the entire plugin system (~8 seconds of namespace loading)."
- ^Path []
- (let [dir-name (or (System/getenv "MB_PLUGINS_DIR")
- "plugins")
- path (u.files/get-path dir-name)]
- (u.files/create-dir-if-not-exists! path)
- path))
-
(defn- jar-resource?
"True if the given resource is inside a JAR (production), false if from filesystem (dev)."
[resource]
@@ -48,41 +39,6 @@
(when-let [url (io/resource resource)]
(.contains (.getFile ^java.net.URL url) ".jar!/"))))
-(defn- copy-dir-recursive!
- "Recursively copy all files from source Path to dest Path.
- Works across filesystems (e.g., from JAR filesystem to default filesystem)."
- [^Path source ^Path dest]
- (doseq [^Path child (u.files/files-seq source)]
- (let [relative-name (str (.getFileName child))
- target (.resolve dest relative-name)]
- (if (Files/isDirectory child (into-array java.nio.file.LinkOption []))
- (do
- (u.files/create-dir-if-not-exists! target)
- (copy-dir-recursive! child target))
- (log/with-no-logs ;; <-- Suppress per-file logs from copy-file! (hundreds of files in sqlglot)
- (u.files/copy-file! child target))))))
-
-(defn- extract-python-sources!
- "Extract python-sources from JAR to plugins directory. Returns the path as a string.
- Uses the same plugins directory as driver modules for consistency.
- Skips extraction if version file matches."
- ^String []
- (let [^Path plugins-path (plugins-dir-path)
- dest-path (.resolve plugins-path "python-sources")
- version-file (.resolve dest-path "pyproject.toml")
- jar-version (some-> (io/resource "python-sources/pyproject.toml") slurp str/trim)
- dest-version (when (u.files/exists? version-file)
- (str/trim (slurp (.toFile version-file))))]
- (if (and jar-version (= jar-version dest-version))
- (log/info "Python sources already extracted (version" jar-version ")")
- (do
- (u.files/create-dir-if-not-exists! dest-path)
- (log/info "Extracting Python sources to" (str dest-path))
- (u.files/with-open-path-to-resource [source-path python-sources-resource]
- (copy-dir-recursive! source-path dest-path))
- (log/info "Python sources extracted")))
- (str dest-path)))
-
;;; -------------------------------------------------- Dev lazy installation --------------------------------------------------
(def ^:private dev-python-sources-dir
@@ -167,16 +123,29 @@
;;; -------------------------------------------------- Python path delay --------------------------------------------------
(defonce ^:private
- ^{:doc "Path to Python sources directory. In dev, this is resources/python-sources.
- When running from a JAR, we extract to the plugins directory.
- In dev, lazily installs sqlglot if not present or version mismatched."}
- python-path
+ ^{:doc "A read-only polyglot FileSystem and the PythonPath within it.
+ In dev: wraps the default filesystem, path is resources/python-sources.
+ In jar: wraps the jar's zip filesystem, path is /python-sources.
+ Either way, the filesystem is read-only so extracted sources cannot be tampered with."}
+ python-fs-and-path
(delay
(if (jar-resource? python-sources-resource)
- (extract-python-sources!)
+ ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib
+ ;; are both inside the jar, so nothing is extracted to disk and there's nothing to tamper with.
+ (let [jar-path (u.files/get-jar-path)
+ jar-uri (java.net.URI. (str "jar:file:" jar-path))
+ nio-fs (FileSystems/newFileSystem jar-uri Collections/EMPTY_MAP)]
+ {:fs (-> (FileSystem/newFileSystem nio-fs)
+ FileSystem/newReadOnlyFileSystem)
+ :python-path "/python-sources"
+ :std-lib-home "/META-INF/resources/libpython"
+ :core-home "/META-INF/resources/libgraalpy"})
+ ;; In dev: use the real filesystem (read-only wrapper). sqlglot is installed locally.
(do
(ensure-sqlglot-installed!)
- dev-python-sources-dir))))
+ {:fs (-> (FileSystem/newFileSystem (FileSystems/getDefault))
+ FileSystem/newReadOnlyFileSystem)
+ :python-path dev-python-sources-dir}))))
;;; -------------------------------------------- Context Wrappers ----------------------------------------------------
@@ -224,15 +193,23 @@
The context is configured with:
- PythonPath pointing to python-sources (contains sql_tools.py and sqlglot)
- Full host access (needed for JSON serialization)
- - IO access enabled (for Python imports)"
+ - Read-only filesystem (sources loaded directly from classpath/jar, no disk extraction in prod)"
^Context []
- (let [ctx (.. (Context/newBuilder (into-array String ["python"]))
- (option "engine.WarnInterpreterOnly" "false")
- ;; python-sources contains both sql_tools.py shim and installed sqlglot
- (option "python.PythonPath" @python-path)
- (allowHostAccess HostAccess/ALL)
- (allowIO true)
- (build))]
+ (let [{:keys [^FileSystem fs python-path std-lib-home core-home]} @python-fs-and-path
+ io-access (.. (IOAccess/newBuilder)
+ (fileSystem fs)
+ (build))
+ builder (.. (Context/newBuilder (into-array String ["python"]))
+ (option "engine.WarnInterpreterOnly" "false")
+ (option "python.PythonPath" python-path)
+ (allowHostAccess HostAccess/ALL)
+ (allowIO io-access))
+ ;; When running from the jar, GraalPy's stdlib is also inside the jar filesystem
+ ;; and we need to tell it where to find the core and stdlib paths explicitly.
+ builder (cond-> builder
+ std-lib-home (.option "python.StdLibHome" std-lib-home)
+ core-home (.option "python.CoreHome" core-home))
+ ctx (.build builder)]
(.eval ctx "python" "import sql_tools")
ctx))