Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix issue with closed filesystem for python sources (#73562)

dpsutton1bed702cc2c1
diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.clj
index b9feb786cca..2b18971f69a 100644
--- a/src/metabase/sql_parsing/pool.clj
+++ b/src/metabase/sql_parsing/pool.clj
@@ -19,7 +19,6 @@
(java.io Closeable File)
(java.nio.file FileSystems)
(java.time Duration)
- (java.util Collections)
(java.util.concurrent TimeUnit TimeoutException)
(org.graalvm.polyglot Context HostAccess)
(org.graalvm.polyglot.io FileSystem IOAccess)))
@@ -122,6 +121,11 @@
;;; -------------------------------------------------- Python path delay --------------------------------------------------
+(defn- read-only-polyglot-fs
+ "Wrap an NIO `java.nio.file.FileSystem` as a read-only polyglot FileSystem suitable for GraalPy."
+ ^FileSystem [^java.nio.file.FileSystem nio-fs]
+ (-> nio-fs FileSystem/newFileSystem FileSystem/newReadOnlyFileSystem))
+
(defonce ^:private
^{:doc "A read-only polyglot FileSystem and the PythonPath within it.
In dev: wraps the default filesystem, path is resources/python-sources.
@@ -130,21 +134,17 @@
python-fs-and-path
(delay
(if (jar-resource? python-sources-resource)
- ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib
- ;; are both inside the jar, so nothing is extracted to disk and there's nothing to tamper with.
- (let [jar-path (u.files/get-jar-path)
- jar-uri (java.net.URI. (str "jar:file:" jar-path))
- nio-fs (FileSystems/newFileSystem jar-uri Collections/EMPTY_MAP)]
- {:fs (-> (FileSystem/newFileSystem nio-fs)
- FileSystem/newReadOnlyFileSystem)
- :python-path "/python-sources"
- :std-lib-home "/META-INF/resources/libpython"
- :core-home "/META-INF/resources/libgraalpy"})
+ ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib are both inside the
+ ;; jar, so nothing is extracted to disk and there's nothing to tamper with. The filesystem lives for the
+ ;; duration of the process (via defonce + delay) and is shared by all pooled Python contexts.
+ {:fs (-> (u.files/get-jar-path) u.files/nio-fs read-only-polyglot-fs)
+ :python-path "/python-sources"
+ :std-lib-home "/META-INF/resources/libpython"
+ :core-home "/META-INF/resources/libgraalpy"}
;; In dev: use the real filesystem (read-only wrapper). sqlglot is installed locally.
(do
(ensure-sqlglot-installed!)
- {:fs (-> (FileSystem/newFileSystem (FileSystems/getDefault))
- FileSystem/newReadOnlyFileSystem)
+ {:fs (read-only-polyglot-fs (FileSystems/getDefault))
:python-path dev-python-sources-dir}))))
;;; -------------------------------------------- Context Wrappers ----------------------------------------------------
diff --git a/src/metabase/util/files.clj b/src/metabase/util/files.clj
index 519ee72516c..57cbfca629c 100644
--- a/src/metabase/util/files.clj
+++ b/src/metabase/util/files.clj
@@ -117,6 +117,18 @@
(log/info "File system at" uri "already exists")
(FileSystems/getFileSystem uri)))))
+(defn nio-fs
+ "Open a fresh NIO zip filesystem for the file at `path`."
+ ^FileSystem [^String path]
+ ;; Use the Path-based FileSystems/newFileSystem overload, NOT the URI-based one (see jar-file-system-from-url above
+ ;; for the URI-based pattern). NIO maintains a global cache of zip filesystems keyed by URI; the URI-based overload
+ ;; registers in that cache, so other code (e.g. with-open-path-to-resource, find-in-current-jar) can obtain the
+ ;; cached instance via FileSystems/getFileSystem and close it inside a with-open block, killing the original
+ ;; filesystem and breaking any long-lived consumer (e.g. GraalPy contexts pinned to that filesystem). The Path-based
+ ;; overload bypasses the cache, giving an independent instance whose lifecycle the caller fully controls.
+ (-> (Path/of path (u/varargs String))
+ (FileSystems/newFileSystem Collections/EMPTY_MAP)))
+
(defn do-with-open-path-to-resource
"Impl for `with-open-path-to-resource`."
[resource f]
diff --git a/test/metabase/util/files_test.clj b/test/metabase/util/files_test.clj
index 0d009b2a237..af94dbc31e4 100644
--- a/test/metabase/util/files_test.clj
+++ b/test/metabase/util/files_test.clj
@@ -2,7 +2,14 @@
(:require
[clojure.test :refer :all]
[metabase.test :as mt]
- [metabase.util.files :as u.files]))
+ [metabase.util :as u]
+ [metabase.util.files :as u.files])
+ (:import
+ (java.io FileOutputStream)
+ (java.nio.file ClosedFileSystemException FileSystem Files)
+ (java.util.zip ZipEntry ZipOutputStream)))
+
+(set! *warn-on-reflection* true)
(deftest is-regular-file-test
(mt/with-temp-file [file "temp-file"]
@@ -13,6 +20,35 @@
(testing (format "dir = %s" (pr-str dir)))
(let [file-in-dir (str (u.files/get-path dir "file"))]
(testing (format "file = %s" (pr-str file-in-dir))
- (spit file-in-dir "abc") ; create a file in the dir to make sure it exists
+ (spit file-in-dir "abc") ; create a file in the dir to make sure it exists
(is (u.files/regular-file? (u.files/get-path file-in-dir)))))
(is (not (u.files/regular-file? (u.files/get-path dir))))))
+
+(defn- write-test-zip!
+ "Write a minimal zip file at `path` containing a single `hello.txt` entry with content `hi`."
+ [^String path]
+ (with-open [zos (ZipOutputStream. (FileOutputStream. path))]
+ (.putNextEntry zos (ZipEntry. "hello.txt"))
+ (.write zos (.getBytes "hi"))
+ (.closeEntry zos)))
+
+(defn- read-hello [^FileSystem fs]
+ (Files/readString (.getPath fs "/hello.txt" (u/varargs String))))
+
+(deftest nio-fs-bypasses-uri-cache-test
+ (testing "nio-fs returns independent filesystems for the same zip — closing one cannot kill another"
+ (mt/with-temp-file [zip-path]
+ (write-test-zip! zip-path)
+ (with-open [fs1 (u.files/nio-fs zip-path)
+ fs2 (u.files/nio-fs zip-path)]
+ ;; Both filesystems read the entry — distinguishes Path-based (works) from URI-based (would throw
+ ;; FileSystemAlreadyExistsException on the second open since the cache entry would still be live).
+ (is (= "hi" (read-hello fs1)))
+ (is (= "hi" (read-hello fs2)))
+ ;; Closing fs1 must not affect fs2.
+ (.close fs1)
+ (is (thrown? ClosedFileSystemException (read-hello fs1)))
+ (is (= "hi" (read-hello fs2))))
+ ;; Subsequent opens still work after previous instance closes — not in a permanently-broken state.
+ (with-open [fs (u.files/nio-fs zip-path)]
+ (is (= "hi" (read-hello fs)))))))