The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix issue with closed filesystem for python sources (#73562)
dpsutton1bed702cc2c1
diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.cljindex b9feb786cca..2b18971f69a 100644--- a/src/metabase/sql_parsing/pool.clj+++ b/src/metabase/sql_parsing/pool.clj@@ -19,7 +19,6 @@(java.io Closeable File)(java.nio.file FileSystems)(java.time Duration)- (java.util Collections)(java.util.concurrent TimeUnit TimeoutException)(org.graalvm.polyglot Context HostAccess)(org.graalvm.polyglot.io FileSystem IOAccess)))@@ -122,6 +121,11 @@;;; -------------------------------------------------- Python path delay --------------------------------------------------+(defn- read-only-polyglot-fs+ "Wrap an NIO `java.nio.file.FileSystem` as a read-only polyglot FileSystem suitable for GraalPy."+ ^FileSystem [^java.nio.file.FileSystem nio-fs]+ (-> nio-fs FileSystem/newFileSystem FileSystem/newReadOnlyFileSystem))+(defonce ^:private^{:doc "A read-only polyglot FileSystem and the PythonPath within it.In dev: wraps the default filesystem, path is resources/python-sources.@@ -130,21 +134,17 @@python-fs-and-path(delay(if (jar-resource? python-sources-resource)- ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib- ;; are both inside the jar, so nothing is extracted to disk and there's nothing to tamper with.- (let [jar-path (u.files/get-jar-path)- jar-uri (java.net.URI. (str "jar:file:" jar-path))- nio-fs (FileSystems/newFileSystem jar-uri Collections/EMPTY_MAP)]- {:fs (-> (FileSystem/newFileSystem nio-fs)- FileSystem/newReadOnlyFileSystem)- :python-path "/python-sources"- :std-lib-home "/META-INF/resources/libpython"- :core-home "/META-INF/resources/libgraalpy"})+ ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib are both inside the+ ;; jar, so nothing is extracted to disk and there's nothing to tamper with. The filesystem lives for the+ ;; duration of the process (via defonce + delay) and is shared by all pooled Python contexts.+ {:fs (-> (u.files/get-jar-path) u.files/nio-fs read-only-polyglot-fs)+ :python-path "/python-sources"+ :std-lib-home "/META-INF/resources/libpython"+ :core-home "/META-INF/resources/libgraalpy"};; In dev: use the real filesystem (read-only wrapper). sqlglot is installed locally.(do(ensure-sqlglot-installed!)- {:fs (-> (FileSystem/newFileSystem (FileSystems/getDefault))- FileSystem/newReadOnlyFileSystem)+ {:fs (read-only-polyglot-fs (FileSystems/getDefault)):python-path dev-python-sources-dir}))));;; -------------------------------------------- Context Wrappers ----------------------------------------------------diff --git a/src/metabase/util/files.clj b/src/metabase/util/files.cljindex 519ee72516c..57cbfca629c 100644--- a/src/metabase/util/files.clj+++ b/src/metabase/util/files.clj@@ -117,6 +117,18 @@(log/info "File system at" uri "already exists")(FileSystems/getFileSystem uri)))))+(defn nio-fs+ "Open a fresh NIO zip filesystem for the file at `path`."+ ^FileSystem [^String path]+ ;; Use the Path-based FileSystems/newFileSystem overload, NOT the URI-based one (see jar-file-system-from-url above+ ;; for the URI-based pattern). NIO maintains a global cache of zip filesystems keyed by URI; the URI-based overload+ ;; registers in that cache, so other code (e.g. with-open-path-to-resource, find-in-current-jar) can obtain the+ ;; cached instance via FileSystems/getFileSystem and close it inside a with-open block, killing the original+ ;; filesystem and breaking any long-lived consumer (e.g. GraalPy contexts pinned to that filesystem). The Path-based+ ;; overload bypasses the cache, giving an independent instance whose lifecycle the caller fully controls.+ (-> (Path/of path (u/varargs String))+ (FileSystems/newFileSystem Collections/EMPTY_MAP)))+(defn do-with-open-path-to-resource"Impl for `with-open-path-to-resource`."[resource f]diff --git a/test/metabase/util/files_test.clj b/test/metabase/util/files_test.cljindex 0d009b2a237..af94dbc31e4 100644--- a/test/metabase/util/files_test.clj+++ b/test/metabase/util/files_test.clj@@ -2,7 +2,14 @@(:require[clojure.test :refer :all][metabase.test :as mt]- [metabase.util.files :as u.files]))+ [metabase.util :as u]+ [metabase.util.files :as u.files])+ (:import+ (java.io FileOutputStream)+ (java.nio.file ClosedFileSystemException FileSystem Files)+ (java.util.zip ZipEntry ZipOutputStream)))++(set! *warn-on-reflection* true)(deftest is-regular-file-test(mt/with-temp-file [file "temp-file"]@@ -13,6 +20,35 @@(testing (format "dir = %s" (pr-str dir)))(let [file-in-dir (str (u.files/get-path dir "file"))](testing (format "file = %s" (pr-str file-in-dir))- (spit file-in-dir "abc") ; create a file in the dir to make sure it exists+ (spit file-in-dir "abc") ; create a file in the dir to make sure it exists(is (u.files/regular-file? (u.files/get-path file-in-dir)))))(is (not (u.files/regular-file? (u.files/get-path dir))))))++(defn- write-test-zip!+ "Write a minimal zip file at `path` containing a single `hello.txt` entry with content `hi`."+ [^String path]+ (with-open [zos (ZipOutputStream. (FileOutputStream. path))]+ (.putNextEntry zos (ZipEntry. "hello.txt"))+ (.write zos (.getBytes "hi"))+ (.closeEntry zos)))++(defn- read-hello [^FileSystem fs]+ (Files/readString (.getPath fs "/hello.txt" (u/varargs String))))++(deftest nio-fs-bypasses-uri-cache-test+ (testing "nio-fs returns independent filesystems for the same zip — closing one cannot kill another"+ (mt/with-temp-file [zip-path]+ (write-test-zip! zip-path)+ (with-open [fs1 (u.files/nio-fs zip-path)+ fs2 (u.files/nio-fs zip-path)]+ ;; Both filesystems read the entry — distinguishes Path-based (works) from URI-based (would throw+ ;; FileSystemAlreadyExistsException on the second open since the cache entry would still be live).+ (is (= "hi" (read-hello fs1)))+ (is (= "hi" (read-hello fs2)))+ ;; Closing fs1 must not affect fs2.+ (.close fs1)+ (is (thrown? ClosedFileSystemException (read-hello fs1)))+ (is (= "hi" (read-hello fs2))))+ ;; Subsequent opens still work after previous instance closes — not in a permanently-broken state.+ (with-open [fs (u.files/nio-fs zip-path)]+ (is (= "hi" (read-hello fs)))))))