Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

mb-sql-parsing-mode=python (#77558)

Alexander Polyankin97f82c0792cb
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 78f34ceaa2a..dc3ff3dc840 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2183,9 +2183,8 @@
sql-parsing
{:team "DevEx"
- :api #{metabase.sql-parsing.core}
- :uses #{analytics-interface
- util}}
+ :api #{metabase.sql-parsing.core metabase.sql-parsing.init}
+ :uses #{analytics-interface settings util}}
sql-tools
{:team "DevEx"
diff --git a/bin/docker/Dockerfile b/bin/docker/Dockerfile
index f6dd3296219..a741764aeb1 100644
--- a/bin/docker/Dockerfile
+++ b/bin/docker/Dockerfile
@@ -9,7 +9,7 @@ ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA}
COPY DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem
# dependencies
-RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts nodejs && \
+RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts nodejs python3 && \
apk upgrade && \
rm -rf /var/cache/apk/* && \
mkdir -p /app/certs && \
diff --git a/bin/docker/Dockerfile_ubuntu b/bin/docker/Dockerfile_ubuntu
index 3fa3cae342b..56c1513c4ed 100644
--- a/bin/docker/Dockerfile_ubuntu
+++ b/bin/docker/Dockerfile_ubuntu
@@ -11,7 +11,7 @@ COPY DigiCertGlobalRootG2.crt.pem /usr/local/share/ca-certificates/DigiCertGloba
# Dependencies
RUN apt-get update && \
apt-get upgrade -y && \
- apt-get install -y ca-certificates ca-certificates-java fonts-noto nodejs && \
+ apt-get install -y ca-certificates ca-certificates-java fonts-noto nodejs python3 && \
apt-get clean && \
curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /usr/local/share/ca-certificates/rds-combined-ca-bundle.pem && \
update-ca-certificates && \
diff --git a/bin/docker/run_metabase.sh b/bin/docker/run_metabase.sh
index 0a03b56c6bf..a7b4bfa8e3d 100755
--- a/bin/docker/run_metabase.sh
+++ b/bin/docker/run_metabase.sh
@@ -4,6 +4,16 @@ if [ -z "$MB_JETTY_HOST" ]; then
export MB_JETTY_HOST=0.0.0.0
fi
+# the image ships python3 and node, so default to the external-process backends (lighter and faster
+# than the in-process GraalVM ones) unless explicitly configured otherwise
+if [ -z "$MB_SQL_PARSING_MODE" ]; then
+ export MB_SQL_PARSING_MODE=python
+fi
+
+if [ -z "$MB_STATIC_VIZ_MODE" ]; then
+ export MB_STATIC_VIZ_MODE=node
+fi
+
# Setup Java Options
JAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions"
JAVA_OPTS="${JAVA_OPTS} -Dfile.encoding=UTF-8"
diff --git a/resources/python-sources/sql_tools_cli.py b/resources/python-sources/sql_tools_cli.py
new file mode 100644
index 00000000000..406894a7f25
--- /dev/null
+++ b/resources/python-sources/sql_tools_cli.py
@@ -0,0 +1,43 @@
+"""Newline-delimited JSON stdin/stdout server around sql_tools, for running sqlglot on a native
+CPython instead of GraalPy. Spawned by metabase.sql-parsing.python with this directory (or a zip of
+it) on PYTHONPATH; prints one ready line, then answers one response line per request line:
+
+ {"fn": "referenced_tables", "args": ["SELECT 1", "postgres"]}
+ -> {"ok": true, "result": "..."} | {"ok": false, "error": "..."}
+"""
+
+import json
+import sys
+
+import sql_tools
+
+
+def _resolve(fn_name):
+ if fn_name.startswith("_"):
+ raise ValueError(f"unknown sql_tools function: {fn_name}")
+ fn = getattr(sql_tools, fn_name, None)
+ if not callable(fn):
+ raise ValueError(f"unknown sql_tools function: {fn_name}")
+ return fn
+
+
+def main():
+ out = sys.stdout
+ out.write(json.dumps({"ready": True}) + "\n")
+ out.flush()
+ for line in sys.stdin:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ request = json.loads(line)
+ result = _resolve(request["fn"])(*request["args"])
+ response = {"ok": True, "result": result}
+ except Exception as e: # noqa: BLE001 -- any failure is reported to the JVM side
+ response = {"ok": False, "error": f"{type(e).__name__}: {e}"}
+ out.write(json.dumps(response) + "\n")
+ out.flush()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/metabase/channel/render/js/common.clj b/src/metabase/channel/render/js/common.clj
index 96ecf96f5b4..810ccfaeae9 100644
--- a/src/metabase/channel/render/js/common.clj
+++ b/src/metabase/channel/render/js/common.clj
@@ -28,14 +28,14 @@
"Build a dirigiste `Pool` of static-viz workers, each held exclusively per render (so at most `:max-size`
renders run at once). `:max-size` is the maximum number of concurrent workers. The utilization
controller targets 100% utilization with a min of 0, so when nothing is rendering the pool shrinks to 0
- and `destroy` is called on each idle worker; it rechecks every 1 minute, so a worker lingers up to 1
- minute before being reaped (keeping it warm through gaps between renders). `(generate)` mints a worker;
- `(destroy worker)` tears one down. The other constructor args (queue size, sampling interval) don't
- matter much."
- ^Pool [generate destroy {:keys [max-size]}]
+ and `destroy` is called on each idle worker; it rechecks every `:idle-minutes`, so a worker
+ lingers up to that long before being reaped (keeping it warm through gaps between renders). `(generate)`
+ mints a worker; `(destroy worker)` tears one down. The other constructor args (queue size, sampling
+ interval) don't matter much."
+ ^Pool [generate destroy {:keys [max-size idle-minutes]}]
(let [max-queued-acquires 65000
sample-period-ms (.toMillis TimeUnit/MILLISECONDS 25)
- control-period-ms (.toMillis TimeUnit/MINUTES 1)]
+ control-period-ms (.toMillis TimeUnit/MINUTES idle-minutes)]
(Pool. (reify IPool$Generator
(generate [_ _] (generate))
(destroy [_ _ worker] (destroy worker)))
diff --git a/src/metabase/channel/render/js/graal.clj b/src/metabase/channel/render/js/graal.clj
index d3efb9abea6..fcd0b324d72 100644
--- a/src/metabase/channel/render/js/graal.clj
+++ b/src/metabase/channel/render/js/graal.clj
@@ -1,6 +1,6 @@
(ns metabase.channel.render.js.graal
"The GraalVM [[metabase.channel.render.js.protocol/StaticVizRenderer]]: runs the static-viz JS
- in-process on a pool of sandboxed GraalVM contexts (up to two by default).
+ in-process on a pool of sandboxed GraalVM contexts (up to three by default).
We run the JS interpreted (no Graal compiler on a stock JDK) and silence the interpreter warning with
the engine-level `engine.WarnInterpreterOnly` option. See
@@ -150,11 +150,12 @@
:static-viz)
(def ^:private ^Pool static-viz-context-pool
- "A pool of up to two static-viz contexts, each held exclusively from acquire to release, so at most two
- renders run at once — one per context, on the shared engine. When idle the pool shrinks to 0 and the
- generator's `destroy` closes the context (and, on the last one, the shared engine); the first render
- after an idle gap rebuilds them. See [[metabase.channel.render.js.common/create-pool]]."
- (common/create-pool generate-context! destroy-context! {:max-size 2}))
+ "A pool of up to three static-viz contexts, each held exclusively from acquire to release, so at most
+ three renders run at once — one per context, on the shared engine. When idle for up to 10 minutes the
+ pool shrinks to 0 and the generator's `destroy` closes the context (and, on the last one, the shared
+ engine); the first render after an idle gap rebuilds them. See
+ [[metabase.channel.render.js.common/create-pool]]."
+ (common/create-pool generate-context! destroy-context! {:max-size 3, :idle-minutes 10}))
(defn- do-with-static-viz-context
"Borrow a pooled static-viz context and call `f` with it, held exclusively for the call (never let it —
diff --git a/src/metabase/channel/render/js/node.clj b/src/metabase/channel/render/js/node.clj
index 29ed97422d9..021855fecdd 100644
--- a/src/metabase/channel/render/js/node.clj
+++ b/src/metabase/channel/render/js/node.clj
@@ -143,7 +143,7 @@
it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because a
warm process is fast and spawning another is costly. See
[[metabase.channel.render.js.common/create-pool]]."
- (common/create-pool start-process! stop-process! {:max-size 1}))
+ (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 1}))
(defn- call-node
"Run static-viz bundle function `fn-name` with `arg` (a Clojure data structure) on a pooled Node process,
diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.clj
index 5d7a3752330..c50465fd643 100644
--- a/src/metabase/core/init.clj
+++ b/src/metabase/core/init.clj
@@ -64,6 +64,7 @@
[metabase.settings.init]
[metabase.setup.init]
[metabase.slackbot.init]
+ [metabase.sql-parsing.init]
[metabase.sql-tools.init]
[metabase.sso.init]
[metabase.store-api.init]
diff --git a/src/metabase/sql_parsing/common.clj b/src/metabase/sql_parsing/common.clj
index 428f92cc419..faf2cd0383f 100644
--- a/src/metabase/sql_parsing/common.clj
+++ b/src/metabase/sql_parsing/common.clj
@@ -1,8 +1,180 @@
(ns metabase.sql-parsing.common
+ "Shared helpers for the pooled sqlglot parsers ([[metabase.sql-parsing.graal]] and
+ [[metabase.sql-parsing.python]]): the python-sources locations, the lazy dev sqlglot install, the
+ protocol reify over a transport function, and the dirigiste worker-pool factory (copied from
+ [[metabase.channel.render.js.common]])."
(:require
- [potemkin :as p]))
+ [clojure.java.io :as io]
+ [clojure.java.shell :as shell]
+ [metabase.sql-parsing.protocol :as protocol]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log])
+ (:import
+ (io.aleph.dirigiste IPool$Generator Pool Pools)
+ (java.io File)
+ (java.util.concurrent TimeUnit)))
-(p/defprotocol+ PythonEval
- "Protocol for evaluating Python code. Abstracts over raw Context and pooled contexts."
- (eval-python [this code]
- "Evaluate Python code."))
+(set! *warn-on-reflection* true)
+
+;;; ---------------------------------------- Python path resolution ---------------------------------------
+
+(def python-sources-resource
+ "Resource path for Python sources (sql_tools.py and sqlglot)."
+ "python-sources")
+
+(defn jar-resource?
+ "True if the given resource is inside a JAR (production), false if from filesystem (dev)."
+ [resource]
+ (boolean
+ (when-let [url (io/resource resource)]
+ (.contains (.getFile ^java.net.URL url) ".jar!/"))))
+
+;;; ---------------------------------------- Dev lazy installation ----------------------------------------
+
+(def dev-python-sources-dir
+ "Directory where sqlglot is installed in dev mode."
+ "resources/python-sources")
+
+(defn- delete-recursive!
+ "Recursively delete a directory and all its contents."
+ [^File f]
+ (when (.isDirectory f)
+ (doseq [child (.listFiles f)]
+ (delete-recursive! child)))
+ (.delete f))
+
+(defn expected-sqlglot-version
+ "Read the expected sqlglot version from pyproject.toml resource file."
+ []
+ (some->> (io/resource "python-sources/pyproject.toml")
+ slurp
+ (re-find #"\"sqlglot==([^\"]+)\"")
+ second))
+
+(defn- package-installer-available?
+ "Check if uv or pip is available for installing Python packages."
+ []
+ (or (zero? (:exit (shell/sh "which" "uv")))
+ (zero? (:exit (shell/sh "which" "pip")))))
+
+(defn- version-installed?
+ "Check if sqlglot is installed with the expected version. Verifies both the dist-info metadata
+ (needed by uv/pip) and the actual module (needed by Python import)."
+ [target-dir expected-version]
+ (and (.exists (io/file target-dir (str "sqlglot-" expected-version ".dist-info") "METADATA"))
+ (.exists (io/file target-dir "sqlglot" "__init__.py"))))
+
+(defn- install-sqlglot!
+ "Install sqlglot via uv (preferred) or pip (fallback).
+ Deletes any existing sqlglot directories first to handle version upgrades."
+ [target-dir version]
+ ;; Delete old versions first (sqlglot/ and any sqlglot-*.dist-info/)
+ (doseq [^File f (.listFiles (io/file target-dir))]
+ (when (or (= "sqlglot" (.getName f))
+ (.startsWith (.getName f) "sqlglot-"))
+ (log/info "Removing old sqlglot:" (.getName f))
+ (delete-recursive! f)))
+ ;; Try uv first (fast), fall back to pip
+ (let [pyproject-file (str target-dir "/pyproject.toml")
+ uv-result (shell/sh "uv" "pip" "install" "-r" pyproject-file "--target" target-dir "--no-compile" "--reinstall")]
+ (if (zero? (:exit uv-result))
+ (log/info "sqlglot" version "installed via uv")
+ (do
+ (log/info "uv not available, trying pip...")
+ (let [pkg (str "sqlglot==" version)
+ pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile" "--force-reinstall")]
+ (when-not (zero? (:exit pip-result))
+ (throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"
+ "Manual install: uv pip install -r " pyproject-file " --target " target-dir "\n"
+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")
+ {:uv-error (:err uv-result)
+ :pip-error (:err pip-result)
+ :version version
+ :target-dir target-dir})))
+ (log/info "sqlglot" version "installed via pip"))))))
+
+(defn ensure-sqlglot-installed!
+ "Ensure sqlglot is installed with correct version. Called lazily on first use in dev.
+ If version mismatch or missing, automatically installs the correct version."
+ []
+ (let [expected-ver (expected-sqlglot-version)]
+ (when-not expected-ver
+ (throw (ex-info "Missing pyproject.toml or unable to parse sqlglot entry in resources/python-sources/"
+ {:resource "python-sources/pyproject.toml"})))
+ (when-not (version-installed? dev-python-sources-dir expected-ver)
+ (if (package-installer-available?)
+ (do
+ (log/info "Installing sqlglot" expected-ver "(first use in dev)...")
+ (install-sqlglot! dev-python-sources-dir expected-ver))
+ (log/warn "sqlglot not installed and no package installer (uv or pip) found."
+ "SQL parsing features will fail until sqlglot is installed."
+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")))))
+
+;;; ------------------------------------------------- parser ----------------------------------------------
+
+(defn call-failed-ex
+ "The uniform exception for a Python-side sqlglot failure, thrown by every transport so callers can
+ handle parse errors without knowing which backend ran ([[metabase.sql-parsing.core/parse-error?]]).
+ `error` is the Python error rendered as `ErrorType: message`."
+ ([error]
+ (call-failed-ex error nil))
+ ([error cause]
+ (ex-info (str "sqlglot call failed: " error)
+ {:sql-parsing/error true
+ :sql-parsing/python-error-type (some->> error (re-find #"^\w+"))}
+ cause)))
+
+(defn make-parser
+ "Build a [[metabase.sql-parsing.protocol/SqlParser]] over `call`, a transport function
+ `(call fn-name & args) → string` that executes `sql_tools.<fn-name>` with `args`. JSON results are
+ decoded into Clojure data; schema and replacement arguments are passed to Python as JSON strings (the
+ `sql_tools` functions take them that way — GraalVM polyglot map conversion is unreliable)."
+ [call]
+ (reify protocol/SqlParser
+ (referenced-tables [_ dialect sql]
+ (vec (json/decode (call "referenced_tables" sql dialect))))
+ (referenced-fields [_ dialect sql]
+ (vec (json/decode (call "referenced_fields" sql dialect))))
+ (returned-columns-lineage [_ dialect sql default-table-schema sqlglot-schema]
+ (vec (json/decode (call "returned_columns_lineage"
+ dialect sql default-table-schema (json/encode sqlglot-schema)))))
+ (validate-query [_ dialect sql default-table-schema sqlglot-schema]
+ ;; nil → "{}" (an empty JSON object), which sql_tools reads as permissive mode; encoding the
+ ;; *string* "{}" here would double-encode and make Python treat a string as the schema
+ (json/decode+kw (call "validate_query"
+ dialect sql default-table-schema (json/encode (or sqlglot-schema {})))))
+ (simple-query [_ dialect sql]
+ (json/decode+kw (call "simple_query" sql dialect)))
+ (add-into-clause [_ dialect sql table-name]
+ (call "add_into_clause" sql table-name dialect))
+ (field-references [_ dialect sql]
+ (json/decode+kw (call "field_references" sql dialect)))
+ (replace-names [_ dialect sql replacements]
+ (call "replace_names" sql (json/encode replacements) dialect))
+ (single-stmt-of-type [_ dialect sql stmt-type]
+ (json/decode+kw (call "is_single_stmt_of_type" sql stmt-type dialect)))
+ (transpile-sql [_ sql from-dialect to-dialect]
+ (json/decode+kw (call "transpile_sql" sql from-dialect to-dialect)))))
+
+;;; -------------------------------------------------- pool -----------------------------------------------
+
+(defn create-pool
+ "Build a dirigiste `Pool` of workers, each held exclusively per call (so at most `:max-size`
+ calls run at once). `:max-size` is the maximum number of concurrent workers. The utilization
+ controller targets 100% utilization with a min of 0, so when nothing is running the pool shrinks to 0
+ and `destroy` is called on each idle worker; it rechecks every `:idle-minutes`, so a worker
+ lingers up to that long before being reaped (keeping it warm through gaps between calls). `(generate)`
+ mints a worker; `(destroy worker)` tears one down. The other constructor args (queue size, sampling
+ interval) don't matter much."
+ ^Pool [generate destroy {:keys [max-size idle-minutes]}]
+ (let [max-queued-acquires 65000
+ sample-period-ms (.toMillis TimeUnit/MILLISECONDS 25)
+ control-period-ms (.toMillis TimeUnit/MINUTES idle-minutes)]
+ (Pool. (reify IPool$Generator
+ (generate [_ _] (generate))
+ (destroy [_ _ worker] (destroy worker)))
+ (Pools/utilizationController 1.0 max-size max-size)
+ max-queued-acquires
+ sample-period-ms
+ control-period-ms
+ TimeUnit/MILLISECONDS)))
diff --git a/src/metabase/sql_parsing/core.clj b/src/metabase/sql_parsing/core.clj
index 3fc48658d0e..7a1991848fd 100644
--- a/src/metabase/sql_parsing/core.clj
+++ b/src/metabase/sql_parsing/core.clj
@@ -8,63 +8,36 @@
(referenced-tables sql dialect) → [[catalog schema table] ...]
(referenced-fields dialect sql) → [[catalog schema table field] ...]
(returned-columns-lineage dialect sql schema schema-map) → [[col pure? deps] ...]
- (validate-query dialect sql schema schema-map) → {:status :ok} | {:status :error ...}"
+ (validate-query dialect sql schema schema-map) → {:status :ok} | {:status :error ...}
+
+ The parsing itself happens behind the [[metabase.sql-parsing.protocol/SqlParser]] protocol (the
+ GraalPy or native-CPython implementation, per [[metabase.sql-parsing.parser/parser]]); this namespace
+ owns the JVM-side pre- and post-processing around it."
(:require
[clojure.string :as str]
[medley.core :as m]
- [metabase.analytics-interface.core :as analytics]
- [metabase.sql-parsing.common :as common]
- [metabase.sql-parsing.pool :as python.pool]
+ [metabase.sql-parsing.parser :as parser]
+ [metabase.sql-parsing.protocol :as protocol]
[metabase.util :as u]
- [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.performance :as perf])
(:import
- (java.io Closeable)
- (java.util.concurrent ExecutionException TimeoutException)
- (org.graalvm.polyglot Value)))
+ (java.util.concurrent TimeoutException)))
(set! *warn-on-reflection* true)
-;;; -------------------------------------------------- Timeout handling --------------------------------------------------
-
-(def ^:private ^:const default-timeout-ms
- "Default timeout for Python operations in milliseconds.
- GraalVM Python can occasionally hang (DEV-1393), so we wrap calls with a timeout."
- 30000) ; 30 seconds
-
-(defn- with-timeout*
- "Execute f in a future with timeout. On timeout, throws TimeoutException.
- The caller is responsible for cleaning up resources (e.g., disposing context)."
- [timeout-ms f]
- (let [fut (future (f))]
- (try
- (deref fut timeout-ms ::timeout)
- (catch ExecutionException e
- ;; Unwrap execution exception to get the real cause
- (throw (or (.getCause e) e)))
- (finally
- ;; If we timed out or got an exception, try to cancel the future
- ;; Note: This won't actually interrupt GraalVM, but prevents resource leaks
- (future-cancel fut)))))
-
-(defmacro ^:private with-python-timeout
- "Execute body with a timeout. If timeout is reached:
- 1. Interrupts the GraalVM context (actually stops execution)
- 2. Poisons context so it gets disposed rather than returned to pool
- 3. Logs warning and throws TimeoutException"
- [ctx timeout-ms & body]
- `(let [result# (with-timeout* ~timeout-ms (^:once fn* [] ~@body))]
- (if (= result# ::timeout)
- (do
- ;; Actually interrupt the GraalVM context (1s grace period for soft interrupt)
- ;; This is necessary because future-cancel doesn't stop GraalVM execution
- (python.pool/interrupt! ~ctx 1000)
- (python.pool/poison! ~ctx)
- (analytics/inc! :metabase-sql-parsing/context-timeouts)
- (log/warn "Python execution timed out after" ~timeout-ms "ms - GraalVM interrupted")
- (throw (TimeoutException. (str "Python execution timed out after " ~timeout-ms "ms"))))
- result#)))
+(defn- parser
+ "The [[metabase.sql-parsing.protocol/SqlParser]] implementation for the configured mode."
+ []
+ (parser/parser))
+
+(defn parse-error?
+ "True if `e` is a sqlglot ParseError — the SQL could not be parsed — regardless of which parser
+ backend threw it. Other Python-side failures (and non-sqlglot exceptions) are false."
+ [e]
+ (let [data (ex-data e)]
+ (boolean (and (:sql-parsing/error data)
+ (= "ParseError" (:sql-parsing/python-error-type data))))))
;;; ----------------------------------------- VALUES clause stripping ------------------------------------------
@@ -214,14 +187,7 @@
This is the pure parsing layer - it returns what's literally in the SQL.
Default schema resolution happens in the matching layer (core.clj)."
[dialect sql]
- (let [sql (strip-large-values sql)]
- (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.referenced_tables")
- (.execute ^Value (object-array [sql dialect]))
- .asString)))
- json/decode
- vec)))
+ (protocol/referenced-tables (parser) dialect (strip-large-values sql)))
(defn referenced-fields
"Extract field references from SQL, returning only fields from actual database tables.
@@ -247,14 +213,7 @@
(referenced-fields \"bigquery\" \"SELECT * FROM myproject.analytics.events\")
=> [[\"myproject\" \"analytics\" \"events\" \"*\"]]"
[dialect sql]
- (let [sql (strip-large-values sql)]
- (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.referenced_fields")
- (.execute ^Value (object-array [sql dialect]))
- .asString)))
- json/decode
- vec)))
+ (protocol/referenced-fields (parser) dialect (strip-large-values sql)))
(defn returned-columns-lineage
"Extract column lineage from SQL query, showing which output columns depend on which source columns.
@@ -274,18 +233,7 @@
(returned-columns-lineage \"postgres\" \"SELECT id + 1 as computed FROM users\" nil schema)
=> [[\"computed\" false [[[nil \"users\" \"id\"]]]]]"
[dialect sql default-table-schema sqlglot-schema]
- (let [sql (strip-large-values sql)]
- (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues
- (-> ^Value (common/eval-python ctx "sql_tools.returned_columns_lineage")
- (.execute ^Value (object-array [dialect
- sql
- default-table-schema
- (json/encode sqlglot-schema)]))
- .asString)))
- json/decode
- vec)))
+ (protocol/returned-columns-lineage (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))
(defn validate-query
"Validate a SQL query against a schema using sqlglot's qualify optimizer.
@@ -317,14 +265,7 @@
- \"invalid_expression\": Syntax/parse error
- \"unhandled\": Other errors"
[dialect sql default-table-schema & [sqlglot-schema]]
- (let [sql (strip-large-values sql)]
- (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues
- (-> ^Value (common/eval-python ctx "sql_tools.validate_query")
- (.execute ^Value (object-array [dialect sql default-table-schema (json/encode (or sqlglot-schema "{}"))]))
- .asString)))
- json/decode+kw)))
+ (protocol/validate-query (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))
(defn simple-query?
"Check if SQL is a simple SELECT without LIMIT, OFFSET, or CTEs.
@@ -346,13 +287,7 @@
(simple-query? nil \"SELECT * FROM users LIMIT 10\")
=> {:is_simple false :reason \"Contains a LIMIT\"}"
[dialect sql]
- (let [sql (strip-large-values sql)]
- (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.simple_query")
- (.execute ^Value (object-array [sql dialect]))
- .asString)))
- json/decode+kw)))
+ (protocol/simple-query (parser) dialect (strip-large-values sql)))
(defn add-into-clause
"Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.
@@ -370,11 +305,7 @@
Returns: Modified SQL string with INTO clause"
[dialect sql table-name]
- (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.add_into_clause")
- (.execute ^Value (object-array [sql table-name dialect]))
- .asString))))
+ (protocol/add-into-clause (parser) dialect sql table-name))
(defn- convert-field-type
"Convert field type string (snake_case) to keyword (kebab-case)."
@@ -487,24 +418,18 @@
On timeout, returns an error map instead of throwing, consistent with the
'fail soft' pattern used for parsing failures."
[dialect sql]
- (let [sql (strip-large-values sql)]
- (try
- (let [raw (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.field_references")
- (.execute ^Value (object-array [sql dialect]))
- .asString)))
- json/decode+kw)
- used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])
- returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])
- errors (or (:errors raw) (get raw "errors") [])]
- {:used-fields (set (map convert-field used-fields))
- :returned-fields (vec (map convert-field returned-fields))
- :errors (set (map convert-error errors))})
- (catch TimeoutException e
- {:used-fields #{}
- :returned-fields []
- :errors #{{:type :timeout :message (.getMessage e)}}}))))
+ (try
+ (let [raw (protocol/field-references (parser) dialect (strip-large-values sql))
+ used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])
+ returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])
+ errors (or (:errors raw) (get raw "errors") [])]
+ {:used-fields (set (map convert-field used-fields))
+ :returned-fields (vec (map convert-field returned-fields))
+ :errors (set (map convert-error errors))})
+ (catch TimeoutException e
+ {:used-fields #{}
+ :returned-fields []
+ :errors #{{:type :timeout :message (.getMessage e)}}})))
(defn replace-names
"Replace schema, table, and column names in SQL.
@@ -527,23 +452,14 @@
(replace-names \"postgres\" \"SELECT * FROM people\" {:tables [[{:table \"people\"} \"users\"]]})
=> \"SELECT * FROM users\""
[dialect sql replacements]
- (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.replace_names")
- (.execute ^Value (object-array [sql (json/encode replacements) dialect]))
- .asString))))
+ (protocol/replace-names (parser) dialect sql replacements))
(defn is-single-stmt-of-type?
"Validates that a query is a single read statement (SELECT) or a single write statement (INSERT, UPDATE, DELETE)
and returns the query reconstructed from the parsed AST."
[dialect sql stmt-type]
(let [stripped-sql (strip-large-values sql)
- result (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.is_single_stmt_of_type")
- (.execute ^Value (object-array [stripped-sql stmt-type dialect]))
- .asString)))
- json/decode+kw
+ result (-> (protocol/single-stmt-of-type (parser) dialect stripped-sql stmt-type)
(perf/update-keys (comp keyword u/->kebab-case-en)))]
;; The `:sql` in the `result` is the reconstructed SQL from the SQLGlot parser.
;; We generally want to use the reconstructed SQL, but if the original SQL had its values stripped (to avoid GraalPy OOM)
@@ -554,16 +470,15 @@
(comment
(referenced-tables "postgres" "select * from transactions")
- (validate-sql-query "postgres" "SELECT * FROM users")
+ (validate-query "postgres" "SELECT * FROM users" nil)
(referenced-fields "postgres" "SELECT id, name FROM users WHERE active = true"))
;;;; Transpile sql
(defn- normalize-transpilation-result
- [json-str]
- (-> json-str
- json/decode+kw
+ [result]
+ (-> result
(perf/update-keys (comp keyword u/->kebab-case-en))
(m/update-existing :status (comp keyword u/->kebab-case-en))
(m/update-existing :reason (comp keyword u/->kebab-case-en))))
@@ -571,9 +486,5 @@
(defn transpile-sql
"Transpiles sql string from one dialect to another."
[sql from-dialect to-dialect]
- (-> (with-open [^Closeable ctx (python.pool/python-context)]
- (with-python-timeout ctx default-timeout-ms
- (-> ^Value (common/eval-python ctx "sql_tools.transpile_sql")
- (.execute ^Value (object-array [sql from-dialect to-dialect]))
- .asString)))
- normalize-transpilation-result))
+ (normalize-transpilation-result
+ (protocol/transpile-sql (parser) sql from-dialect to-dialect)))
diff --git a/src/metabase/sql_parsing/graal.clj b/src/metabase/sql_parsing/graal.clj
new file mode 100644
index 00000000000..189fdb2d1a2
--- /dev/null
+++ b/src/metabase/sql_parsing/graal.clj
@@ -0,0 +1,269 @@
+(ns metabase.sql-parsing.graal
+ "The `:graalvm` [[metabase.sql-parsing.protocol/SqlParser]]: runs sqlglot on GraalPy in-process on a
+ pool of sandboxed GraalVM contexts (up to three by default).
+
+ The pooled contexts share one `Engine` and one parsed bootstrap `Source`: the engine (and its code
+ cache, which holds the parsed sqlglot modules) is created with the first context and closed with the
+ last, and each context evaluates the shared source into its own realm. A context is held exclusively
+ per call (so calls serialize per context); the utilization controller has min 0, so when idle the pool
+ shrinks to 0 and the last `destroy` closes the engine (GraalVM reclaims neither context nor engine on
+ GC, and GraalPy contexts hold significant native memory). The first call after an idle gap rebuilds
+ them.
+
+ GraalPy can occasionally hang (DEV-1393), so every call runs under a timeout; a timed-out context is
+ interrupted and disposed from the pool instead of released."
+ (:require
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.sql-parsing.common :as common]
+ [metabase.util.files :as u.files]
+ [metabase.util.log :as log])
+ (:import
+ (io.aleph.dirigiste Pool)
+ (java.net URI)
+ (java.nio.channels SeekableByteChannel)
+ (java.nio.file AccessMode DirectoryStream DirectoryStream$Filter Files FileSystems Path)
+ (java.time Duration)
+ (java.util Map Set)
+ (java.util.concurrent ExecutionException TimeoutException)
+ (org.graalvm.polyglot Context Engine HostAccess PolyglotException Source Value)
+ (org.graalvm.polyglot.io FileSystem IOAccess)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------ Python filesystem ------------------------------------------
+
+(defn- nio-polyglot-fs
+ "Wrap a NIO `java.nio.file.FileSystem` as a polyglot `FileSystem`, routing reads to
+ `Files/newByteChannel` instead of the provider's `newFileChannel`.
+
+ The stock wrapper triggers ZipFileSystem to extract deflated entries to a temp file beside the jar,
+ which fails when the jar's parent directory isn't writable (e.g. non-root in a Kubernetes pod).
+ See https://github.com/metabase/metabase/issues/73541."
+ ^FileSystem [^java.nio.file.FileSystem nio-fs]
+ (let [provider (.provider nio-fs)]
+ (reify FileSystem
+ (^Path parsePath [_ ^URI uri]
+ (.getPath provider uri))
+ (^Path parsePath [_ ^String s]
+ (.getPath nio-fs s (into-array String [])))
+ (^void checkAccess [_ ^Path p ^Set modes ^"[Ljava.nio.file.LinkOption;" _opts]
+ ;; LinkOption opts are dropped: the only value is NOFOLLOW_LINKS, and neither our jar zip FS
+ ;; nor resources/python-sources contains symlinks.
+ (.checkAccess provider p ^"[Ljava.nio.file.AccessMode;" (into-array AccessMode modes)))
+ (^void createDirectory [_ ^Path p ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]
+ (Files/createDirectory p attrs))
+ (^void delete [_ ^Path p]
+ (Files/delete p))
+ (^SeekableByteChannel newByteChannel
+ [_ ^Path p ^Set opts ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]
+ (Files/newByteChannel p opts attrs))
+ (^DirectoryStream newDirectoryStream [_ ^Path dir ^DirectoryStream$Filter filter]
+ (Files/newDirectoryStream dir filter))
+ (^Path toAbsolutePath [_ ^Path p]
+ (.toAbsolutePath p))
+ (^Path toRealPath [_ ^Path p ^"[Ljava.nio.file.LinkOption;" opts]
+ (.toRealPath p opts))
+ (^Map readAttributes [_ ^Path p ^String attrs ^"[Ljava.nio.file.LinkOption;" opts]
+ (Files/readAttributes p attrs opts)))))
+
+(defn- read-only-polyglot-fs
+ "Wrap an NIO `java.nio.file.FileSystem` as a read-only polyglot FileSystem suitable for GraalPy."
+ ^FileSystem [^java.nio.file.FileSystem nio-fs]
+ (-> nio-fs nio-polyglot-fs FileSystem/newReadOnlyFileSystem))
+
+(defonce ^:private
+ ^{:doc "A read-only polyglot FileSystem and the PythonPath within it.
+ In dev: wraps the default filesystem, path is resources/python-sources.
+ In jar: wraps the jar's zip filesystem, path is /python-sources.
+ Either way, the filesystem is read-only so extracted sources cannot be tampered with."}
+ python-fs-and-path
+ (delay
+ (if (common/jar-resource? common/python-sources-resource)
+ ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib are both inside the
+ ;; jar, so nothing is extracted to disk and there's nothing to tamper with. The filesystem lives for the
+ ;; duration of the process (via defonce + delay) and is shared by all pooled Python contexts.
+ {:fs (-> (u.files/get-jar-path) u.files/nio-fs read-only-polyglot-fs)
+ :python-path "/python-sources"
+ :std-lib-home "/META-INF/resources/libpython"
+ :core-home "/META-INF/resources/libgraalpy"}
+ ;; In dev: use the real filesystem (read-only wrapper). sqlglot is installed locally.
+ (do
+ (common/ensure-sqlglot-installed!)
+ {:fs (read-only-polyglot-fs (FileSystems/getDefault))
+ :python-path common/dev-python-sources-dir}))))
+
+;;; ---------------------------------------- shared engine + contexts -------------------------------------
+
+(def ^:private engine-lock (Object.))
+
+(def ^:private shared-engine
+ "Atom holding `{:engine <Engine>, :source <Source>, :refs <live context count>}`, or nil when no context
+ is live. Guarded by [[engine-lock]]. The engine and parsed bootstrap source are shared by every pooled
+ context; they're created with the first context ([[acquire-engine!]]) and closed with the last
+ ([[release-engine!]])."
+ (atom nil))
+
+(defn- create-engine
+ "Build a Python `Engine`. We run interpreted (no Graal compiler on a stock JDK), so
+ `engine.WarnInterpreterOnly` is silenced here (it's an engine-level option)."
+ ^Engine []
+ (.. (Engine/newBuilder)
+ (option "engine.WarnInterpreterOnly" "false")
+ (build)))
+
+(defn- bootstrap-source
+ "Build the bootstrap `Source` evaluated into every context: importing sql_tools loads it (and,
+ transitively, sqlglot) into the shared engine's code cache."
+ ^Source []
+ (.buildLiteral (Source/newBuilder "python" "import sql_tools" "sql-tools-bootstrap")))
+
+(defn- acquire-engine!
+ "Return the shared `{:engine, :source}`, creating them with the first context. Bumps the ref count."
+ []
+ (locking engine-lock
+ (let [state (or @shared-engine
+ {:engine (create-engine)
+ :source (bootstrap-source)
+ :refs 0})]
+ (reset! shared-engine (update state :refs inc))
+ state)))
+
+(defn- release-engine!
+ "Drop a ref on the shared engine, closing it once the last context is gone."
+ []
+ (locking engine-lock
+ (let [{:keys [^Engine engine refs]} @shared-engine]
+ (if (<= refs 1)
+ (do (try (.close engine) (catch Exception _))
+ (reset! shared-engine nil))
+ (swap! shared-engine update :refs dec)))))
+
+(defn- create-context
+ "Create a sandboxed GraalPy context on `engine` configured for sqlglot: PythonPath pointing at the
+ Python sources, host access for string interop, and a read-only filesystem (sources are read directly
+ from the classpath/jar; nothing is extracted to disk in prod)."
+ ^Context [^Engine engine]
+ (let [{:keys [^FileSystem fs python-path std-lib-home core-home]} @python-fs-and-path
+ io-access (.. (IOAccess/newBuilder)
+ (fileSystem fs)
+ (build))
+ builder (.. (Context/newBuilder (into-array String ["python"]))
+ (engine engine)
+ (option "python.PythonPath" python-path)
+ (allowHostAccess HostAccess/ALL)
+ (allowIO io-access))
+ ;; When running from the jar, GraalPy's stdlib is also inside the jar filesystem
+ ;; and we need to tell it where to find the core and stdlib paths explicitly.
+ builder (cond-> builder
+ std-lib-home (.option "python.StdLibHome" std-lib-home)
+ core-home (.option "python.CoreHome" core-home))]
+ (.build builder)))
+
+(defn- generate-context!
+ "Build a context on the shared engine and evaluate the bootstrap source into it (creating the engine +
+ source if this is the first context)."
+ ^Context []
+ (analytics/inc! :metabase-sql-parsing/context-creations)
+ (let [{:keys [^Engine engine ^Source source]} (acquire-engine!)]
+ (try
+ (doto (create-context engine)
+ (.eval source))
+ (catch Throwable t
+ (release-engine!)
+ (throw t)))))
+
+(defn- destroy-context!
+ "Close a context and drop its ref on the shared engine (closing the engine if it was the last context)."
+ [^Context context]
+ (try (.close context true) (catch Exception _))
+ (release-engine!))
+
+;;; ------------------------------------------------ context pool -----------------------------------------
+
+(def ^:private pool-key
+ "Dirigiste pools are keyed; the key itself is arbitrary, it just has to be the same for every operation."
+ :sql-parsing)
+
+(def ^:private ^Pool python-context-pool
+ "A pool of up to three GraalPy contexts, each held exclusively from acquire to release, so at most
+ three sqlglot calls run at once — one per context, on the shared engine. When idle for up to 10
+ minutes the pool shrinks to 0 and the generator's `destroy` closes the context (and, on the last one,
+ the shared engine); the first call after an idle gap rebuilds them. See
+ [[metabase.sql-parsing.common/create-pool]]."
+ (common/create-pool generate-context! destroy-context! {:max-size 3, :idle-minutes 10}))
+
+;;; ---------------------------------------------- timeout handling ---------------------------------------
+
+(def ^:private ^:const call-timeout-ms
+ "Timeout for Python calls: GraalPy can occasionally hang (DEV-1393)."
+ 30000)
+
+(defn- with-timeout*
+ "Execute f in a future; return its result, or `::timeout` after `timeout-ms`."
+ [timeout-ms f]
+ (let [fut (future (f))]
+ (try
+ (deref fut timeout-ms ::timeout)
+ (catch ExecutionException e
+ ;; Unwrap execution exception to get the real cause
+ (throw (or (.getCause e) e)))
+ (finally
+ ;; If we timed out or got an exception, try to cancel the future
+ ;; Note: This won't actually interrupt GraalVM, but prevents resource leaks
+ (future-cancel fut)))))
+
+(defn- interrupt!
+ "Interrupt Python execution in `context`. If the interrupt times out (guest stuck in uninterruptible
+ native code), force-close the context — force close cannot be denied by the guest."
+ [^Context context ^long timeout-ms]
+ (try
+ (.interrupt context (Duration/ofMillis timeout-ms))
+ (catch TimeoutException _
+ (log/warn "GraalVM interrupt timed out, forcing context closure")
+ (.close context true))))
+
+(defn- do-with-python-context
+ "Borrow a pooled GraalPy context and call `f` with it, held exclusively for the call (never let it — or
+ a context-bound `Value` — escape). The call is guarded by a timeout: a timed-out context is
+ interrupted and disposed from the pool (the next call generates a fresh one) instead of released."
+ [f]
+ (let [^Context context (.acquire python-context-pool pool-key)
+ poisoned? (volatile! false)]
+ (analytics/inc! :metabase-sql-parsing/context-acquisitions)
+ (try
+ (let [result (with-timeout* call-timeout-ms #(f context))]
+ (if (= result ::timeout)
+ (do
+ (vreset! poisoned? true)
+ ;; future-cancel doesn't stop GraalVM execution, so interrupt the context itself
+ ;; (1s grace period for the soft interrupt)
+ (interrupt! context 1000)
+ (analytics/inc! :metabase-sql-parsing/context-timeouts)
+ (log/warn "Python execution timed out after" call-timeout-ms "ms - GraalVM interrupted")
+ (throw (TimeoutException. (str "Python execution timed out after " call-timeout-ms "ms"))))
+ result))
+ (finally
+ (if @poisoned?
+ (.dispose python-context-pool pool-key context)
+ (.release python-context-pool pool-key context))))))
+
+;;; ------------------------------------------------- backend ---------------------------------------------
+
+(defn- call-python
+ "Execute `sql_tools.<fn-name>` with `args` on a pooled context and return the result as a string.
+ Python-side failures are rethrown as the transport-agnostic
+ [[metabase.sql-parsing.common/call-failed-ex]]."
+ ^String [fn-name & args]
+ (do-with-python-context
+ (fn [^Context context]
+ (try
+ (let [^Value fn-ref (.eval context "python" (str "sql_tools." fn-name))]
+ (.asString ^Value (.execute fn-ref (object-array args))))
+ (catch PolyglotException e
+ (throw (common/call-failed-ex (.getMessage e) e)))))))
+
+(defn parser
+ "The `:graalvm` [[metabase.sql-parsing.protocol/SqlParser]] — runs sqlglot on GraalPy on the pooled
+ contexts."
+ []
+ (common/make-parser call-python))
diff --git a/src/metabase/sql_parsing/init.clj b/src/metabase/sql_parsing/init.clj
new file mode 100644
index 00000000000..88bded2543e
--- /dev/null
+++ b/src/metabase/sql_parsing/init.clj
@@ -0,0 +1,3 @@
+(ns metabase.sql-parsing.init
+ (:require
+ [metabase.sql-parsing.settings]))
diff --git a/src/metabase/sql_parsing/parser.clj b/src/metabase/sql_parsing/parser.clj
new file mode 100644
index 00000000000..cad9dbe356e
--- /dev/null
+++ b/src/metabase/sql_parsing/parser.clj
@@ -0,0 +1,18 @@
+(ns metabase.sql-parsing.parser
+ "Hands callers the [[metabase.sql-parsing.protocol/SqlParser]] implementation for the configured
+ [[metabase.sql-parsing.settings/sql-parsing-mode]]: the in-process GraalPy parser (default) or a pool
+ of external native CPython processes. Kept behind the protocol so the caller
+ ([[metabase.sql-parsing.core]]) doesn't care which backend runs."
+ (:require
+ [metabase.sql-parsing.graal :as graal]
+ [metabase.sql-parsing.python :as python]
+ [metabase.sql-parsing.settings :as settings]))
+
+(set! *warn-on-reflection* true)
+
+(defn parser
+ "The sqlglot parser for the configured mode."
+ []
+ (if (= :python (settings/sql-parsing-mode))
+ (python/parser)
+ (graal/parser)))
diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.clj
deleted file mode 100644
index d9790504509..00000000000
--- a/src/metabase/sql_parsing/pool.clj
+++ /dev/null
@@ -1,343 +0,0 @@
-(ns metabase.sql-parsing.pool
- "Namespace to handle python environment. Has only a single public function, [[python-context]] which will return a
- python context. This is an internal detail to the sql-parsing module. If you want something from this, add a
- function to the core namespace that uses this. No one should know about this implmentation detail."
- (:require
- [clojure.java.io :as io]
- [clojure.java.shell :as shell]
- [metabase.analytics-interface.core :as analytics]
- [metabase.sql-parsing.common :as common]
- [metabase.util.files :as u.files]
- [metabase.util.log :as log]
- [potemkin :as p])
- (:import
- (io.aleph.dirigiste
- IPool$Controller
- IPool$Generator
- Pool
- Pools)
- (java.io Closeable File)
- (java.net URI)
- (java.nio.channels SeekableByteChannel)
- (java.nio.file AccessMode DirectoryStream DirectoryStream$Filter Files FileSystems Path)
- (java.time Duration)
- (java.util Map Set)
- (java.util.concurrent TimeUnit TimeoutException)
- (org.graalvm.polyglot Context Engine HostAccess)
- (org.graalvm.polyglot.io FileSystem IOAccess)))
-
-(set! *warn-on-reflection* true)
-
-;;; -------------------------------------------------- Python path resolution --------------------------------------------------
-
-(def ^:private python-sources-resource
- "Resource path for Python sources (sql_tools.py and sqlglot)."
- "python-sources")
-
-(defn- jar-resource?
- "True if the given resource is inside a JAR (production), false if from filesystem (dev)."
- [resource]
- (boolean
- (when-let [url (io/resource resource)]
- (.contains (.getFile ^java.net.URL url) ".jar!/"))))
-
-;;; -------------------------------------------------- Dev lazy installation --------------------------------------------------
-
-(def ^:private dev-python-sources-dir
- "Directory where sqlglot is installed in dev mode."
- "resources/python-sources")
-
-(defn- delete-recursive!
- "Recursively delete a directory and all its contents."
- [^File f]
- (when (.isDirectory f)
- (doseq [child (.listFiles f)]
- (delete-recursive! child)))
- (.delete f))
-
-(defn- expected-sqlglot-version
- "Read the expected sqlglot version from pyproject.toml resource file."
- []
- (some->> (io/resource "python-sources/pyproject.toml")
- slurp
- (re-find #"\"sqlglot==([^\"]+)\"")
- second))
-
-(defn- package-installer-available?
- "Check if uv or pip is available for installing Python packages."
- []
- (or (zero? (:exit (shell/sh "which" "uv")))
- (zero? (:exit (shell/sh "which" "pip")))))
-
-(defn- version-installed?
- "Check if sqlglot is installed with the expected version. Verifies both the dist-info metadata
- (needed by uv/pip) and the actual module (needed by Python import)."
- [target-dir expected-version]
- (and (.exists (io/file target-dir (str "sqlglot-" expected-version ".dist-info") "METADATA"))
- (.exists (io/file target-dir "sqlglot" "__init__.py"))))
-
-(defn- install-sqlglot!
- "Install sqlglot via uv (preferred) or pip (fallback).
- Deletes any existing sqlglot directories first to handle version upgrades."
- [target-dir version]
- ;; Delete old versions first (sqlglot/ and any sqlglot-*.dist-info/)
- (doseq [^File f (.listFiles (io/file target-dir))]
- (when (or (= "sqlglot" (.getName f))
- (.startsWith (.getName f) "sqlglot-"))
- (log/info "Removing old sqlglot:" (.getName f))
- (delete-recursive! f)))
- ;; Try uv first (fast), fall back to pip
- (let [pyproject-file (str target-dir "/pyproject.toml")
- uv-result (shell/sh "uv" "pip" "install" "-r" pyproject-file "--target" target-dir "--no-compile" "--reinstall")]
- (if (zero? (:exit uv-result))
- (log/info "sqlglot" version "installed via uv")
- (do
- (log/info "uv not available, trying pip...")
- (let [pkg (str "sqlglot==" version)
- pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile" "--force-reinstall")]
- (when-not (zero? (:exit pip-result))
- (throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"
- "Manual install: uv pip install -r " pyproject-file " --target " target-dir "\n"
- "Install uv: https://docs.astral.sh/uv/getting-started/installation/")
- {:uv-error (:err uv-result)
- :pip-error (:err pip-result)
- :version version
- :target-dir target-dir})))
- (log/info "sqlglot" version "installed via pip"))))))
-
-(defn- ensure-sqlglot-installed!
- "Ensure sqlglot is installed with correct version. Called lazily on first use in dev.
- If version mismatch or missing, automatically installs the correct version."
- []
- (let [expected-ver (expected-sqlglot-version)]
- (when-not expected-ver
- (throw (ex-info "Missing pyproject.toml or unable to parse sqlglot entry in resources/python-sources/"
- {:resource "python-sources/pyproject.toml"})))
- (when-not (version-installed? dev-python-sources-dir expected-ver)
- (if (package-installer-available?)
- (do
- (log/info "Installing sqlglot" expected-ver "(first use in dev)...")
- (install-sqlglot! dev-python-sources-dir expected-ver))
- (log/warn "sqlglot not installed and no package installer (uv or pip) found."
- "SQL parsing features will fail until sqlglot is installed."
- "Install uv: https://docs.astral.sh/uv/getting-started/installation/")))))
-
-;;; -------------------------------------------------- Python path delay --------------------------------------------------
-
-(defn- nio-polyglot-fs
- "Wrap a NIO `java.nio.file.FileSystem` as a polyglot `FileSystem`, routing reads to
- `Files/newByteChannel` instead of the provider's `newFileChannel`.
-
- The stock wrapper triggers ZipFileSystem to extract deflated entries to a temp file beside the jar,
- which fails when the jar's parent directory isn't writable (e.g. non-root in a Kubernetes pod).
- See https://github.com/metabase/metabase/issues/73541."
- ^FileSystem [^java.nio.file.FileSystem nio-fs]
- (let [provider (.provider nio-fs)]
- (reify FileSystem
- (^Path parsePath [_ ^URI uri]
- (.getPath provider uri))
- (^Path parsePath [_ ^String s]
- (.getPath nio-fs s (into-array String [])))
- (^void checkAccess [_ ^Path p ^Set modes ^"[Ljava.nio.file.LinkOption;" _opts]
- ;; LinkOption opts are dropped: the only value is NOFOLLOW_LINKS, and neither our jar zip FS
- ;; nor resources/python-sources contains symlinks.
- (.checkAccess provider p ^"[Ljava.nio.file.AccessMode;" (into-array AccessMode modes)))
- (^void createDirectory [_ ^Path p ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]
- (Files/createDirectory p attrs))
- (^void delete [_ ^Path p]
- (Files/delete p))
- (^SeekableByteChannel newByteChannel
- [_ ^Path p ^Set opts ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]
- (Files/newByteChannel p opts attrs))
- (^DirectoryStream newDirectoryStream [_ ^Path dir ^DirectoryStream$Filter filter]
- (Files/newDirectoryStream dir filter))
- (^Path toAbsolutePath [_ ^Path p]
- (.toAbsolutePath p))
- (^Path toRealPath [_ ^Path p ^"[Ljava.nio.file.LinkOption;" opts]
- (.toRealPath p opts))
- (^Map readAttributes [_ ^Path p ^String attrs ^"[Ljava.nio.file.LinkOption;" opts]
- (Files/readAttributes p attrs opts)))))
-
-(defn- read-only-polyglot-fs
- "Wrap an NIO `java.nio.file.FileSystem` as a read-only polyglot FileSystem suitable for GraalPy."
- ^FileSystem [^java.nio.file.FileSystem nio-fs]
- (-> nio-fs nio-polyglot-fs FileSystem/newReadOnlyFileSystem))
-
-(defonce ^:private
- ^{:doc "A read-only polyglot FileSystem and the PythonPath within it.
- In dev: wraps the default filesystem, path is resources/python-sources.
- In jar: wraps the jar's zip filesystem, path is /python-sources.
- Either way, the filesystem is read-only so extracted sources cannot be tampered with."}
- python-fs-and-path
- (delay
- (if (jar-resource? python-sources-resource)
- ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib are both inside the
- ;; jar, so nothing is extracted to disk and there's nothing to tamper with. The filesystem lives for the
- ;; duration of the process (via defonce + delay) and is shared by all pooled Python contexts.
- {:fs (-> (u.files/get-jar-path) u.files/nio-fs read-only-polyglot-fs)
- :python-path "/python-sources"
- :std-lib-home "/META-INF/resources/libpython"
- :core-home "/META-INF/resources/libgraalpy"}
- ;; In dev: use the real filesystem (read-only wrapper). sqlglot is installed locally.
- (do
- (ensure-sqlglot-installed!)
- {:fs (read-only-polyglot-fs (FileSystems/getDefault))
- :python-path dev-python-sources-dir}))))
-
-;;; -------------------------------------------- Context Wrappers ----------------------------------------------------
-
-(p/defrecord+ PooledContext [^Context context ^Pool pool tuple poisoned?]
- common/PythonEval
- (eval-python [_this code]
- (.eval context "python" ^String code))
-
- Closeable
- (close [_this]
- (if @poisoned?
- (do
- (log/warn "Disposing poisoned Python context (likely hung GraalVM)")
- (.dispose pool :python tuple))
- (.release pool :python tuple))))
-
-(defn poison!
- "Mark a PooledContext as poisoned. When closed, it will be disposed from the pool
- rather than released back. Use this when GraalVM hangs to prevent returning a
- broken context to the pool."
- [ctx]
- (when (instance? PooledContext ctx)
- (reset! (:poisoned? ctx) true)))
-
-(defn interrupt!
- "Interrupt Python execution in this context. Returns true if interrupted successfully.
- If interrupt times out (guest in uninterruptible code), forces context closure with close(true).
- This is necessary because future-cancel doesn't actually stop GraalVM execution."
- [ctx ^long timeout-ms]
- (when-let [^Context raw-ctx (when (instance? PooledContext ctx)
- (:context ctx))]
- (try
- (.interrupt raw-ctx (Duration/ofMillis timeout-ms))
- true
- (catch TimeoutException _
- ;; Interrupt timed out - guest app may be in uninterruptible native code
- ;; Force close cannot be denied by guest application
- (log/warn "GraalVM interrupt timed out, forcing context closure")
- (.close raw-ctx true)
- false))))
-
-(defonce ^:private
- ^{:doc "A single GraalVM `Engine` shared by every pooled Python context. The engine owns the Truffle runtime and the
- (parsed/compiled) code cache; sharing it across contexts means recycling a context on its TTL no longer
- recreates an engine or re-loads sqlglot into a fresh code cache. That per-context engine state is the dominant
- native-memory cost and, because contexts churn on a 10-minute TTL and GraalPy doesn't reliably return all of
- it to the OS on close, a major source of the native-memory growth. The engine is a process-lifetime singleton
- (it is intentionally never closed)."}
- shared-python-engine
- (delay
- (.. (Engine/newBuilder)
- (option "engine.WarnInterpreterOnly" "false")
- (build))))
-
-(defn- create-graalvm-context
- "Create a new GraalVM Python context configured for sqlglot.
-
- The context is configured with:
- - The shared [[shared-python-engine]] (so contexts share one Truffle runtime + code cache)
- - PythonPath pointing to python-sources (contains sql_tools.py and sqlglot)
- - Full host access (needed for JSON serialization)
- - Read-only filesystem (sources loaded directly from classpath/jar, no disk extraction in prod)"
- ^Context []
- (let [{:keys [^FileSystem fs python-path std-lib-home core-home]} @python-fs-and-path
- io-access (.. (IOAccess/newBuilder)
- (fileSystem fs)
- (build))
- builder (.. (Context/newBuilder (into-array String ["python"]))
- (engine @shared-python-engine)
- (option "python.PythonPath" python-path)
- (allowHostAccess HostAccess/ALL)
- (allowIO io-access))
- ;; When running from the jar, GraalPy's stdlib is also inside the jar filesystem
- ;; and we need to tell it where to find the core and stdlib paths explicitly.
- builder (cond-> builder
- std-lib-home (.option "python.StdLibHome" std-lib-home)
- core-home (.option "python.CoreHome" core-home))
- ctx (.build builder)]
- (.eval ctx "python" "import sql_tools")
- ctx))
-
-(defn- make-python-context-pool
- "Create a pool of Python contexts. Accepts a generator function and optional config map.
-
- Config options:
- - :max-size - Maximum pool size (default: 3)
- - :min-size - Minimum pool size (default: 1)
- - :ttl-minutes - How long contexts live before expiry (default: 10)
- - :utilization - Target utilization (default: 1.0 = 100%)
-
- This allows for easy testing by injecting mock context generators and custom pool configs."
- ([context-generator]
- (make-python-context-pool context-generator {}))
- ([context-generator {:keys [max-size min-size ttl-minutes utilization]
- :or {max-size 3
- min-size 1
- ttl-minutes 10
- utilization 1.0}}]
- (let [base-controller (Pools/utilizationController utilization max-size max-size)]
- (Pool. (reify IPool$Generator
- (generate [_ _]
- ;; Generate a tuple of the context and the expiry timestamp.
- (analytics/inc! :metabase-sql-parsing/context-creations)
- [(context-generator)
- (+ (System/nanoTime) (.toNanos TimeUnit/MINUTES ttl-minutes))])
- (destroy [_ _ [^Context ctx _expiry]]
- ;; Close context when disposed from pool (expiry, poison, or shutdown)
- (try
- (.close ctx true) ;; Force close - can't wait for running code
- (catch Exception _))))
- ;; Wrap the utilization controller with a modification that doesn't allow the pool to go below min-size.
- (reify IPool$Controller
- (shouldIncrement [_ k a b] (.shouldIncrement base-controller k a b))
- (adjustment [_ stats]
- (let [adj (.adjustment base-controller stats)
- ;; :python is arbitrary key, it just has to be consistent everywhere when working with the pool.
- n (some-> ^io.aleph.dirigiste.Stats (:python stats) .getNumWorkers)
- python-adj (:python adj)]
- (if (and n python-adj (<= (+ n python-adj) min-size))
- ;; If the adjustment is going to bring the pool below min-size, return empty adjustment instead.
- {}
- adj))))
- 65000 ;; Queue size - doesn't matter much.
- 25 ;; Sampling interval - doesn't matter much.
- 10000 ;; Recheck every 10 seconds
- TimeUnit/MILLISECONDS))))
-
-(def ^:private python-context-pool
- "Pool of Python context objects. They are not thread-safe, so access to them has to be carefully managed between
- threads. Each context with loaded Python interpreter takes significant memory, so we don't want too many of them.
- However, one takes significant time to initialize, so we don't want to load them anew each time in prod. Under some
- circumstances, the GraalVM Python context tends to leak memory, so we don't want to keep the reference to the
- context forever. Considering all that, this pool targets 100% utilization (so, if the utilization is lower, the pool
- will start dropping objects) and the maximum of 3 objects (to prevent OOMs), but at least 1 object will always be in
- the pool to pick up. However, together with each context keep its creation timestamp so that we can throwaway
- instances that are too old to avoid leaks.
-
- Wrapped in delay to avoid initialization during AOT compilation."
- (delay (make-python-context-pool create-graalvm-context {:max-size 3
- :min-size 1
- :ttl-minutes 10})))
-
-(defn- acquire-context
- "Acquire a context from the pool, handling expiry. Returns a PooledContext."
- [^Pool pool]
- (loop []
- (let [[context expiry-ts :as tuple] (.acquire pool :python)]
- (if (>= (System/nanoTime) expiry-ts)
- (do (analytics/inc! :metabase-sql-parsing/context-disposals-expired)
- (.dispose pool :python tuple)
- (recur))
- (do (analytics/inc! :metabase-sql-parsing/context-acquisitions)
- (->PooledContext context pool tuple (atom false)))))))
-
-(defn python-context
- "Acquire a python context from the pool. Must be closed. Use in a `with-open` context."
- []
- (acquire-context @python-context-pool))
diff --git a/src/metabase/sql_parsing/protocol.clj b/src/metabase/sql_parsing/protocol.clj
new file mode 100644
index 00000000000..400695d5500
--- /dev/null
+++ b/src/metabase/sql_parsing/protocol.clj
@@ -0,0 +1,39 @@
+(ns metabase.sql-parsing.protocol
+ "The strongly-typed protocol implemented by the sqlglot SQL parsers (currently
+ [[metabase.sql-parsing.graal]]). Callers go through [[metabase.sql-parsing.core]], which owns the
+ JVM-side pre/post-processing and invokes these methods on the implementation.")
+
+(set! *warn-on-reflection* true)
+
+(defprotocol SqlParser
+ "Dialect-aware SQL parsing via sqlglot. Each method calls the corresponding `sql_tools` Python function
+ and returns its result parsed from JSON into Clojure data, except the SQL-rewriting methods
+ ([[add-into-clause]], [[replace-names]]) which return the rewritten SQL string. `dialect` is a sqlglot
+ dialect string, or nil for sqlglot's default dialect."
+ (referenced-tables [parser dialect sql]
+ "Table references literally present in the SQL, as a vector of `[catalog schema table]` 3-tuples.")
+ (referenced-fields [parser dialect sql]
+ "Field references from actual database tables, as a vector of `[catalog schema table field]`
+ 4-tuples (wildcards as `\"*\"`).")
+ (returned-columns-lineage [parser dialect sql default-table-schema sqlglot-schema]
+ "Column lineage of the query's output, as a vector of `[alias pure? deps]` tuples.")
+ (validate-query [parser dialect sql default-table-schema sqlglot-schema]
+ "Validate the query against `sqlglot-schema` (strict) or its own structure (permissive when the
+ schema is nil/empty). Returns a keywordized `{:status \"ok\"}` or `{:status \"error\" ...}` map.")
+ (simple-query [parser dialect sql]
+ "Whether the SQL is a simple SELECT without LIMIT, OFFSET, or CTEs. Returns a keywordized
+ `{:is_simple ...}` map with a `:reason` when false.")
+ (add-into-clause [parser dialect sql table-name]
+ "Add an `INTO table-name` clause to a SELECT statement (SQL Server SELECT INTO syntax). Returns the
+ rewritten SQL string.")
+ (field-references [parser dialect sql]
+ "Used and returned field specs, as a keywordized map of `:used_fields`, `:returned_fields`, and
+ `:errors` in the raw Python shape.")
+ (replace-names [parser dialect sql replacements]
+ "Replace schema/table/column names in the SQL. Returns the rewritten SQL string. Replacement values
+ are injected into the AST unsanitized — callers must ensure they are system-generated.")
+ (single-stmt-of-type [parser dialect sql stmt-type]
+ "Whether the SQL is a single statement of `stmt-type`, as a keywordized map that includes the SQL
+ reconstructed from the parsed AST.")
+ (transpile-sql [parser sql from-dialect to-dialect]
+ "Transpile the SQL from one dialect to another, as a keywordized result map."))
diff --git a/src/metabase/sql_parsing/python.clj b/src/metabase/sql_parsing/python.clj
new file mode 100644
index 00000000000..0776da7e92b
--- /dev/null
+++ b/src/metabase/sql_parsing/python.clj
@@ -0,0 +1,199 @@
+(ns metabase.sql-parsing.python
+ "The `:python` [[metabase.sql-parsing.protocol/SqlParser]]: runs sqlglot in a pool of external native
+ CPython child processes rather than in-process on GraalPy. Each process runs `sql_tools_cli.py`, which
+ answers newline-delimited JSON requests over stdin/stdout.
+
+ The Python sources come straight out of the classpath: in dev the child imports directly from
+ `resources/python-sources`; from the jar they are written to a fresh temp zip at every process spawn
+ (CPython imports from zips via zipimport), put on the child's PYTHONPATH, and deleted when the process
+ is destroyed — so there is no long-lived on-disk copy to drift out of sync or be tampered with, and no
+ deploy-time provisioning.
+
+ The pool holds a single process, held exclusively per call (so calls serialize onto it); with a min of
+ 0 it shrinks to 0 when idle, killing the process after 10 minutes with no work. We cap it at one
+ process — spawning a fresh `python3` (interpreter init + sqlglot import) is costly, and a warm process
+ parses fast enough that queuing a second call behind it beats paying that startup cost for
+ concurrency.
+
+ Requires a `python3` binary on the host's PATH."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabase.sql-parsing.common :as common]
+ [metabase.util.files :as u.files]
+ [metabase.util.json :as json])
+ (:import
+ (io.aleph.dirigiste Pool)
+ (java.io BufferedReader BufferedWriter File IOException InputStreamReader OutputStreamWriter)
+ (java.lang ProcessBuilder$Redirect)
+ (java.nio.charset StandardCharsets)
+ (java.nio.file FileVisitOption Files LinkOption)
+ (java.nio.file.attribute FileAttribute)
+ (java.util.concurrent TimeoutException)
+ (java.util.zip ZipEntry ZipOutputStream)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------ process ----------------------------------------------
+
+(def ^:private ^:const call-timeout-ms
+ "How long to wait for the process to answer before assuming it has wedged: we kill it and fail the
+ call, so a stuck process can't hold its pool slot forever."
+ 30000)
+
+(defrecord ^:private PythonProcess [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip])
+
+(defn- read-response-line
+ "Read one line from the process, killing it and throwing if it doesn't answer within
+ [[call-timeout-ms]] (so a wedged process can't hold its pool slot forever). `destroyForcibly`
+ (SIGKILL) rather than the polite `destroy`: a process wedged hard enough to miss the deadline may
+ ignore SIGTERM, and killing it also closes its stdout, which unblocks the reader future."
+ ^String [^BufferedReader reader ^Process process]
+ (let [fut (future (.readLine reader))
+ line (deref fut call-timeout-ms ::timeout)]
+ (if (identical? ::timeout line)
+ (do (.destroyForcibly process)
+ (future-cancel fut)
+ (throw (TimeoutException. (str "Python execution timed out after " call-timeout-ms "ms"))))
+ line)))
+
+(defn- create-sources-zip!
+ "Write the python-sources tree from the jar to a fresh temp zip, importable by the child via zipimport,
+ to be put on its PYTHONPATH and deleted when its process is destroyed (`deleteOnExit` is only a
+ backstop for JVM death)."
+ ^File []
+ (let [file (.toFile (Files/createTempFile "mb-sql-tools" ".zip" (into-array FileAttribute [])))]
+ (.deleteOnExit file)
+ (try
+ (with-open [jar-fs (u.files/nio-fs (u.files/get-jar-path))
+ zos (ZipOutputStream. (io/output-stream file))]
+ (let [root (.getPath jar-fs (str "/" common/python-sources-resource) (into-array String []))]
+ (with-open [paths (Files/walk root (into-array FileVisitOption []))]
+ (doseq [^java.nio.file.Path path (iterator-seq (.iterator paths))
+ :when (and (not (Files/isDirectory path (into-array LinkOption [])))
+ (not (str/includes? (str path) "__pycache__")))]
+ (.putNextEntry zos (ZipEntry. (str (.relativize root path))))
+ (Files/copy path zos)
+ (.closeEntry zos)))))
+ file
+ (catch Throwable t
+ (.delete file)
+ (throw t)))))
+
+(defn- python-path
+ "The PYTHONPATH entry for a new child process, plus the temp zip to delete with it (nil in dev, where
+ the child imports straight from `resources/python-sources`)."
+ []
+ (if (common/jar-resource? common/python-sources-resource)
+ (let [zip (create-sources-zip!)]
+ {:path (.getAbsolutePath zip) :sources-zip zip})
+ (do
+ (common/ensure-sqlglot-installed!)
+ {:path common/dev-python-sources-dir :sources-zip nil})))
+
+(defn- start-process!
+ "Spawn `python3 -m sql_tools_cli` with the Python sources on PYTHONPATH and block until it reports it
+ has loaded. The process's stderr is inherited (for debugging); stdout carries the line protocol. On
+ any failure to reach a ready state the process is destroyed and the temp zip deleted, so nothing
+ leaks."
+ ^PythonProcess []
+ (let [{:keys [path ^File sources-zip]} (python-path)
+ ^Process process (try
+ (let [builder (.. (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String ["python3" "-m" "sql_tools_cli"]))
+ (redirectError ProcessBuilder$Redirect/INHERIT))]
+ (doto (.environment builder)
+ (.put "PYTHONPATH" path)
+ (.put "PYTHONUNBUFFERED" "1"))
+ (.start builder))
+ (catch Throwable t
+ (some-> sources-zip .delete)
+ (if (instance? IOException t)
+ (throw (ex-info (str "could not start `python3` for SQL parsing — is python3"
+ " installed and on PATH? (otherwise set MB_SQL_PARSING_MODE"
+ " to graalvm)")
+ {} t))
+ (throw t))))]
+ (try
+ (let [writer (BufferedWriter. (OutputStreamWriter. (.getOutputStream process) StandardCharsets/UTF_8))
+ reader (BufferedReader. (InputStreamReader. (.getInputStream process) StandardCharsets/UTF_8))
+ ready (read-response-line reader process)]
+ (when-not (some-> ready json/decode+kw :ready)
+ (throw (ex-info "sql-parsing python process failed to start" {:line ready})))
+ (->PythonProcess process writer reader sources-zip))
+ (catch Throwable t
+ (.destroy process)
+ (some-> sources-zip .delete)
+ (throw t)))))
+
+(defn- stop-process!
+ [{:keys [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip]}]
+ (try (.close writer) (catch Exception _))
+ (try (.close reader) (catch Exception _))
+ (try (.destroy process) (catch Exception _))
+ (some-> sources-zip .delete))
+
+(defn- call-on-process
+ "Send one request to `python-process` and read its response. Returns the `sql_tools` function's result
+ string. Not thread-safe on its own — the pool hands a process to one call at a time. A write failure
+ (broken pipe) means the pooled process had already died; it is thrown as `:retryable` so [[call-python]]
+ can get a fresh one."
+ ^String [{:keys [^BufferedWriter writer ^BufferedReader reader ^Process process]} fn-name args]
+ (try
+ (.write writer ^String (str (json/encode {:fn fn-name :args args}) "\n"))
+ (.flush writer)
+ (catch IOException e
+ (throw (ex-info "sql-parsing python process is not writable" {:retryable true} e))))
+ (let [line (read-response-line reader process)]
+ (when (nil? line)
+ (throw (ex-info "sql-parsing python process closed unexpectedly" {:retryable true})))
+ (let [{:keys [ok result error]} (json/decode+kw line)]
+ (if ok
+ result
+ ;; The process answered the protocol correctly, so it is healthy; only this call failed.
+ ;; [[call-python]] releases (rather than disposes) the process on these.
+ (throw (common/call-failed-ex error))))))
+
+;;; ------------------------------------------------ pool -------------------------------------------------
+
+(def ^:private pool-key
+ "Dirigiste pools are keyed; the key is arbitrary, it just has to be the same for every operation."
+ :sql-parsing-python)
+
+(def ^:private ^Pool python-process-pool
+ "A pool of a single sql-parsing CPython process, held exclusively per call (so calls serialize onto
+ it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because
+ a warm process is fast and spawning another is costly. See
+ [[metabase.sql-parsing.common/create-pool]]."
+ (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 10}))
+
+(defn- call-python
+ "Execute `sql_tools.<fn-name>` with `args` on a pooled CPython process and return the result as a
+ string. A Python-side failure (the process answered `ok: false` — it is healthy, only the call
+ failed, thrown as the transport-agnostic [[metabase.sql-parsing.common/call-failed-ex]]) releases the
+ process back for reuse, like the graal parser does with its context; any other failure may have left
+ the process in a bad state, so it is disposed (dropped from the pool). Retries once if the acquired
+ process turned out to be already dead (e.g. reaped or killed while parked in the pool), since a fresh
+ one should succeed."
+ ^String [fn-name & args]
+ (loop [retries 1]
+ (let [python-process (.acquire python-process-pool pool-key)
+ outcome (try
+ (let [result (call-on-process python-process fn-name args)]
+ (.release python-process-pool pool-key python-process)
+ {:result result})
+ (catch Throwable t
+ (if (:sql-parsing/error (ex-data t))
+ (.release python-process-pool pool-key python-process)
+ (.dispose python-process-pool pool-key python-process))
+ (if (and (:retryable (ex-data t)) (pos? retries))
+ {:retry true}
+ (throw t))))]
+ (if (:retry outcome)
+ (recur (dec retries))
+ (:result outcome)))))
+
+(defn parser
+ "The `:python` [[metabase.sql-parsing.protocol/SqlParser]] — runs sqlglot in a pool of external native
+ CPython processes."
+ []
+ (common/make-parser call-python))
diff --git a/src/metabase/sql_parsing/settings.clj b/src/metabase/sql_parsing/settings.clj
new file mode 100644
index 00000000000..476e8bbb435
--- /dev/null
+++ b/src/metabase/sql_parsing/settings.clj
@@ -0,0 +1,18 @@
+(ns metabase.sql-parsing.settings
+ (:require
+ [metabase.settings.core :as setting :refer [defsetting]]
+ [metabase.util.i18n :refer [deferred-tru tru]]))
+
+(set! *warn-on-reflection* true)
+
+(defsetting sql-parsing-mode
+ (deferred-tru "How SQL parsing (sqlglot) runs: `graalvm` runs it in-process on pooled GraalPy contexts (default), or `python` runs it in a pool of external native CPython child processes (requires a `python3` binary on the host PATH).")
+ :type :keyword
+ :visibility :internal
+ :default :graalvm
+ :export? false
+ :setter (fn [new-value]
+ (when (some? new-value)
+ (assert (#{:graalvm :python} (keyword new-value))
+ (tru "Invalid sql-parsing-mode! Only values of graalvm and python are allowed.")))
+ (setting/set-value-of-type! :keyword :sql-parsing-mode new-value)))
diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.clj
index 1fdd5f6fa56..9ecc383ab2d 100644
--- a/src/metabase/sql_tools/sqlglot/core.clj
+++ b/src/metabase/sql_tools/sqlglot/core.clj
@@ -1,15 +1,12 @@
(ns metabase.sql-tools.sqlglot.core
(:require
- [clojure.string :as str]
[metabase.driver.sql.normalize :as sql.normalize]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.sql-parsing.core :as sql-parsing]
[metabase.sql-tools.common :as sql-tools.common]
[metabase.sql-tools.interface :as sql-tools]
- [metabase.util.log :as log])
- (:import
- (org.graalvm.polyglot PolyglotException)))
+ [metabase.util.log :as log]))
(set! *warn-on-reflection* true)
@@ -67,9 +64,9 @@
driver {:table table
:schema (or table-schema default-schema)}))))
query-tables))
- (catch PolyglotException e
+ (catch Exception e
;; Return empty sequence on parse error to follow the Macaw implementation behavior.
- (if (str/starts-with? (str (.getMessage e)) "ParseError")
+ (if (sql-parsing/parse-error? e)
#{}
(throw e)))))
@@ -152,9 +149,9 @@
(mapv (fn [[_catalog schema table]]
{:schema schema :table table})
table-tuples))
- (catch PolyglotException e
+ (catch Exception e
;; Return empty sequence on parse error to follow the Macaw implementation behavior.
- (if (str/starts-with? (str (.getMessage e)) "ParseError")
+ (if (sql-parsing/parse-error? e)
[]
(throw e)))))
diff --git a/test/metabase/sql_parsing/common_test.clj b/test/metabase/sql_parsing/common_test.clj
new file mode 100644
index 00000000000..32e45fa1800
--- /dev/null
+++ b/test/metabase/sql_parsing/common_test.clj
@@ -0,0 +1,151 @@
+(ns metabase.sql-parsing.common-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.sql-parsing.common :as common])
+ (:import
+ (io.aleph.dirigiste Pool)
+ (java.util.concurrent CountDownLatch)))
+
+(set! *warn-on-reflection* true)
+
+(defn- mock-worker
+ "Create a mock 'worker' object. The pool doesn't care what type the objects are - it just stores and
+ returns them."
+ [id]
+ {:worker-id id
+ :created-at (System/currentTimeMillis)})
+
+(defn- counting-generator
+ "Returns a generator function that counts how many workers it has created."
+ [counter-atom]
+ (fn []
+ (mock-worker (swap! counter-atom inc))))
+
+(defn- failing-generator
+ "Returns a generator that fails after creating n workers."
+ [counter-atom fail-after]
+ (fn []
+ (let [id (swap! counter-atom inc)]
+ (when (> id fail-after)
+ (throw (ex-info "Generator limit reached" {:count id :limit fail-after})))
+ (mock-worker id))))
+
+;;; ------------------------------------------------- Test Helpers ---------------------------------------------------
+
+(defn- test-pool
+ "Create a pool for testing with sensible defaults.
+ Returns [pool created-count-atom destroyed-atom]."
+ ([]
+ (test-pool {}))
+ ([config]
+ (let [created (atom 0)
+ destroyed (atom [])
+ pool (common/create-pool (counting-generator created)
+ (fn [worker] (swap! destroyed conj worker))
+ (merge {:max-size 2, :idle-minutes 1} config))]
+ [pool created destroyed])))
+
+(defn- with-worker
+ "Execute f with a worker from pool, releasing it afterwards."
+ [^Pool pool f]
+ (let [worker (.acquire pool :test)]
+ (try
+ (f worker)
+ (finally
+ (.release pool :test worker)))))
+
+(defn- run-concurrent
+ "Run n threads concurrently, each executing f with a worker from pool.
+ Returns vector of results."
+ [pool n f]
+ (let [latch (CountDownLatch. n)
+ results (atom [])
+ futures (doall
+ (for [i (range n)]
+ (future
+ (.countDown latch)
+ (.await latch)
+ (let [result (with-worker pool (fn [worker] (f i worker)))]
+ (swap! results conj result)))))]
+ (doseq [fut futures] @fut)
+ @results))
+
+;;; ------------------------------------------------ Pool Size Tests -------------------------------------------------
+
+(deftest pool-respects-maximum-size-test
+ (testing "Pool never creates more than max workers even under concurrent load"
+ (let [[pool created] (test-pool {:max-size 2})
+ num-threads 20
+ results (run-concurrent pool num-threads
+ (fn [_i worker]
+ (Thread/sleep (long (+ 10 (rand-int 50))))
+ worker))]
+ (is (<= @created 2)
+ (str "Pool created " @created " workers but max is 2"))
+ (is (= num-threads (count results))
+ "All threads should have received a worker"))))
+
+(deftest pool-reuses-idle-workers-test
+ (testing "Sequential uses reuse the same worker instead of creating new ones"
+ (let [[pool created] (test-pool)]
+ (with-worker pool (fn [_worker] (is (= 1 @created) "First use creates one worker")))
+ (with-worker pool (fn [worker]
+ (is (= 1 @created) "Second use reuses the same worker")
+ (is (= 1 (:worker-id worker))))))))
+
+;;; ------------------------------------------------ Disposal Tests --------------------------------------------------
+
+(deftest pool-dispose-destroys-and-replaces-workers-test
+ (testing "Disposing a worker calls destroy and the next acquire generates a fresh one"
+ (let [[pool created destroyed] (test-pool)
+ worker (.acquire ^Pool pool :test)]
+ (.dispose ^Pool pool :test worker)
+ (is (= [worker] @destroyed) "Disposed worker should be destroyed")
+ (with-worker pool (fn [_worker] (is (= 2 @created) "Disposed worker should be replaced"))))))
+
+;;; --------------------------------------------- Generator Failure Tests --------------------------------------------
+
+(deftest pool-handles-generator-failures-test
+ (testing "Pool gracefully handles when generator fails"
+ (let [counter (atom 0)
+ pool (common/create-pool (failing-generator counter 1) (fn [_worker]) {:max-size 2, :idle-minutes 1})]
+ (is (some? (with-worker pool identity)) "First worker creation succeeds")
+ ;; Dispose to force new creation
+ (let [worker (.acquire ^Pool pool :test)]
+ (.dispose ^Pool pool :test worker))
+ (is (thrown? Exception (with-worker pool identity)) "Generator failure should propagate"))))
+
+;;; ------------------------------------------- Concurrent Access Tests ----------------------------------------------
+
+(deftest pool-handles-concurrent-access-safely-test
+ (testing "Pool handles concurrent access without race conditions"
+ (let [[pool created] (test-pool {:max-size 2})
+ num-threads 50
+ results (run-concurrent pool num-threads
+ (fn [i _worker]
+ (Thread/sleep (long (rand-int 20)))
+ i))]
+ (is (= num-threads (count results)) "All threads should complete successfully")
+ (is (<= @created 2) (str "Created " @created " workers, expected <= 2")))))
+
+;;; ----------------------------------------- Version Parsing --------------------------------------------------------
+
+(deftest expected-sqlglot-version-test
+ (testing "Sqlglot version is properly parsed from pyproject.toml"
+ (let [version (common/expected-sqlglot-version)]
+ (is (string? version))
+ (is (re-matches #"\d+\.\d+\.\d+" version)))))
+
+;;; -------------------------------------------- Pool Behavior Tests -------------------------------------------------
+
+(deftest pool-stores-and-returns-objects-test
+ (testing "Pool correctly stores and returns arbitrary objects"
+ (let [[pool created] (test-pool)
+ result1 (with-worker pool identity)
+ result2 (with-worker pool identity)]
+ (is (map? result1))
+ (is (= 1 (:worker-id result1)))
+ (is (= 1 @created))
+ (is (map? result2))
+ (is (= 1 (:worker-id result2))) ;; Same worker ID
+ (is (= 1 @created)))))
diff --git a/test/metabase/sql_parsing/core_test.clj b/test/metabase/sql_parsing/core_test.clj
index 556a0049209..561569cc314 100644
--- a/test/metabase/sql_parsing/core_test.clj
+++ b/test/metabase/sql_parsing/core_test.clj
@@ -524,3 +524,12 @@
"Should include users wildcard")
(is (some #(= ["transactions" "total"] %) (normalize-fields result))
"Should include transactions.total")))))
+
+;;; ------------------------------------------------ Parse errors --------------------------------------------------
+
+(deftest parse-error-test
+ (testing "unparseable SQL surfaces as a transport-agnostic parse error"
+ (let [e (try
+ (sql-parsing/referenced-tables "postgres" "SELECT !!!")
+ (catch Exception e e))]
+ (is (sql-parsing/parse-error? e)))))
diff --git a/test/metabase/sql_parsing/jar_python_fs_repro_test.clj b/test/metabase/sql_parsing/jar_python_fs_repro_test.clj
index 5d2ab44d47e..1380ffa923b 100644
--- a/test/metabase/sql_parsing/jar_python_fs_repro_test.clj
+++ b/test/metabase/sql_parsing/jar_python_fs_repro_test.clj
@@ -4,11 +4,11 @@
GraalVM's bundled polyglot `FileSystem` wrapper routes `newByteChannel` through
`provider.newFileChannel`, which forces JDK ZipFileSystem to extract deflated entries to a temp file
beside the jar — fails when that directory isn't writable (non-root in a Kubernetes pod).
- The fix in `metabase.sql-parsing.pool/nio-polyglot-fs` overrides `newByteChannel` to call
+ The fix in `metabase.sql-parsing.graal/nio-polyglot-fs` overrides `newByteChannel` to call
`Files/newByteChannel`, which returns an in-memory `ByteArrayChannel` for compressed entries."
(:require
[clojure.test :refer :all]
- [metabase.sql-parsing.pool :as pool]
+ [metabase.sql-parsing.graal :as graal]
[metabase.util.files :as u.files])
(:import
(java.io FileOutputStream)
@@ -37,7 +37,7 @@
(.write zos (.getBytes "print('hello')\n" "UTF-8"))
(.closeEntry zos)))
(with-open [^java.nio.file.FileSystem nio-fs (u.files/nio-fs (str zip-file))]
- (let [^FileSystem polyfs (#'pool/read-only-polyglot-fs nio-fs)
+ (let [^FileSystem polyfs (#'graal/read-only-polyglot-fs nio-fs)
path (.parsePath polyfs "/python-sources/sql_tools.py")]
(with-open [ch (.newByteChannel polyfs path
(EnumSet/of StandardOpenOption/READ)
diff --git a/test/metabase/sql_parsing/pool_test.clj b/test/metabase/sql_parsing/pool_test.clj
deleted file mode 100644
index 30fa0a8d009..00000000000
--- a/test/metabase/sql_parsing/pool_test.clj
+++ /dev/null
@@ -1,161 +0,0 @@
-(ns metabase.sql-parsing.pool-test
- (:require
- [clojure.test :refer :all]
- [metabase.sql-parsing.pool :as pool])
- (:import
- (io.aleph.dirigiste Pool)
- (java.util.concurrent CountDownLatch)))
-
-(set! *warn-on-reflection* true)
-
-(defn- mock-context
- "Create a mock 'context' object. Since Context is final, we just return a map with metadata.
- The pool doesn't care what type the objects are - it just stores and returns them."
- [id]
- {:context-id id
- :created-at (System/currentTimeMillis)})
-
-(defn- counting-generator
- "Returns a generator function that counts how many contexts it has created."
- [counter-atom]
- (fn []
- (let [id (swap! counter-atom inc)]
- (mock-context id))))
-
-(defn- failing-generator
- "Returns a generator that fails after creating n contexts."
- [counter-atom fail-after]
- (fn []
- (let [id (swap! counter-atom inc)]
- (when (> id fail-after)
- (throw (ex-info "Generator limit reached" {:count id :limit fail-after})))
- (mock-context id))))
-
-;;; ------------------------------------------------- Test Helpers ---------------------------------------------------
-
-(defn- test-pool
- "Create a pool for testing with sensible defaults.
- Returns [pool created-count-atom]."
- ([]
- (test-pool {}))
- ([config]
- (let [counter (atom 0)
- generator (counting-generator counter)
- pool (#'pool/make-python-context-pool generator config)]
- [pool counter])))
-
-(defn- with-pool
- "Execute f with a context from pool. Uses with-open for proper resource management."
- [pool f]
- (with-open [^java.io.Closeable ctx (#'pool/acquire-context pool)]
- (f ctx)))
-
-(defn- run-concurrent
- "Run n threads concurrently, each executing f with a context from pool.
- Returns vector of results."
- [pool n f]
- (let [latch (CountDownLatch. n)
- results (atom [])
- futures (doall
- (for [i (range n)]
- (future
- (.countDown latch)
- (.await latch)
- (let [result (with-pool pool (fn [ctx] (f i ctx)))]
- (swap! results conj result)))))]
- (doseq [fut futures] @fut)
- @results))
-
-;;; ------------------------------------------------ Pool Size Tests -------------------------------------------------
-
-(deftest pool-respects-maximum-size-test
- (testing "Pool never creates more than max contexts even under concurrent load"
- (let [[pool created-count] (test-pool {:max-size 3})
- num-threads 20
- results (run-concurrent pool num-threads
- (fn [_i ctx]
- (Thread/sleep (long (+ 10 (rand-int 50))))
- ctx))]
- (is (<= @created-count 3)
- (str "Pool created " @created-count " contexts but max is 3"))
- (is (= num-threads (count results))
- "All threads should have received a context"))))
-
-(deftest pool-maintains-minimum-size-test
- (testing "Pool keeps at least 1 context available"
- (let [[pool created-count] (test-pool)]
- (with-pool pool (fn [_ctx] (is (= 1 @created-count) "First use creates one context")))
- (Thread/sleep (long 100))
- (with-pool pool (fn [_ctx] (is (= 1 @created-count) "Second use reuses the same context"))))))
-
-;;; ----------------------------------------------- Context Expiry Tests ---------------------------------------------
-
-(deftest pool-expires-old-contexts-test
- (testing "Pool disposes of contexts that have exceeded their TTL"
- (let [[pool _created-count] (test-pool {:ttl-minutes 10})
- [context expiry-ts :as tuple] (.acquire ^Pool pool :python)]
- (try
- (is (some? context) "Context should be created")
- (is (number? expiry-ts) "Expiry timestamp should be set")
- (is (< (System/nanoTime) expiry-ts) "Expiry should be in the future")
- (let [expiry-in-minutes (/ (- expiry-ts (System/nanoTime)) (* 1000000000 60))]
- (is (< 9 expiry-in-minutes 11) "Expiry should be approximately 10 minutes"))
- (finally
- (.release ^Pool pool :python tuple))))))
-
-(deftest with-pooled-context-handles-expired-contexts-test
- (testing "with-pooled-context properly handles and replaces expired contexts"
- (let [[pool created-count] (test-pool)]
- (with-pool pool (fn [_ctx] (is (= 1 @created-count))))
- ;; Manually expire by disposing
- (let [tuple (.acquire ^Pool pool :python)]
- (.dispose ^Pool pool :python tuple))
- (with-pool pool (fn [_ctx] (is (= 2 @created-count) "Expired context should be replaced"))))))
-
-;;; --------------------------------------------- Generator Failure Tests --------------------------------------------
-
-(deftest pool-handles-generator-failures-test
- (testing "Pool gracefully handles when generator fails"
- (let [counter (atom 0)
- generator (failing-generator counter 1)
- pool (#'pool/make-python-context-pool generator)]
- (is (some? (with-pool pool identity)) "First context creation succeeds")
- ;; Dispose to force new creation
- (let [tuple (.acquire ^Pool pool :python)]
- (.dispose ^Pool pool :python tuple))
- (is (thrown? Exception (with-pool pool identity)) "Generator failure should propagate"))))
-
-;;; ------------------------------------------- Concurrent Access Tests ----------------------------------------------
-
-(deftest pool-handles-concurrent-access-safely-test
- (testing "Pool handles concurrent access without race conditions"
- (let [[pool created-count] (test-pool {:max-size 3})
- num-threads 50
- results (run-concurrent pool num-threads
- (fn [i _ctx]
- (Thread/sleep (long (rand-int 20)))
- i))]
- (is (= num-threads (count results)) "All threads should complete successfully")
- (is (<= @created-count 3) (str "Created " @created-count " contexts, expected <= 3")))))
-
-;;; -------------------------------------------- Pool Behavior Tests -------------------------------------------------
-
-(deftest pool-stores-and-returns-objects-test
- (testing "Pool correctly stores and returns arbitrary objects"
- (let [[pool created-count] (test-pool)
- result1 (with-pool pool :context)
- result2 (with-pool pool :context)]
- (is (map? result1))
- (is (= 1 (:context-id result1)))
- (is (= 1 @created-count))
- (is (map? result2))
- (is (= 1 (:context-id result2))) ;; Same context ID
- (is (= 1 @created-count)))))
-
-;;; ----------------------------------------- Version Parsing --------------------------------------------------------
-
-(deftest expected-sqlglot-version-test
- (testing "Sqlglot version is properly parsed from pyproject.toml"
- (let [version (#'pool/expected-sqlglot-version)]
- (is (string? version))
- (is (re-matches #"\d+\.\d+\.\d+" version)))))
diff --git a/test/metabase/sql_parsing/python_test.clj b/test/metabase/sql_parsing/python_test.clj
new file mode 100644
index 00000000000..2cebd795ecf
--- /dev/null
+++ b/test/metabase/sql_parsing/python_test.clj
@@ -0,0 +1,45 @@
+(ns metabase.sql-parsing.python-test
+ "Tests for the `:python` sqlglot parser: the same assertions the graal parser is trusted for, through
+ the external-process path. Requires a `python3` binary on the PATH (sqlglot itself is installed into
+ `resources/python-sources` on first use, like the graal tests)."
+ (:require
+ [clojure.test :refer :all]
+ [metabase.sql-parsing.core :as sql-parsing]
+ [metabase.sql-parsing.protocol :as protocol]
+ [metabase.sql-parsing.python :as python]))
+
+(set! *warn-on-reflection* true)
+
+(deftest referenced-tables-test
+ (testing "extracts table references through the external CPython process"
+ (is (= [[nil nil "transactions"] [nil "public" "orders"]]
+ (protocol/referenced-tables
+ (python/parser)
+ "postgres"
+ "SELECT * FROM transactions t JOIN public.orders o ON t.id = o.tid")))))
+
+(deftest validate-query-test
+ (testing "validates syntax through the external CPython process"
+ (is (= "ok"
+ (:status (protocol/validate-query (python/parser) "postgres" "SELECT 1" nil nil))))
+ (is (= "error"
+ (:status (protocol/validate-query (python/parser) "postgres" "SELECT FROM WHERE" nil nil))))))
+
+(deftest call-error-keeps-process-healthy-test
+ (testing "a failed call surfaces the Python error and the process still answers the next call"
+ (is (thrown-with-msg? Exception #"sqlglot call failed"
+ (protocol/add-into-clause (python/parser) nil nil "t")))
+ (is (= [[nil nil "users"]]
+ (protocol/referenced-tables (python/parser) "postgres" "SELECT * FROM users")))))
+
+(deftest parse-error-test
+ (testing "unparseable SQL surfaces as the same transport-agnostic parse error the graal parser throws"
+ (let [e (try
+ (protocol/referenced-tables (python/parser) "postgres" "SELECT !!!")
+ (catch Exception e e))]
+ (is (sql-parsing/parse-error? e))))
+ (testing "other Python-side failures are not parse errors"
+ (let [e (try
+ (protocol/add-into-clause (python/parser) nil nil "t")
+ (catch Exception e e))]
+ (is (not (sql-parsing/parse-error? e))))))