The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
mb-sql-parsing-mode=python (#77558)
Alexander Polyankin97f82c0792cb
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 78f34ceaa2a..dc3ff3dc840 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2183,9 +2183,8 @@sql-parsing{:team "DevEx"- :api #{metabase.sql-parsing.core}- :uses #{analytics-interface- util}}+ :api #{metabase.sql-parsing.core metabase.sql-parsing.init}+ :uses #{analytics-interface settings util}}sql-tools{:team "DevEx"diff --git a/bin/docker/Dockerfile b/bin/docker/Dockerfileindex f6dd3296219..a741764aeb1 100644--- a/bin/docker/Dockerfile+++ b/bin/docker/Dockerfile@@ -9,7 +9,7 @@ ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA}COPY DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem# dependencies-RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts nodejs && \+RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts nodejs python3 && \apk upgrade && \rm -rf /var/cache/apk/* && \mkdir -p /app/certs && \diff --git a/bin/docker/Dockerfile_ubuntu b/bin/docker/Dockerfile_ubuntuindex 3fa3cae342b..56c1513c4ed 100644--- a/bin/docker/Dockerfile_ubuntu+++ b/bin/docker/Dockerfile_ubuntu@@ -11,7 +11,7 @@ COPY DigiCertGlobalRootG2.crt.pem /usr/local/share/ca-certificates/DigiCertGloba# DependenciesRUN apt-get update && \apt-get upgrade -y && \- apt-get install -y ca-certificates ca-certificates-java fonts-noto nodejs && \+ apt-get install -y ca-certificates ca-certificates-java fonts-noto nodejs python3 && \apt-get clean && \curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /usr/local/share/ca-certificates/rds-combined-ca-bundle.pem && \update-ca-certificates && \diff --git a/bin/docker/run_metabase.sh b/bin/docker/run_metabase.shindex 0a03b56c6bf..a7b4bfa8e3d 100755--- a/bin/docker/run_metabase.sh+++ b/bin/docker/run_metabase.sh@@ -4,6 +4,16 @@ if [ -z "$MB_JETTY_HOST" ]; thenexport MB_JETTY_HOST=0.0.0.0fi+# the image ships python3 and node, so default to the external-process backends (lighter and faster+# than the in-process GraalVM ones) unless explicitly configured otherwise+if [ -z "$MB_SQL_PARSING_MODE" ]; then+ export MB_SQL_PARSING_MODE=python+fi++if [ -z "$MB_STATIC_VIZ_MODE" ]; then+ export MB_STATIC_VIZ_MODE=node+fi+# Setup Java OptionsJAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions"JAVA_OPTS="${JAVA_OPTS} -Dfile.encoding=UTF-8"diff --git a/resources/python-sources/sql_tools_cli.py b/resources/python-sources/sql_tools_cli.pynew file mode 100644index 00000000000..406894a7f25--- /dev/null+++ b/resources/python-sources/sql_tools_cli.py@@ -0,0 +1,43 @@+"""Newline-delimited JSON stdin/stdout server around sql_tools, for running sqlglot on a native+CPython instead of GraalPy. Spawned by metabase.sql-parsing.python with this directory (or a zip of+it) on PYTHONPATH; prints one ready line, then answers one response line per request line:++ {"fn": "referenced_tables", "args": ["SELECT 1", "postgres"]}+ -> {"ok": true, "result": "..."} | {"ok": false, "error": "..."}+"""++import json+import sys++import sql_tools+++def _resolve(fn_name):+ if fn_name.startswith("_"):+ raise ValueError(f"unknown sql_tools function: {fn_name}")+ fn = getattr(sql_tools, fn_name, None)+ if not callable(fn):+ raise ValueError(f"unknown sql_tools function: {fn_name}")+ return fn+++def main():+ out = sys.stdout+ out.write(json.dumps({"ready": True}) + "\n")+ out.flush()+ for line in sys.stdin:+ line = line.strip()+ if not line:+ continue+ try:+ request = json.loads(line)+ result = _resolve(request["fn"])(*request["args"])+ response = {"ok": True, "result": result}+ except Exception as e: # noqa: BLE001 -- any failure is reported to the JVM side+ response = {"ok": False, "error": f"{type(e).__name__}: {e}"}+ out.write(json.dumps(response) + "\n")+ out.flush()+++if __name__ == "__main__":+ main()diff --git a/src/metabase/channel/render/js/common.clj b/src/metabase/channel/render/js/common.cljindex 96ecf96f5b4..810ccfaeae9 100644--- a/src/metabase/channel/render/js/common.clj+++ b/src/metabase/channel/render/js/common.clj@@ -28,14 +28,14 @@"Build a dirigiste `Pool` of static-viz workers, each held exclusively per render (so at most `:max-size`renders run at once). `:max-size` is the maximum number of concurrent workers. The utilizationcontroller targets 100% utilization with a min of 0, so when nothing is rendering the pool shrinks to 0- and `destroy` is called on each idle worker; it rechecks every 1 minute, so a worker lingers up to 1- minute before being reaped (keeping it warm through gaps between renders). `(generate)` mints a worker;- `(destroy worker)` tears one down. The other constructor args (queue size, sampling interval) don't- matter much."- ^Pool [generate destroy {:keys [max-size]}]+ and `destroy` is called on each idle worker; it rechecks every `:idle-minutes`, so a worker+ lingers up to that long before being reaped (keeping it warm through gaps between renders). `(generate)`+ mints a worker; `(destroy worker)` tears one down. The other constructor args (queue size, sampling+ interval) don't matter much."+ ^Pool [generate destroy {:keys [max-size idle-minutes]}](let [max-queued-acquires 65000sample-period-ms (.toMillis TimeUnit/MILLISECONDS 25)- control-period-ms (.toMillis TimeUnit/MINUTES 1)]+ control-period-ms (.toMillis TimeUnit/MINUTES idle-minutes)](Pool. (reify IPool$Generator(generate [_ _] (generate))(destroy [_ _ worker] (destroy worker)))diff --git a/src/metabase/channel/render/js/graal.clj b/src/metabase/channel/render/js/graal.cljindex d3efb9abea6..fcd0b324d72 100644--- a/src/metabase/channel/render/js/graal.clj+++ b/src/metabase/channel/render/js/graal.clj@@ -1,6 +1,6 @@(ns metabase.channel.render.js.graal"The GraalVM [[metabase.channel.render.js.protocol/StaticVizRenderer]]: runs the static-viz JS- in-process on a pool of sandboxed GraalVM contexts (up to two by default).+ in-process on a pool of sandboxed GraalVM contexts (up to three by default).We run the JS interpreted (no Graal compiler on a stock JDK) and silence the interpreter warning withthe engine-level `engine.WarnInterpreterOnly` option. See@@ -150,11 +150,12 @@:static-viz)(def ^:private ^Pool static-viz-context-pool- "A pool of up to two static-viz contexts, each held exclusively from acquire to release, so at most two- renders run at once — one per context, on the shared engine. When idle the pool shrinks to 0 and the- generator's `destroy` closes the context (and, on the last one, the shared engine); the first render- after an idle gap rebuilds them. See [[metabase.channel.render.js.common/create-pool]]."- (common/create-pool generate-context! destroy-context! {:max-size 2}))+ "A pool of up to three static-viz contexts, each held exclusively from acquire to release, so at most+ three renders run at once — one per context, on the shared engine. When idle for up to 10 minutes the+ pool shrinks to 0 and the generator's `destroy` closes the context (and, on the last one, the shared+ engine); the first render after an idle gap rebuilds them. See+ [[metabase.channel.render.js.common/create-pool]]."+ (common/create-pool generate-context! destroy-context! {:max-size 3, :idle-minutes 10}))(defn- do-with-static-viz-context"Borrow a pooled static-viz context and call `f` with it, held exclusively for the call (never let it —diff --git a/src/metabase/channel/render/js/node.clj b/src/metabase/channel/render/js/node.cljindex 29ed97422d9..021855fecdd 100644--- a/src/metabase/channel/render/js/node.clj+++ b/src/metabase/channel/render/js/node.clj@@ -143,7 +143,7 @@it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because awarm process is fast and spawning another is costly. See[[metabase.channel.render.js.common/create-pool]]."- (common/create-pool start-process! stop-process! {:max-size 1}))+ (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 1}))(defn- call-node"Run static-viz bundle function `fn-name` with `arg` (a Clojure data structure) on a pooled Node process,diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.cljindex 5d7a3752330..c50465fd643 100644--- a/src/metabase/core/init.clj+++ b/src/metabase/core/init.clj@@ -64,6 +64,7 @@[metabase.settings.init][metabase.setup.init][metabase.slackbot.init]+ [metabase.sql-parsing.init][metabase.sql-tools.init][metabase.sso.init][metabase.store-api.init]diff --git a/src/metabase/sql_parsing/common.clj b/src/metabase/sql_parsing/common.cljindex 428f92cc419..faf2cd0383f 100644--- a/src/metabase/sql_parsing/common.clj+++ b/src/metabase/sql_parsing/common.clj@@ -1,8 +1,180 @@(ns metabase.sql-parsing.common+ "Shared helpers for the pooled sqlglot parsers ([[metabase.sql-parsing.graal]] and+ [[metabase.sql-parsing.python]]): the python-sources locations, the lazy dev sqlglot install, the+ protocol reify over a transport function, and the dirigiste worker-pool factory (copied from+ [[metabase.channel.render.js.common]])."(:require- [potemkin :as p]))+ [clojure.java.io :as io]+ [clojure.java.shell :as shell]+ [metabase.sql-parsing.protocol :as protocol]+ [metabase.util.json :as json]+ [metabase.util.log :as log])+ (:import+ (io.aleph.dirigiste IPool$Generator Pool Pools)+ (java.io File)+ (java.util.concurrent TimeUnit)))-(p/defprotocol+ PythonEval- "Protocol for evaluating Python code. Abstracts over raw Context and pooled contexts."- (eval-python [this code]- "Evaluate Python code."))+(set! *warn-on-reflection* true)++;;; ---------------------------------------- Python path resolution ---------------------------------------++(def python-sources-resource+ "Resource path for Python sources (sql_tools.py and sqlglot)."+ "python-sources")++(defn jar-resource?+ "True if the given resource is inside a JAR (production), false if from filesystem (dev)."+ [resource]+ (boolean+ (when-let [url (io/resource resource)]+ (.contains (.getFile ^java.net.URL url) ".jar!/"))))++;;; ---------------------------------------- Dev lazy installation ----------------------------------------++(def dev-python-sources-dir+ "Directory where sqlglot is installed in dev mode."+ "resources/python-sources")++(defn- delete-recursive!+ "Recursively delete a directory and all its contents."+ [^File f]+ (when (.isDirectory f)+ (doseq [child (.listFiles f)]+ (delete-recursive! child)))+ (.delete f))++(defn expected-sqlglot-version+ "Read the expected sqlglot version from pyproject.toml resource file."+ []+ (some->> (io/resource "python-sources/pyproject.toml")+ slurp+ (re-find #"\"sqlglot==([^\"]+)\"")+ second))++(defn- package-installer-available?+ "Check if uv or pip is available for installing Python packages."+ []+ (or (zero? (:exit (shell/sh "which" "uv")))+ (zero? (:exit (shell/sh "which" "pip")))))++(defn- version-installed?+ "Check if sqlglot is installed with the expected version. Verifies both the dist-info metadata+ (needed by uv/pip) and the actual module (needed by Python import)."+ [target-dir expected-version]+ (and (.exists (io/file target-dir (str "sqlglot-" expected-version ".dist-info") "METADATA"))+ (.exists (io/file target-dir "sqlglot" "__init__.py"))))++(defn- install-sqlglot!+ "Install sqlglot via uv (preferred) or pip (fallback).+ Deletes any existing sqlglot directories first to handle version upgrades."+ [target-dir version]+ ;; Delete old versions first (sqlglot/ and any sqlglot-*.dist-info/)+ (doseq [^File f (.listFiles (io/file target-dir))]+ (when (or (= "sqlglot" (.getName f))+ (.startsWith (.getName f) "sqlglot-"))+ (log/info "Removing old sqlglot:" (.getName f))+ (delete-recursive! f)))+ ;; Try uv first (fast), fall back to pip+ (let [pyproject-file (str target-dir "/pyproject.toml")+ uv-result (shell/sh "uv" "pip" "install" "-r" pyproject-file "--target" target-dir "--no-compile" "--reinstall")]+ (if (zero? (:exit uv-result))+ (log/info "sqlglot" version "installed via uv")+ (do+ (log/info "uv not available, trying pip...")+ (let [pkg (str "sqlglot==" version)+ pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile" "--force-reinstall")]+ (when-not (zero? (:exit pip-result))+ (throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"+ "Manual install: uv pip install -r " pyproject-file " --target " target-dir "\n"+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")+ {:uv-error (:err uv-result)+ :pip-error (:err pip-result)+ :version version+ :target-dir target-dir})))+ (log/info "sqlglot" version "installed via pip"))))))++(defn ensure-sqlglot-installed!+ "Ensure sqlglot is installed with correct version. Called lazily on first use in dev.+ If version mismatch or missing, automatically installs the correct version."+ []+ (let [expected-ver (expected-sqlglot-version)]+ (when-not expected-ver+ (throw (ex-info "Missing pyproject.toml or unable to parse sqlglot entry in resources/python-sources/"+ {:resource "python-sources/pyproject.toml"})))+ (when-not (version-installed? dev-python-sources-dir expected-ver)+ (if (package-installer-available?)+ (do+ (log/info "Installing sqlglot" expected-ver "(first use in dev)...")+ (install-sqlglot! dev-python-sources-dir expected-ver))+ (log/warn "sqlglot not installed and no package installer (uv or pip) found."+ "SQL parsing features will fail until sqlglot is installed."+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")))))++;;; ------------------------------------------------- parser ----------------------------------------------++(defn call-failed-ex+ "The uniform exception for a Python-side sqlglot failure, thrown by every transport so callers can+ handle parse errors without knowing which backend ran ([[metabase.sql-parsing.core/parse-error?]]).+ `error` is the Python error rendered as `ErrorType: message`."+ ([error]+ (call-failed-ex error nil))+ ([error cause]+ (ex-info (str "sqlglot call failed: " error)+ {:sql-parsing/error true+ :sql-parsing/python-error-type (some->> error (re-find #"^\w+"))}+ cause)))++(defn make-parser+ "Build a [[metabase.sql-parsing.protocol/SqlParser]] over `call`, a transport function+ `(call fn-name & args) → string` that executes `sql_tools.<fn-name>` with `args`. JSON results are+ decoded into Clojure data; schema and replacement arguments are passed to Python as JSON strings (the+ `sql_tools` functions take them that way — GraalVM polyglot map conversion is unreliable)."+ [call]+ (reify protocol/SqlParser+ (referenced-tables [_ dialect sql]+ (vec (json/decode (call "referenced_tables" sql dialect))))+ (referenced-fields [_ dialect sql]+ (vec (json/decode (call "referenced_fields" sql dialect))))+ (returned-columns-lineage [_ dialect sql default-table-schema sqlglot-schema]+ (vec (json/decode (call "returned_columns_lineage"+ dialect sql default-table-schema (json/encode sqlglot-schema)))))+ (validate-query [_ dialect sql default-table-schema sqlglot-schema]+ ;; nil → "{}" (an empty JSON object), which sql_tools reads as permissive mode; encoding the+ ;; *string* "{}" here would double-encode and make Python treat a string as the schema+ (json/decode+kw (call "validate_query"+ dialect sql default-table-schema (json/encode (or sqlglot-schema {})))))+ (simple-query [_ dialect sql]+ (json/decode+kw (call "simple_query" sql dialect)))+ (add-into-clause [_ dialect sql table-name]+ (call "add_into_clause" sql table-name dialect))+ (field-references [_ dialect sql]+ (json/decode+kw (call "field_references" sql dialect)))+ (replace-names [_ dialect sql replacements]+ (call "replace_names" sql (json/encode replacements) dialect))+ (single-stmt-of-type [_ dialect sql stmt-type]+ (json/decode+kw (call "is_single_stmt_of_type" sql stmt-type dialect)))+ (transpile-sql [_ sql from-dialect to-dialect]+ (json/decode+kw (call "transpile_sql" sql from-dialect to-dialect)))))++;;; -------------------------------------------------- pool -----------------------------------------------++(defn create-pool+ "Build a dirigiste `Pool` of workers, each held exclusively per call (so at most `:max-size`+ calls run at once). `:max-size` is the maximum number of concurrent workers. The utilization+ controller targets 100% utilization with a min of 0, so when nothing is running the pool shrinks to 0+ and `destroy` is called on each idle worker; it rechecks every `:idle-minutes`, so a worker+ lingers up to that long before being reaped (keeping it warm through gaps between calls). `(generate)`+ mints a worker; `(destroy worker)` tears one down. The other constructor args (queue size, sampling+ interval) don't matter much."+ ^Pool [generate destroy {:keys [max-size idle-minutes]}]+ (let [max-queued-acquires 65000+ sample-period-ms (.toMillis TimeUnit/MILLISECONDS 25)+ control-period-ms (.toMillis TimeUnit/MINUTES idle-minutes)]+ (Pool. (reify IPool$Generator+ (generate [_ _] (generate))+ (destroy [_ _ worker] (destroy worker)))+ (Pools/utilizationController 1.0 max-size max-size)+ max-queued-acquires+ sample-period-ms+ control-period-ms+ TimeUnit/MILLISECONDS)))diff --git a/src/metabase/sql_parsing/core.clj b/src/metabase/sql_parsing/core.cljindex 3fc48658d0e..7a1991848fd 100644--- a/src/metabase/sql_parsing/core.clj+++ b/src/metabase/sql_parsing/core.clj@@ -8,63 +8,36 @@(referenced-tables sql dialect) → [[catalog schema table] ...](referenced-fields dialect sql) → [[catalog schema table field] ...](returned-columns-lineage dialect sql schema schema-map) → [[col pure? deps] ...]- (validate-query dialect sql schema schema-map) → {:status :ok} | {:status :error ...}"+ (validate-query dialect sql schema schema-map) → {:status :ok} | {:status :error ...}++ The parsing itself happens behind the [[metabase.sql-parsing.protocol/SqlParser]] protocol (the+ GraalPy or native-CPython implementation, per [[metabase.sql-parsing.parser/parser]]); this namespace+ owns the JVM-side pre- and post-processing around it."(:require[clojure.string :as str][medley.core :as m]- [metabase.analytics-interface.core :as analytics]- [metabase.sql-parsing.common :as common]- [metabase.sql-parsing.pool :as python.pool]+ [metabase.sql-parsing.parser :as parser]+ [metabase.sql-parsing.protocol :as protocol][metabase.util :as u]- [metabase.util.json :as json][metabase.util.log :as log][metabase.util.performance :as perf])(:import- (java.io Closeable)- (java.util.concurrent ExecutionException TimeoutException)- (org.graalvm.polyglot Value)))+ (java.util.concurrent TimeoutException)))(set! *warn-on-reflection* true)-;;; -------------------------------------------------- Timeout handling ----------------------------------------------------(def ^:private ^:const default-timeout-ms- "Default timeout for Python operations in milliseconds.- GraalVM Python can occasionally hang (DEV-1393), so we wrap calls with a timeout."- 30000) ; 30 seconds--(defn- with-timeout*- "Execute f in a future with timeout. On timeout, throws TimeoutException.- The caller is responsible for cleaning up resources (e.g., disposing context)."- [timeout-ms f]- (let [fut (future (f))]- (try- (deref fut timeout-ms ::timeout)- (catch ExecutionException e- ;; Unwrap execution exception to get the real cause- (throw (or (.getCause e) e)))- (finally- ;; If we timed out or got an exception, try to cancel the future- ;; Note: This won't actually interrupt GraalVM, but prevents resource leaks- (future-cancel fut)))))--(defmacro ^:private with-python-timeout- "Execute body with a timeout. If timeout is reached:- 1. Interrupts the GraalVM context (actually stops execution)- 2. Poisons context so it gets disposed rather than returned to pool- 3. Logs warning and throws TimeoutException"- [ctx timeout-ms & body]- `(let [result# (with-timeout* ~timeout-ms (^:once fn* [] ~@body))]- (if (= result# ::timeout)- (do- ;; Actually interrupt the GraalVM context (1s grace period for soft interrupt)- ;; This is necessary because future-cancel doesn't stop GraalVM execution- (python.pool/interrupt! ~ctx 1000)- (python.pool/poison! ~ctx)- (analytics/inc! :metabase-sql-parsing/context-timeouts)- (log/warn "Python execution timed out after" ~timeout-ms "ms - GraalVM interrupted")- (throw (TimeoutException. (str "Python execution timed out after " ~timeout-ms "ms"))))- result#)))+(defn- parser+ "The [[metabase.sql-parsing.protocol/SqlParser]] implementation for the configured mode."+ []+ (parser/parser))++(defn parse-error?+ "True if `e` is a sqlglot ParseError — the SQL could not be parsed — regardless of which parser+ backend threw it. Other Python-side failures (and non-sqlglot exceptions) are false."+ [e]+ (let [data (ex-data e)]+ (boolean (and (:sql-parsing/error data)+ (= "ParseError" (:sql-parsing/python-error-type data))))));;; ----------------------------------------- VALUES clause stripping ------------------------------------------@@ -214,14 +187,7 @@This is the pure parsing layer - it returns what's literally in the SQL.Default schema resolution happens in the matching layer (core.clj)."[dialect sql]- (let [sql (strip-large-values sql)]- (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.referenced_tables")- (.execute ^Value (object-array [sql dialect]))- .asString)))- json/decode- vec)))+ (protocol/referenced-tables (parser) dialect (strip-large-values sql)))(defn referenced-fields"Extract field references from SQL, returning only fields from actual database tables.@@ -247,14 +213,7 @@(referenced-fields \"bigquery\" \"SELECT * FROM myproject.analytics.events\")=> [[\"myproject\" \"analytics\" \"events\" \"*\"]]"[dialect sql]- (let [sql (strip-large-values sql)]- (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.referenced_fields")- (.execute ^Value (object-array [sql dialect]))- .asString)))- json/decode- vec)))+ (protocol/referenced-fields (parser) dialect (strip-large-values sql)))(defn returned-columns-lineage"Extract column lineage from SQL query, showing which output columns depend on which source columns.@@ -274,18 +233,7 @@(returned-columns-lineage \"postgres\" \"SELECT id + 1 as computed FROM users\" nil schema)=> [[\"computed\" false [[[nil \"users\" \"id\"]]]]]"[dialect sql default-table-schema sqlglot-schema]- (let [sql (strip-large-values sql)]- (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues- (-> ^Value (common/eval-python ctx "sql_tools.returned_columns_lineage")- (.execute ^Value (object-array [dialect- sql- default-table-schema- (json/encode sqlglot-schema)]))- .asString)))- json/decode- vec)))+ (protocol/returned-columns-lineage (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))(defn validate-query"Validate a SQL query against a schema using sqlglot's qualify optimizer.@@ -317,14 +265,7 @@- \"invalid_expression\": Syntax/parse error- \"unhandled\": Other errors"[dialect sql default-table-schema & [sqlglot-schema]]- (let [sql (strip-large-values sql)]- (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues- (-> ^Value (common/eval-python ctx "sql_tools.validate_query")- (.execute ^Value (object-array [dialect sql default-table-schema (json/encode (or sqlglot-schema "{}"))]))- .asString)))- json/decode+kw)))+ (protocol/validate-query (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))(defn simple-query?"Check if SQL is a simple SELECT without LIMIT, OFFSET, or CTEs.@@ -346,13 +287,7 @@(simple-query? nil \"SELECT * FROM users LIMIT 10\")=> {:is_simple false :reason \"Contains a LIMIT\"}"[dialect sql]- (let [sql (strip-large-values sql)]- (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.simple_query")- (.execute ^Value (object-array [sql dialect]))- .asString)))- json/decode+kw)))+ (protocol/simple-query (parser) dialect (strip-large-values sql)))(defn add-into-clause"Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.@@ -370,11 +305,7 @@Returns: Modified SQL string with INTO clause"[dialect sql table-name]- (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.add_into_clause")- (.execute ^Value (object-array [sql table-name dialect]))- .asString))))+ (protocol/add-into-clause (parser) dialect sql table-name))(defn- convert-field-type"Convert field type string (snake_case) to keyword (kebab-case)."@@ -487,24 +418,18 @@On timeout, returns an error map instead of throwing, consistent with the'fail soft' pattern used for parsing failures."[dialect sql]- (let [sql (strip-large-values sql)]- (try- (let [raw (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.field_references")- (.execute ^Value (object-array [sql dialect]))- .asString)))- json/decode+kw)- used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])- returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])- errors (or (:errors raw) (get raw "errors") [])]- {:used-fields (set (map convert-field used-fields))- :returned-fields (vec (map convert-field returned-fields))- :errors (set (map convert-error errors))})- (catch TimeoutException e- {:used-fields #{}- :returned-fields []- :errors #{{:type :timeout :message (.getMessage e)}}}))))+ (try+ (let [raw (protocol/field-references (parser) dialect (strip-large-values sql))+ used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])+ returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])+ errors (or (:errors raw) (get raw "errors") [])]+ {:used-fields (set (map convert-field used-fields))+ :returned-fields (vec (map convert-field returned-fields))+ :errors (set (map convert-error errors))})+ (catch TimeoutException e+ {:used-fields #{}+ :returned-fields []+ :errors #{{:type :timeout :message (.getMessage e)}}})))(defn replace-names"Replace schema, table, and column names in SQL.@@ -527,23 +452,14 @@(replace-names \"postgres\" \"SELECT * FROM people\" {:tables [[{:table \"people\"} \"users\"]]})=> \"SELECT * FROM users\""[dialect sql replacements]- (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.replace_names")- (.execute ^Value (object-array [sql (json/encode replacements) dialect]))- .asString))))+ (protocol/replace-names (parser) dialect sql replacements))(defn is-single-stmt-of-type?"Validates that a query is a single read statement (SELECT) or a single write statement (INSERT, UPDATE, DELETE)and returns the query reconstructed from the parsed AST."[dialect sql stmt-type](let [stripped-sql (strip-large-values sql)- result (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.is_single_stmt_of_type")- (.execute ^Value (object-array [stripped-sql stmt-type dialect]))- .asString)))- json/decode+kw+ result (-> (protocol/single-stmt-of-type (parser) dialect stripped-sql stmt-type)(perf/update-keys (comp keyword u/->kebab-case-en)))];; The `:sql` in the `result` is the reconstructed SQL from the SQLGlot parser.;; We generally want to use the reconstructed SQL, but if the original SQL had its values stripped (to avoid GraalPy OOM)@@ -554,16 +470,15 @@(comment(referenced-tables "postgres" "select * from transactions")- (validate-sql-query "postgres" "SELECT * FROM users")+ (validate-query "postgres" "SELECT * FROM users" nil)(referenced-fields "postgres" "SELECT id, name FROM users WHERE active = true"));;;; Transpile sql(defn- normalize-transpilation-result- [json-str]- (-> json-str- json/decode+kw+ [result]+ (-> result(perf/update-keys (comp keyword u/->kebab-case-en))(m/update-existing :status (comp keyword u/->kebab-case-en))(m/update-existing :reason (comp keyword u/->kebab-case-en))))@@ -571,9 +486,5 @@(defn transpile-sql"Transpiles sql string from one dialect to another."[sql from-dialect to-dialect]- (-> (with-open [^Closeable ctx (python.pool/python-context)]- (with-python-timeout ctx default-timeout-ms- (-> ^Value (common/eval-python ctx "sql_tools.transpile_sql")- (.execute ^Value (object-array [sql from-dialect to-dialect]))- .asString)))- normalize-transpilation-result))+ (normalize-transpilation-result+ (protocol/transpile-sql (parser) sql from-dialect to-dialect)))diff --git a/src/metabase/sql_parsing/graal.clj b/src/metabase/sql_parsing/graal.cljnew file mode 100644index 00000000000..189fdb2d1a2--- /dev/null+++ b/src/metabase/sql_parsing/graal.clj@@ -0,0 +1,269 @@+(ns metabase.sql-parsing.graal+ "The `:graalvm` [[metabase.sql-parsing.protocol/SqlParser]]: runs sqlglot on GraalPy in-process on a+ pool of sandboxed GraalVM contexts (up to three by default).++ The pooled contexts share one `Engine` and one parsed bootstrap `Source`: the engine (and its code+ cache, which holds the parsed sqlglot modules) is created with the first context and closed with the+ last, and each context evaluates the shared source into its own realm. A context is held exclusively+ per call (so calls serialize per context); the utilization controller has min 0, so when idle the pool+ shrinks to 0 and the last `destroy` closes the engine (GraalVM reclaims neither context nor engine on+ GC, and GraalPy contexts hold significant native memory). The first call after an idle gap rebuilds+ them.++ GraalPy can occasionally hang (DEV-1393), so every call runs under a timeout; a timed-out context is+ interrupted and disposed from the pool instead of released."+ (:require+ [metabase.analytics-interface.core :as analytics]+ [metabase.sql-parsing.common :as common]+ [metabase.util.files :as u.files]+ [metabase.util.log :as log])+ (:import+ (io.aleph.dirigiste Pool)+ (java.net URI)+ (java.nio.channels SeekableByteChannel)+ (java.nio.file AccessMode DirectoryStream DirectoryStream$Filter Files FileSystems Path)+ (java.time Duration)+ (java.util Map Set)+ (java.util.concurrent ExecutionException TimeoutException)+ (org.graalvm.polyglot Context Engine HostAccess PolyglotException Source Value)+ (org.graalvm.polyglot.io FileSystem IOAccess)))++(set! *warn-on-reflection* true)++;;; ------------------------------------------ Python filesystem ------------------------------------------++(defn- nio-polyglot-fs+ "Wrap a NIO `java.nio.file.FileSystem` as a polyglot `FileSystem`, routing reads to+ `Files/newByteChannel` instead of the provider's `newFileChannel`.++ The stock wrapper triggers ZipFileSystem to extract deflated entries to a temp file beside the jar,+ which fails when the jar's parent directory isn't writable (e.g. non-root in a Kubernetes pod).+ See https://github.com/metabase/metabase/issues/73541."+ ^FileSystem [^java.nio.file.FileSystem nio-fs]+ (let [provider (.provider nio-fs)]+ (reify FileSystem+ (^Path parsePath [_ ^URI uri]+ (.getPath provider uri))+ (^Path parsePath [_ ^String s]+ (.getPath nio-fs s (into-array String [])))+ (^void checkAccess [_ ^Path p ^Set modes ^"[Ljava.nio.file.LinkOption;" _opts]+ ;; LinkOption opts are dropped: the only value is NOFOLLOW_LINKS, and neither our jar zip FS+ ;; nor resources/python-sources contains symlinks.+ (.checkAccess provider p ^"[Ljava.nio.file.AccessMode;" (into-array AccessMode modes)))+ (^void createDirectory [_ ^Path p ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]+ (Files/createDirectory p attrs))+ (^void delete [_ ^Path p]+ (Files/delete p))+ (^SeekableByteChannel newByteChannel+ [_ ^Path p ^Set opts ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]+ (Files/newByteChannel p opts attrs))+ (^DirectoryStream newDirectoryStream [_ ^Path dir ^DirectoryStream$Filter filter]+ (Files/newDirectoryStream dir filter))+ (^Path toAbsolutePath [_ ^Path p]+ (.toAbsolutePath p))+ (^Path toRealPath [_ ^Path p ^"[Ljava.nio.file.LinkOption;" opts]+ (.toRealPath p opts))+ (^Map readAttributes [_ ^Path p ^String attrs ^"[Ljava.nio.file.LinkOption;" opts]+ (Files/readAttributes p attrs opts)))))++(defn- read-only-polyglot-fs+ "Wrap an NIO `java.nio.file.FileSystem` as a read-only polyglot FileSystem suitable for GraalPy."+ ^FileSystem [^java.nio.file.FileSystem nio-fs]+ (-> nio-fs nio-polyglot-fs FileSystem/newReadOnlyFileSystem))++(defonce ^:private+ ^{:doc "A read-only polyglot FileSystem and the PythonPath within it.+ In dev: wraps the default filesystem, path is resources/python-sources.+ In jar: wraps the jar's zip filesystem, path is /python-sources.+ Either way, the filesystem is read-only so extracted sources cannot be tampered with."}+ python-fs-and-path+ (delay+ (if (common/jar-resource? common/python-sources-resource)+ ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib are both inside the+ ;; jar, so nothing is extracted to disk and there's nothing to tamper with. The filesystem lives for the+ ;; duration of the process (via defonce + delay) and is shared by all pooled Python contexts.+ {:fs (-> (u.files/get-jar-path) u.files/nio-fs read-only-polyglot-fs)+ :python-path "/python-sources"+ :std-lib-home "/META-INF/resources/libpython"+ :core-home "/META-INF/resources/libgraalpy"}+ ;; In dev: use the real filesystem (read-only wrapper). sqlglot is installed locally.+ (do+ (common/ensure-sqlglot-installed!)+ {:fs (read-only-polyglot-fs (FileSystems/getDefault))+ :python-path common/dev-python-sources-dir}))))++;;; ---------------------------------------- shared engine + contexts -------------------------------------++(def ^:private engine-lock (Object.))++(def ^:private shared-engine+ "Atom holding `{:engine <Engine>, :source <Source>, :refs <live context count>}`, or nil when no context+ is live. Guarded by [[engine-lock]]. The engine and parsed bootstrap source are shared by every pooled+ context; they're created with the first context ([[acquire-engine!]]) and closed with the last+ ([[release-engine!]])."+ (atom nil))++(defn- create-engine+ "Build a Python `Engine`. We run interpreted (no Graal compiler on a stock JDK), so+ `engine.WarnInterpreterOnly` is silenced here (it's an engine-level option)."+ ^Engine []+ (.. (Engine/newBuilder)+ (option "engine.WarnInterpreterOnly" "false")+ (build)))++(defn- bootstrap-source+ "Build the bootstrap `Source` evaluated into every context: importing sql_tools loads it (and,+ transitively, sqlglot) into the shared engine's code cache."+ ^Source []+ (.buildLiteral (Source/newBuilder "python" "import sql_tools" "sql-tools-bootstrap")))++(defn- acquire-engine!+ "Return the shared `{:engine, :source}`, creating them with the first context. Bumps the ref count."+ []+ (locking engine-lock+ (let [state (or @shared-engine+ {:engine (create-engine)+ :source (bootstrap-source)+ :refs 0})]+ (reset! shared-engine (update state :refs inc))+ state)))++(defn- release-engine!+ "Drop a ref on the shared engine, closing it once the last context is gone."+ []+ (locking engine-lock+ (let [{:keys [^Engine engine refs]} @shared-engine]+ (if (<= refs 1)+ (do (try (.close engine) (catch Exception _))+ (reset! shared-engine nil))+ (swap! shared-engine update :refs dec)))))++(defn- create-context+ "Create a sandboxed GraalPy context on `engine` configured for sqlglot: PythonPath pointing at the+ Python sources, host access for string interop, and a read-only filesystem (sources are read directly+ from the classpath/jar; nothing is extracted to disk in prod)."+ ^Context [^Engine engine]+ (let [{:keys [^FileSystem fs python-path std-lib-home core-home]} @python-fs-and-path+ io-access (.. (IOAccess/newBuilder)+ (fileSystem fs)+ (build))+ builder (.. (Context/newBuilder (into-array String ["python"]))+ (engine engine)+ (option "python.PythonPath" python-path)+ (allowHostAccess HostAccess/ALL)+ (allowIO io-access))+ ;; When running from the jar, GraalPy's stdlib is also inside the jar filesystem+ ;; and we need to tell it where to find the core and stdlib paths explicitly.+ builder (cond-> builder+ std-lib-home (.option "python.StdLibHome" std-lib-home)+ core-home (.option "python.CoreHome" core-home))]+ (.build builder)))++(defn- generate-context!+ "Build a context on the shared engine and evaluate the bootstrap source into it (creating the engine ++ source if this is the first context)."+ ^Context []+ (analytics/inc! :metabase-sql-parsing/context-creations)+ (let [{:keys [^Engine engine ^Source source]} (acquire-engine!)]+ (try+ (doto (create-context engine)+ (.eval source))+ (catch Throwable t+ (release-engine!)+ (throw t)))))++(defn- destroy-context!+ "Close a context and drop its ref on the shared engine (closing the engine if it was the last context)."+ [^Context context]+ (try (.close context true) (catch Exception _))+ (release-engine!))++;;; ------------------------------------------------ context pool -----------------------------------------++(def ^:private pool-key+ "Dirigiste pools are keyed; the key itself is arbitrary, it just has to be the same for every operation."+ :sql-parsing)++(def ^:private ^Pool python-context-pool+ "A pool of up to three GraalPy contexts, each held exclusively from acquire to release, so at most+ three sqlglot calls run at once — one per context, on the shared engine. When idle for up to 10+ minutes the pool shrinks to 0 and the generator's `destroy` closes the context (and, on the last one,+ the shared engine); the first call after an idle gap rebuilds them. See+ [[metabase.sql-parsing.common/create-pool]]."+ (common/create-pool generate-context! destroy-context! {:max-size 3, :idle-minutes 10}))++;;; ---------------------------------------------- timeout handling ---------------------------------------++(def ^:private ^:const call-timeout-ms+ "Timeout for Python calls: GraalPy can occasionally hang (DEV-1393)."+ 30000)++(defn- with-timeout*+ "Execute f in a future; return its result, or `::timeout` after `timeout-ms`."+ [timeout-ms f]+ (let [fut (future (f))]+ (try+ (deref fut timeout-ms ::timeout)+ (catch ExecutionException e+ ;; Unwrap execution exception to get the real cause+ (throw (or (.getCause e) e)))+ (finally+ ;; If we timed out or got an exception, try to cancel the future+ ;; Note: This won't actually interrupt GraalVM, but prevents resource leaks+ (future-cancel fut)))))++(defn- interrupt!+ "Interrupt Python execution in `context`. If the interrupt times out (guest stuck in uninterruptible+ native code), force-close the context — force close cannot be denied by the guest."+ [^Context context ^long timeout-ms]+ (try+ (.interrupt context (Duration/ofMillis timeout-ms))+ (catch TimeoutException _+ (log/warn "GraalVM interrupt timed out, forcing context closure")+ (.close context true))))++(defn- do-with-python-context+ "Borrow a pooled GraalPy context and call `f` with it, held exclusively for the call (never let it — or+ a context-bound `Value` — escape). The call is guarded by a timeout: a timed-out context is+ interrupted and disposed from the pool (the next call generates a fresh one) instead of released."+ [f]+ (let [^Context context (.acquire python-context-pool pool-key)+ poisoned? (volatile! false)]+ (analytics/inc! :metabase-sql-parsing/context-acquisitions)+ (try+ (let [result (with-timeout* call-timeout-ms #(f context))]+ (if (= result ::timeout)+ (do+ (vreset! poisoned? true)+ ;; future-cancel doesn't stop GraalVM execution, so interrupt the context itself+ ;; (1s grace period for the soft interrupt)+ (interrupt! context 1000)+ (analytics/inc! :metabase-sql-parsing/context-timeouts)+ (log/warn "Python execution timed out after" call-timeout-ms "ms - GraalVM interrupted")+ (throw (TimeoutException. (str "Python execution timed out after " call-timeout-ms "ms"))))+ result))+ (finally+ (if @poisoned?+ (.dispose python-context-pool pool-key context)+ (.release python-context-pool pool-key context))))))++;;; ------------------------------------------------- backend ---------------------------------------------++(defn- call-python+ "Execute `sql_tools.<fn-name>` with `args` on a pooled context and return the result as a string.+ Python-side failures are rethrown as the transport-agnostic+ [[metabase.sql-parsing.common/call-failed-ex]]."+ ^String [fn-name & args]+ (do-with-python-context+ (fn [^Context context]+ (try+ (let [^Value fn-ref (.eval context "python" (str "sql_tools." fn-name))]+ (.asString ^Value (.execute fn-ref (object-array args))))+ (catch PolyglotException e+ (throw (common/call-failed-ex (.getMessage e) e)))))))++(defn parser+ "The `:graalvm` [[metabase.sql-parsing.protocol/SqlParser]] — runs sqlglot on GraalPy on the pooled+ contexts."+ []+ (common/make-parser call-python))diff --git a/src/metabase/sql_parsing/init.clj b/src/metabase/sql_parsing/init.cljnew file mode 100644index 00000000000..88bded2543e--- /dev/null+++ b/src/metabase/sql_parsing/init.clj@@ -0,0 +1,3 @@+(ns metabase.sql-parsing.init+ (:require+ [metabase.sql-parsing.settings]))diff --git a/src/metabase/sql_parsing/parser.clj b/src/metabase/sql_parsing/parser.cljnew file mode 100644index 00000000000..cad9dbe356e--- /dev/null+++ b/src/metabase/sql_parsing/parser.clj@@ -0,0 +1,18 @@+(ns metabase.sql-parsing.parser+ "Hands callers the [[metabase.sql-parsing.protocol/SqlParser]] implementation for the configured+ [[metabase.sql-parsing.settings/sql-parsing-mode]]: the in-process GraalPy parser (default) or a pool+ of external native CPython processes. Kept behind the protocol so the caller+ ([[metabase.sql-parsing.core]]) doesn't care which backend runs."+ (:require+ [metabase.sql-parsing.graal :as graal]+ [metabase.sql-parsing.python :as python]+ [metabase.sql-parsing.settings :as settings]))++(set! *warn-on-reflection* true)++(defn parser+ "The sqlglot parser for the configured mode."+ []+ (if (= :python (settings/sql-parsing-mode))+ (python/parser)+ (graal/parser)))diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.cljdeleted file mode 100644index d9790504509..00000000000--- a/src/metabase/sql_parsing/pool.clj+++ /dev/null@@ -1,343 +0,0 @@-(ns metabase.sql-parsing.pool- "Namespace to handle python environment. Has only a single public function, [[python-context]] which will return a- python context. This is an internal detail to the sql-parsing module. If you want something from this, add a- function to the core namespace that uses this. No one should know about this implmentation detail."- (:require- [clojure.java.io :as io]- [clojure.java.shell :as shell]- [metabase.analytics-interface.core :as analytics]- [metabase.sql-parsing.common :as common]- [metabase.util.files :as u.files]- [metabase.util.log :as log]- [potemkin :as p])- (:import- (io.aleph.dirigiste- IPool$Controller- IPool$Generator- Pool- Pools)- (java.io Closeable File)- (java.net URI)- (java.nio.channels SeekableByteChannel)- (java.nio.file AccessMode DirectoryStream DirectoryStream$Filter Files FileSystems Path)- (java.time Duration)- (java.util Map Set)- (java.util.concurrent TimeUnit TimeoutException)- (org.graalvm.polyglot Context Engine HostAccess)- (org.graalvm.polyglot.io FileSystem IOAccess)))--(set! *warn-on-reflection* true)--;;; -------------------------------------------------- Python path resolution ----------------------------------------------------(def ^:private python-sources-resource- "Resource path for Python sources (sql_tools.py and sqlglot)."- "python-sources")--(defn- jar-resource?- "True if the given resource is inside a JAR (production), false if from filesystem (dev)."- [resource]- (boolean- (when-let [url (io/resource resource)]- (.contains (.getFile ^java.net.URL url) ".jar!/"))))--;;; -------------------------------------------------- Dev lazy installation ----------------------------------------------------(def ^:private dev-python-sources-dir- "Directory where sqlglot is installed in dev mode."- "resources/python-sources")--(defn- delete-recursive!- "Recursively delete a directory and all its contents."- [^File f]- (when (.isDirectory f)- (doseq [child (.listFiles f)]- (delete-recursive! child)))- (.delete f))--(defn- expected-sqlglot-version- "Read the expected sqlglot version from pyproject.toml resource file."- []- (some->> (io/resource "python-sources/pyproject.toml")- slurp- (re-find #"\"sqlglot==([^\"]+)\"")- second))--(defn- package-installer-available?- "Check if uv or pip is available for installing Python packages."- []- (or (zero? (:exit (shell/sh "which" "uv")))- (zero? (:exit (shell/sh "which" "pip")))))--(defn- version-installed?- "Check if sqlglot is installed with the expected version. Verifies both the dist-info metadata- (needed by uv/pip) and the actual module (needed by Python import)."- [target-dir expected-version]- (and (.exists (io/file target-dir (str "sqlglot-" expected-version ".dist-info") "METADATA"))- (.exists (io/file target-dir "sqlglot" "__init__.py"))))--(defn- install-sqlglot!- "Install sqlglot via uv (preferred) or pip (fallback).- Deletes any existing sqlglot directories first to handle version upgrades."- [target-dir version]- ;; Delete old versions first (sqlglot/ and any sqlglot-*.dist-info/)- (doseq [^File f (.listFiles (io/file target-dir))]- (when (or (= "sqlglot" (.getName f))- (.startsWith (.getName f) "sqlglot-"))- (log/info "Removing old sqlglot:" (.getName f))- (delete-recursive! f)))- ;; Try uv first (fast), fall back to pip- (let [pyproject-file (str target-dir "/pyproject.toml")- uv-result (shell/sh "uv" "pip" "install" "-r" pyproject-file "--target" target-dir "--no-compile" "--reinstall")]- (if (zero? (:exit uv-result))- (log/info "sqlglot" version "installed via uv")- (do- (log/info "uv not available, trying pip...")- (let [pkg (str "sqlglot==" version)- pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile" "--force-reinstall")]- (when-not (zero? (:exit pip-result))- (throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"- "Manual install: uv pip install -r " pyproject-file " --target " target-dir "\n"- "Install uv: https://docs.astral.sh/uv/getting-started/installation/")- {:uv-error (:err uv-result)- :pip-error (:err pip-result)- :version version- :target-dir target-dir})))- (log/info "sqlglot" version "installed via pip"))))))--(defn- ensure-sqlglot-installed!- "Ensure sqlglot is installed with correct version. Called lazily on first use in dev.- If version mismatch or missing, automatically installs the correct version."- []- (let [expected-ver (expected-sqlglot-version)]- (when-not expected-ver- (throw (ex-info "Missing pyproject.toml or unable to parse sqlglot entry in resources/python-sources/"- {:resource "python-sources/pyproject.toml"})))- (when-not (version-installed? dev-python-sources-dir expected-ver)- (if (package-installer-available?)- (do- (log/info "Installing sqlglot" expected-ver "(first use in dev)...")- (install-sqlglot! dev-python-sources-dir expected-ver))- (log/warn "sqlglot not installed and no package installer (uv or pip) found."- "SQL parsing features will fail until sqlglot is installed."- "Install uv: https://docs.astral.sh/uv/getting-started/installation/")))))--;;; -------------------------------------------------- Python path delay ----------------------------------------------------(defn- nio-polyglot-fs- "Wrap a NIO `java.nio.file.FileSystem` as a polyglot `FileSystem`, routing reads to- `Files/newByteChannel` instead of the provider's `newFileChannel`.-- The stock wrapper triggers ZipFileSystem to extract deflated entries to a temp file beside the jar,- which fails when the jar's parent directory isn't writable (e.g. non-root in a Kubernetes pod).- See https://github.com/metabase/metabase/issues/73541."- ^FileSystem [^java.nio.file.FileSystem nio-fs]- (let [provider (.provider nio-fs)]- (reify FileSystem- (^Path parsePath [_ ^URI uri]- (.getPath provider uri))- (^Path parsePath [_ ^String s]- (.getPath nio-fs s (into-array String [])))- (^void checkAccess [_ ^Path p ^Set modes ^"[Ljava.nio.file.LinkOption;" _opts]- ;; LinkOption opts are dropped: the only value is NOFOLLOW_LINKS, and neither our jar zip FS- ;; nor resources/python-sources contains symlinks.- (.checkAccess provider p ^"[Ljava.nio.file.AccessMode;" (into-array AccessMode modes)))- (^void createDirectory [_ ^Path p ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]- (Files/createDirectory p attrs))- (^void delete [_ ^Path p]- (Files/delete p))- (^SeekableByteChannel newByteChannel- [_ ^Path p ^Set opts ^"[Ljava.nio.file.attribute.FileAttribute;" attrs]- (Files/newByteChannel p opts attrs))- (^DirectoryStream newDirectoryStream [_ ^Path dir ^DirectoryStream$Filter filter]- (Files/newDirectoryStream dir filter))- (^Path toAbsolutePath [_ ^Path p]- (.toAbsolutePath p))- (^Path toRealPath [_ ^Path p ^"[Ljava.nio.file.LinkOption;" opts]- (.toRealPath p opts))- (^Map readAttributes [_ ^Path p ^String attrs ^"[Ljava.nio.file.LinkOption;" opts]- (Files/readAttributes p attrs opts)))))--(defn- read-only-polyglot-fs- "Wrap an NIO `java.nio.file.FileSystem` as a read-only polyglot FileSystem suitable for GraalPy."- ^FileSystem [^java.nio.file.FileSystem nio-fs]- (-> nio-fs nio-polyglot-fs FileSystem/newReadOnlyFileSystem))--(defonce ^:private- ^{:doc "A read-only polyglot FileSystem and the PythonPath within it.- In dev: wraps the default filesystem, path is resources/python-sources.- In jar: wraps the jar's zip filesystem, path is /python-sources.- Either way, the filesystem is read-only so extracted sources cannot be tampered with."}- python-fs-and-path- (delay- (if (jar-resource? python-sources-resource)- ;; In the jar: use the jar's zip filesystem directly. Python sources and GraalPy's stdlib are both inside the- ;; jar, so nothing is extracted to disk and there's nothing to tamper with. The filesystem lives for the- ;; duration of the process (via defonce + delay) and is shared by all pooled Python contexts.- {:fs (-> (u.files/get-jar-path) u.files/nio-fs read-only-polyglot-fs)- :python-path "/python-sources"- :std-lib-home "/META-INF/resources/libpython"- :core-home "/META-INF/resources/libgraalpy"}- ;; In dev: use the real filesystem (read-only wrapper). sqlglot is installed locally.- (do- (ensure-sqlglot-installed!)- {:fs (read-only-polyglot-fs (FileSystems/getDefault))- :python-path dev-python-sources-dir}))))--;;; -------------------------------------------- Context Wrappers ------------------------------------------------------(p/defrecord+ PooledContext [^Context context ^Pool pool tuple poisoned?]- common/PythonEval- (eval-python [_this code]- (.eval context "python" ^String code))-- Closeable- (close [_this]- (if @poisoned?- (do- (log/warn "Disposing poisoned Python context (likely hung GraalVM)")- (.dispose pool :python tuple))- (.release pool :python tuple))))--(defn poison!- "Mark a PooledContext as poisoned. When closed, it will be disposed from the pool- rather than released back. Use this when GraalVM hangs to prevent returning a- broken context to the pool."- [ctx]- (when (instance? PooledContext ctx)- (reset! (:poisoned? ctx) true)))--(defn interrupt!- "Interrupt Python execution in this context. Returns true if interrupted successfully.- If interrupt times out (guest in uninterruptible code), forces context closure with close(true).- This is necessary because future-cancel doesn't actually stop GraalVM execution."- [ctx ^long timeout-ms]- (when-let [^Context raw-ctx (when (instance? PooledContext ctx)- (:context ctx))]- (try- (.interrupt raw-ctx (Duration/ofMillis timeout-ms))- true- (catch TimeoutException _- ;; Interrupt timed out - guest app may be in uninterruptible native code- ;; Force close cannot be denied by guest application- (log/warn "GraalVM interrupt timed out, forcing context closure")- (.close raw-ctx true)- false))))--(defonce ^:private- ^{:doc "A single GraalVM `Engine` shared by every pooled Python context. The engine owns the Truffle runtime and the- (parsed/compiled) code cache; sharing it across contexts means recycling a context on its TTL no longer- recreates an engine or re-loads sqlglot into a fresh code cache. That per-context engine state is the dominant- native-memory cost and, because contexts churn on a 10-minute TTL and GraalPy doesn't reliably return all of- it to the OS on close, a major source of the native-memory growth. The engine is a process-lifetime singleton- (it is intentionally never closed)."}- shared-python-engine- (delay- (.. (Engine/newBuilder)- (option "engine.WarnInterpreterOnly" "false")- (build))))--(defn- create-graalvm-context- "Create a new GraalVM Python context configured for sqlglot.-- The context is configured with:- - The shared [[shared-python-engine]] (so contexts share one Truffle runtime + code cache)- - PythonPath pointing to python-sources (contains sql_tools.py and sqlglot)- - Full host access (needed for JSON serialization)- - Read-only filesystem (sources loaded directly from classpath/jar, no disk extraction in prod)"- ^Context []- (let [{:keys [^FileSystem fs python-path std-lib-home core-home]} @python-fs-and-path- io-access (.. (IOAccess/newBuilder)- (fileSystem fs)- (build))- builder (.. (Context/newBuilder (into-array String ["python"]))- (engine @shared-python-engine)- (option "python.PythonPath" python-path)- (allowHostAccess HostAccess/ALL)- (allowIO io-access))- ;; When running from the jar, GraalPy's stdlib is also inside the jar filesystem- ;; and we need to tell it where to find the core and stdlib paths explicitly.- builder (cond-> builder- std-lib-home (.option "python.StdLibHome" std-lib-home)- core-home (.option "python.CoreHome" core-home))- ctx (.build builder)]- (.eval ctx "python" "import sql_tools")- ctx))--(defn- make-python-context-pool- "Create a pool of Python contexts. Accepts a generator function and optional config map.-- Config options:- - :max-size - Maximum pool size (default: 3)- - :min-size - Minimum pool size (default: 1)- - :ttl-minutes - How long contexts live before expiry (default: 10)- - :utilization - Target utilization (default: 1.0 = 100%)-- This allows for easy testing by injecting mock context generators and custom pool configs."- ([context-generator]- (make-python-context-pool context-generator {}))- ([context-generator {:keys [max-size min-size ttl-minutes utilization]- :or {max-size 3- min-size 1- ttl-minutes 10- utilization 1.0}}]- (let [base-controller (Pools/utilizationController utilization max-size max-size)]- (Pool. (reify IPool$Generator- (generate [_ _]- ;; Generate a tuple of the context and the expiry timestamp.- (analytics/inc! :metabase-sql-parsing/context-creations)- [(context-generator)- (+ (System/nanoTime) (.toNanos TimeUnit/MINUTES ttl-minutes))])- (destroy [_ _ [^Context ctx _expiry]]- ;; Close context when disposed from pool (expiry, poison, or shutdown)- (try- (.close ctx true) ;; Force close - can't wait for running code- (catch Exception _))))- ;; Wrap the utilization controller with a modification that doesn't allow the pool to go below min-size.- (reify IPool$Controller- (shouldIncrement [_ k a b] (.shouldIncrement base-controller k a b))- (adjustment [_ stats]- (let [adj (.adjustment base-controller stats)- ;; :python is arbitrary key, it just has to be consistent everywhere when working with the pool.- n (some-> ^io.aleph.dirigiste.Stats (:python stats) .getNumWorkers)- python-adj (:python adj)]- (if (and n python-adj (<= (+ n python-adj) min-size))- ;; If the adjustment is going to bring the pool below min-size, return empty adjustment instead.- {}- adj))))- 65000 ;; Queue size - doesn't matter much.- 25 ;; Sampling interval - doesn't matter much.- 10000 ;; Recheck every 10 seconds- TimeUnit/MILLISECONDS))))--(def ^:private python-context-pool- "Pool of Python context objects. They are not thread-safe, so access to them has to be carefully managed between- threads. Each context with loaded Python interpreter takes significant memory, so we don't want too many of them.- However, one takes significant time to initialize, so we don't want to load them anew each time in prod. Under some- circumstances, the GraalVM Python context tends to leak memory, so we don't want to keep the reference to the- context forever. Considering all that, this pool targets 100% utilization (so, if the utilization is lower, the pool- will start dropping objects) and the maximum of 3 objects (to prevent OOMs), but at least 1 object will always be in- the pool to pick up. However, together with each context keep its creation timestamp so that we can throwaway- instances that are too old to avoid leaks.-- Wrapped in delay to avoid initialization during AOT compilation."- (delay (make-python-context-pool create-graalvm-context {:max-size 3- :min-size 1- :ttl-minutes 10})))--(defn- acquire-context- "Acquire a context from the pool, handling expiry. Returns a PooledContext."- [^Pool pool]- (loop []- (let [[context expiry-ts :as tuple] (.acquire pool :python)]- (if (>= (System/nanoTime) expiry-ts)- (do (analytics/inc! :metabase-sql-parsing/context-disposals-expired)- (.dispose pool :python tuple)- (recur))- (do (analytics/inc! :metabase-sql-parsing/context-acquisitions)- (->PooledContext context pool tuple (atom false)))))))--(defn python-context- "Acquire a python context from the pool. Must be closed. Use in a `with-open` context."- []- (acquire-context @python-context-pool))diff --git a/src/metabase/sql_parsing/protocol.clj b/src/metabase/sql_parsing/protocol.cljnew file mode 100644index 00000000000..400695d5500--- /dev/null+++ b/src/metabase/sql_parsing/protocol.clj@@ -0,0 +1,39 @@+(ns metabase.sql-parsing.protocol+ "The strongly-typed protocol implemented by the sqlglot SQL parsers (currently+ [[metabase.sql-parsing.graal]]). Callers go through [[metabase.sql-parsing.core]], which owns the+ JVM-side pre/post-processing and invokes these methods on the implementation.")++(set! *warn-on-reflection* true)++(defprotocol SqlParser+ "Dialect-aware SQL parsing via sqlglot. Each method calls the corresponding `sql_tools` Python function+ and returns its result parsed from JSON into Clojure data, except the SQL-rewriting methods+ ([[add-into-clause]], [[replace-names]]) which return the rewritten SQL string. `dialect` is a sqlglot+ dialect string, or nil for sqlglot's default dialect."+ (referenced-tables [parser dialect sql]+ "Table references literally present in the SQL, as a vector of `[catalog schema table]` 3-tuples.")+ (referenced-fields [parser dialect sql]+ "Field references from actual database tables, as a vector of `[catalog schema table field]`+ 4-tuples (wildcards as `\"*\"`).")+ (returned-columns-lineage [parser dialect sql default-table-schema sqlglot-schema]+ "Column lineage of the query's output, as a vector of `[alias pure? deps]` tuples.")+ (validate-query [parser dialect sql default-table-schema sqlglot-schema]+ "Validate the query against `sqlglot-schema` (strict) or its own structure (permissive when the+ schema is nil/empty). Returns a keywordized `{:status \"ok\"}` or `{:status \"error\" ...}` map.")+ (simple-query [parser dialect sql]+ "Whether the SQL is a simple SELECT without LIMIT, OFFSET, or CTEs. Returns a keywordized+ `{:is_simple ...}` map with a `:reason` when false.")+ (add-into-clause [parser dialect sql table-name]+ "Add an `INTO table-name` clause to a SELECT statement (SQL Server SELECT INTO syntax). Returns the+ rewritten SQL string.")+ (field-references [parser dialect sql]+ "Used and returned field specs, as a keywordized map of `:used_fields`, `:returned_fields`, and+ `:errors` in the raw Python shape.")+ (replace-names [parser dialect sql replacements]+ "Replace schema/table/column names in the SQL. Returns the rewritten SQL string. Replacement values+ are injected into the AST unsanitized — callers must ensure they are system-generated.")+ (single-stmt-of-type [parser dialect sql stmt-type]+ "Whether the SQL is a single statement of `stmt-type`, as a keywordized map that includes the SQL+ reconstructed from the parsed AST.")+ (transpile-sql [parser sql from-dialect to-dialect]+ "Transpile the SQL from one dialect to another, as a keywordized result map."))diff --git a/src/metabase/sql_parsing/python.clj b/src/metabase/sql_parsing/python.cljnew file mode 100644index 00000000000..0776da7e92b--- /dev/null+++ b/src/metabase/sql_parsing/python.clj@@ -0,0 +1,199 @@+(ns metabase.sql-parsing.python+ "The `:python` [[metabase.sql-parsing.protocol/SqlParser]]: runs sqlglot in a pool of external native+ CPython child processes rather than in-process on GraalPy. Each process runs `sql_tools_cli.py`, which+ answers newline-delimited JSON requests over stdin/stdout.++ The Python sources come straight out of the classpath: in dev the child imports directly from+ `resources/python-sources`; from the jar they are written to a fresh temp zip at every process spawn+ (CPython imports from zips via zipimport), put on the child's PYTHONPATH, and deleted when the process+ is destroyed — so there is no long-lived on-disk copy to drift out of sync or be tampered with, and no+ deploy-time provisioning.++ The pool holds a single process, held exclusively per call (so calls serialize onto it); with a min of+ 0 it shrinks to 0 when idle, killing the process after 10 minutes with no work. We cap it at one+ process — spawning a fresh `python3` (interpreter init + sqlglot import) is costly, and a warm process+ parses fast enough that queuing a second call behind it beats paying that startup cost for+ concurrency.++ Requires a `python3` binary on the host's PATH."+ (:require+ [clojure.java.io :as io]+ [clojure.string :as str]+ [metabase.sql-parsing.common :as common]+ [metabase.util.files :as u.files]+ [metabase.util.json :as json])+ (:import+ (io.aleph.dirigiste Pool)+ (java.io BufferedReader BufferedWriter File IOException InputStreamReader OutputStreamWriter)+ (java.lang ProcessBuilder$Redirect)+ (java.nio.charset StandardCharsets)+ (java.nio.file FileVisitOption Files LinkOption)+ (java.nio.file.attribute FileAttribute)+ (java.util.concurrent TimeoutException)+ (java.util.zip ZipEntry ZipOutputStream)))++(set! *warn-on-reflection* true)++;;; ------------------------------------------------ process ----------------------------------------------++(def ^:private ^:const call-timeout-ms+ "How long to wait for the process to answer before assuming it has wedged: we kill it and fail the+ call, so a stuck process can't hold its pool slot forever."+ 30000)++(defrecord ^:private PythonProcess [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip])++(defn- read-response-line+ "Read one line from the process, killing it and throwing if it doesn't answer within+ [[call-timeout-ms]] (so a wedged process can't hold its pool slot forever). `destroyForcibly`+ (SIGKILL) rather than the polite `destroy`: a process wedged hard enough to miss the deadline may+ ignore SIGTERM, and killing it also closes its stdout, which unblocks the reader future."+ ^String [^BufferedReader reader ^Process process]+ (let [fut (future (.readLine reader))+ line (deref fut call-timeout-ms ::timeout)]+ (if (identical? ::timeout line)+ (do (.destroyForcibly process)+ (future-cancel fut)+ (throw (TimeoutException. (str "Python execution timed out after " call-timeout-ms "ms"))))+ line)))++(defn- create-sources-zip!+ "Write the python-sources tree from the jar to a fresh temp zip, importable by the child via zipimport,+ to be put on its PYTHONPATH and deleted when its process is destroyed (`deleteOnExit` is only a+ backstop for JVM death)."+ ^File []+ (let [file (.toFile (Files/createTempFile "mb-sql-tools" ".zip" (into-array FileAttribute [])))]+ (.deleteOnExit file)+ (try+ (with-open [jar-fs (u.files/nio-fs (u.files/get-jar-path))+ zos (ZipOutputStream. (io/output-stream file))]+ (let [root (.getPath jar-fs (str "/" common/python-sources-resource) (into-array String []))]+ (with-open [paths (Files/walk root (into-array FileVisitOption []))]+ (doseq [^java.nio.file.Path path (iterator-seq (.iterator paths))+ :when (and (not (Files/isDirectory path (into-array LinkOption [])))+ (not (str/includes? (str path) "__pycache__")))]+ (.putNextEntry zos (ZipEntry. (str (.relativize root path))))+ (Files/copy path zos)+ (.closeEntry zos)))))+ file+ (catch Throwable t+ (.delete file)+ (throw t)))))++(defn- python-path+ "The PYTHONPATH entry for a new child process, plus the temp zip to delete with it (nil in dev, where+ the child imports straight from `resources/python-sources`)."+ []+ (if (common/jar-resource? common/python-sources-resource)+ (let [zip (create-sources-zip!)]+ {:path (.getAbsolutePath zip) :sources-zip zip})+ (do+ (common/ensure-sqlglot-installed!)+ {:path common/dev-python-sources-dir :sources-zip nil})))++(defn- start-process!+ "Spawn `python3 -m sql_tools_cli` with the Python sources on PYTHONPATH and block until it reports it+ has loaded. The process's stderr is inherited (for debugging); stdout carries the line protocol. On+ any failure to reach a ready state the process is destroyed and the temp zip deleted, so nothing+ leaks."+ ^PythonProcess []+ (let [{:keys [path ^File sources-zip]} (python-path)+ ^Process process (try+ (let [builder (.. (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String ["python3" "-m" "sql_tools_cli"]))+ (redirectError ProcessBuilder$Redirect/INHERIT))]+ (doto (.environment builder)+ (.put "PYTHONPATH" path)+ (.put "PYTHONUNBUFFERED" "1"))+ (.start builder))+ (catch Throwable t+ (some-> sources-zip .delete)+ (if (instance? IOException t)+ (throw (ex-info (str "could not start `python3` for SQL parsing — is python3"+ " installed and on PATH? (otherwise set MB_SQL_PARSING_MODE"+ " to graalvm)")+ {} t))+ (throw t))))]+ (try+ (let [writer (BufferedWriter. (OutputStreamWriter. (.getOutputStream process) StandardCharsets/UTF_8))+ reader (BufferedReader. (InputStreamReader. (.getInputStream process) StandardCharsets/UTF_8))+ ready (read-response-line reader process)]+ (when-not (some-> ready json/decode+kw :ready)+ (throw (ex-info "sql-parsing python process failed to start" {:line ready})))+ (->PythonProcess process writer reader sources-zip))+ (catch Throwable t+ (.destroy process)+ (some-> sources-zip .delete)+ (throw t)))))++(defn- stop-process!+ [{:keys [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip]}]+ (try (.close writer) (catch Exception _))+ (try (.close reader) (catch Exception _))+ (try (.destroy process) (catch Exception _))+ (some-> sources-zip .delete))++(defn- call-on-process+ "Send one request to `python-process` and read its response. Returns the `sql_tools` function's result+ string. Not thread-safe on its own — the pool hands a process to one call at a time. A write failure+ (broken pipe) means the pooled process had already died; it is thrown as `:retryable` so [[call-python]]+ can get a fresh one."+ ^String [{:keys [^BufferedWriter writer ^BufferedReader reader ^Process process]} fn-name args]+ (try+ (.write writer ^String (str (json/encode {:fn fn-name :args args}) "\n"))+ (.flush writer)+ (catch IOException e+ (throw (ex-info "sql-parsing python process is not writable" {:retryable true} e))))+ (let [line (read-response-line reader process)]+ (when (nil? line)+ (throw (ex-info "sql-parsing python process closed unexpectedly" {:retryable true})))+ (let [{:keys [ok result error]} (json/decode+kw line)]+ (if ok+ result+ ;; The process answered the protocol correctly, so it is healthy; only this call failed.+ ;; [[call-python]] releases (rather than disposes) the process on these.+ (throw (common/call-failed-ex error))))))++;;; ------------------------------------------------ pool -------------------------------------------------++(def ^:private pool-key+ "Dirigiste pools are keyed; the key is arbitrary, it just has to be the same for every operation."+ :sql-parsing-python)++(def ^:private ^Pool python-process-pool+ "A pool of a single sql-parsing CPython process, held exclusively per call (so calls serialize onto+ it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because+ a warm process is fast and spawning another is costly. See+ [[metabase.sql-parsing.common/create-pool]]."+ (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 10}))++(defn- call-python+ "Execute `sql_tools.<fn-name>` with `args` on a pooled CPython process and return the result as a+ string. A Python-side failure (the process answered `ok: false` — it is healthy, only the call+ failed, thrown as the transport-agnostic [[metabase.sql-parsing.common/call-failed-ex]]) releases the+ process back for reuse, like the graal parser does with its context; any other failure may have left+ the process in a bad state, so it is disposed (dropped from the pool). Retries once if the acquired+ process turned out to be already dead (e.g. reaped or killed while parked in the pool), since a fresh+ one should succeed."+ ^String [fn-name & args]+ (loop [retries 1]+ (let [python-process (.acquire python-process-pool pool-key)+ outcome (try+ (let [result (call-on-process python-process fn-name args)]+ (.release python-process-pool pool-key python-process)+ {:result result})+ (catch Throwable t+ (if (:sql-parsing/error (ex-data t))+ (.release python-process-pool pool-key python-process)+ (.dispose python-process-pool pool-key python-process))+ (if (and (:retryable (ex-data t)) (pos? retries))+ {:retry true}+ (throw t))))]+ (if (:retry outcome)+ (recur (dec retries))+ (:result outcome)))))++(defn parser+ "The `:python` [[metabase.sql-parsing.protocol/SqlParser]] — runs sqlglot in a pool of external native+ CPython processes."+ []+ (common/make-parser call-python))diff --git a/src/metabase/sql_parsing/settings.clj b/src/metabase/sql_parsing/settings.cljnew file mode 100644index 00000000000..476e8bbb435--- /dev/null+++ b/src/metabase/sql_parsing/settings.clj@@ -0,0 +1,18 @@+(ns metabase.sql-parsing.settings+ (:require+ [metabase.settings.core :as setting :refer [defsetting]]+ [metabase.util.i18n :refer [deferred-tru tru]]))++(set! *warn-on-reflection* true)++(defsetting sql-parsing-mode+ (deferred-tru "How SQL parsing (sqlglot) runs: `graalvm` runs it in-process on pooled GraalPy contexts (default), or `python` runs it in a pool of external native CPython child processes (requires a `python3` binary on the host PATH).")+ :type :keyword+ :visibility :internal+ :default :graalvm+ :export? false+ :setter (fn [new-value]+ (when (some? new-value)+ (assert (#{:graalvm :python} (keyword new-value))+ (tru "Invalid sql-parsing-mode! Only values of graalvm and python are allowed.")))+ (setting/set-value-of-type! :keyword :sql-parsing-mode new-value)))diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.cljindex 1fdd5f6fa56..9ecc383ab2d 100644--- a/src/metabase/sql_tools/sqlglot/core.clj+++ b/src/metabase/sql_tools/sqlglot/core.clj@@ -1,15 +1,12 @@(ns metabase.sql-tools.sqlglot.core(:require- [clojure.string :as str][metabase.driver.sql.normalize :as sql.normalize][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.sql-parsing.core :as sql-parsing][metabase.sql-tools.common :as sql-tools.common][metabase.sql-tools.interface :as sql-tools]- [metabase.util.log :as log])- (:import- (org.graalvm.polyglot PolyglotException)))+ [metabase.util.log :as log]))(set! *warn-on-reflection* true)@@ -67,9 +64,9 @@driver {:table table:schema (or table-schema default-schema)}))))query-tables))- (catch PolyglotException e+ (catch Exception e;; Return empty sequence on parse error to follow the Macaw implementation behavior.- (if (str/starts-with? (str (.getMessage e)) "ParseError")+ (if (sql-parsing/parse-error? e)#{}(throw e)))))@@ -152,9 +149,9 @@(mapv (fn [[_catalog schema table]]{:schema schema :table table})table-tuples))- (catch PolyglotException e+ (catch Exception e;; Return empty sequence on parse error to follow the Macaw implementation behavior.- (if (str/starts-with? (str (.getMessage e)) "ParseError")+ (if (sql-parsing/parse-error? e)[](throw e)))))diff --git a/test/metabase/sql_parsing/common_test.clj b/test/metabase/sql_parsing/common_test.cljnew file mode 100644index 00000000000..32e45fa1800--- /dev/null+++ b/test/metabase/sql_parsing/common_test.clj@@ -0,0 +1,151 @@+(ns metabase.sql-parsing.common-test+ (:require+ [clojure.test :refer :all]+ [metabase.sql-parsing.common :as common])+ (:import+ (io.aleph.dirigiste Pool)+ (java.util.concurrent CountDownLatch)))++(set! *warn-on-reflection* true)++(defn- mock-worker+ "Create a mock 'worker' object. The pool doesn't care what type the objects are - it just stores and+ returns them."+ [id]+ {:worker-id id+ :created-at (System/currentTimeMillis)})++(defn- counting-generator+ "Returns a generator function that counts how many workers it has created."+ [counter-atom]+ (fn []+ (mock-worker (swap! counter-atom inc))))++(defn- failing-generator+ "Returns a generator that fails after creating n workers."+ [counter-atom fail-after]+ (fn []+ (let [id (swap! counter-atom inc)]+ (when (> id fail-after)+ (throw (ex-info "Generator limit reached" {:count id :limit fail-after})))+ (mock-worker id))))++;;; ------------------------------------------------- Test Helpers ---------------------------------------------------++(defn- test-pool+ "Create a pool for testing with sensible defaults.+ Returns [pool created-count-atom destroyed-atom]."+ ([]+ (test-pool {}))+ ([config]+ (let [created (atom 0)+ destroyed (atom [])+ pool (common/create-pool (counting-generator created)+ (fn [worker] (swap! destroyed conj worker))+ (merge {:max-size 2, :idle-minutes 1} config))]+ [pool created destroyed])))++(defn- with-worker+ "Execute f with a worker from pool, releasing it afterwards."+ [^Pool pool f]+ (let [worker (.acquire pool :test)]+ (try+ (f worker)+ (finally+ (.release pool :test worker)))))++(defn- run-concurrent+ "Run n threads concurrently, each executing f with a worker from pool.+ Returns vector of results."+ [pool n f]+ (let [latch (CountDownLatch. n)+ results (atom [])+ futures (doall+ (for [i (range n)]+ (future+ (.countDown latch)+ (.await latch)+ (let [result (with-worker pool (fn [worker] (f i worker)))]+ (swap! results conj result)))))]+ (doseq [fut futures] @fut)+ @results))++;;; ------------------------------------------------ Pool Size Tests -------------------------------------------------++(deftest pool-respects-maximum-size-test+ (testing "Pool never creates more than max workers even under concurrent load"+ (let [[pool created] (test-pool {:max-size 2})+ num-threads 20+ results (run-concurrent pool num-threads+ (fn [_i worker]+ (Thread/sleep (long (+ 10 (rand-int 50))))+ worker))]+ (is (<= @created 2)+ (str "Pool created " @created " workers but max is 2"))+ (is (= num-threads (count results))+ "All threads should have received a worker"))))++(deftest pool-reuses-idle-workers-test+ (testing "Sequential uses reuse the same worker instead of creating new ones"+ (let [[pool created] (test-pool)]+ (with-worker pool (fn [_worker] (is (= 1 @created) "First use creates one worker")))+ (with-worker pool (fn [worker]+ (is (= 1 @created) "Second use reuses the same worker")+ (is (= 1 (:worker-id worker))))))))++;;; ------------------------------------------------ Disposal Tests --------------------------------------------------++(deftest pool-dispose-destroys-and-replaces-workers-test+ (testing "Disposing a worker calls destroy and the next acquire generates a fresh one"+ (let [[pool created destroyed] (test-pool)+ worker (.acquire ^Pool pool :test)]+ (.dispose ^Pool pool :test worker)+ (is (= [worker] @destroyed) "Disposed worker should be destroyed")+ (with-worker pool (fn [_worker] (is (= 2 @created) "Disposed worker should be replaced"))))))++;;; --------------------------------------------- Generator Failure Tests --------------------------------------------++(deftest pool-handles-generator-failures-test+ (testing "Pool gracefully handles when generator fails"+ (let [counter (atom 0)+ pool (common/create-pool (failing-generator counter 1) (fn [_worker]) {:max-size 2, :idle-minutes 1})]+ (is (some? (with-worker pool identity)) "First worker creation succeeds")+ ;; Dispose to force new creation+ (let [worker (.acquire ^Pool pool :test)]+ (.dispose ^Pool pool :test worker))+ (is (thrown? Exception (with-worker pool identity)) "Generator failure should propagate"))))++;;; ------------------------------------------- Concurrent Access Tests ----------------------------------------------++(deftest pool-handles-concurrent-access-safely-test+ (testing "Pool handles concurrent access without race conditions"+ (let [[pool created] (test-pool {:max-size 2})+ num-threads 50+ results (run-concurrent pool num-threads+ (fn [i _worker]+ (Thread/sleep (long (rand-int 20)))+ i))]+ (is (= num-threads (count results)) "All threads should complete successfully")+ (is (<= @created 2) (str "Created " @created " workers, expected <= 2")))))++;;; ----------------------------------------- Version Parsing --------------------------------------------------------++(deftest expected-sqlglot-version-test+ (testing "Sqlglot version is properly parsed from pyproject.toml"+ (let [version (common/expected-sqlglot-version)]+ (is (string? version))+ (is (re-matches #"\d+\.\d+\.\d+" version)))))++;;; -------------------------------------------- Pool Behavior Tests -------------------------------------------------++(deftest pool-stores-and-returns-objects-test+ (testing "Pool correctly stores and returns arbitrary objects"+ (let [[pool created] (test-pool)+ result1 (with-worker pool identity)+ result2 (with-worker pool identity)]+ (is (map? result1))+ (is (= 1 (:worker-id result1)))+ (is (= 1 @created))+ (is (map? result2))+ (is (= 1 (:worker-id result2))) ;; Same worker ID+ (is (= 1 @created)))))diff --git a/test/metabase/sql_parsing/core_test.clj b/test/metabase/sql_parsing/core_test.cljindex 556a0049209..561569cc314 100644--- a/test/metabase/sql_parsing/core_test.clj+++ b/test/metabase/sql_parsing/core_test.clj@@ -524,3 +524,12 @@"Should include users wildcard")(is (some #(= ["transactions" "total"] %) (normalize-fields result))"Should include transactions.total")))))++;;; ------------------------------------------------ Parse errors --------------------------------------------------++(deftest parse-error-test+ (testing "unparseable SQL surfaces as a transport-agnostic parse error"+ (let [e (try+ (sql-parsing/referenced-tables "postgres" "SELECT !!!")+ (catch Exception e e))]+ (is (sql-parsing/parse-error? e)))))diff --git a/test/metabase/sql_parsing/jar_python_fs_repro_test.clj b/test/metabase/sql_parsing/jar_python_fs_repro_test.cljindex 5d2ab44d47e..1380ffa923b 100644--- a/test/metabase/sql_parsing/jar_python_fs_repro_test.clj+++ b/test/metabase/sql_parsing/jar_python_fs_repro_test.clj@@ -4,11 +4,11 @@GraalVM's bundled polyglot `FileSystem` wrapper routes `newByteChannel` through`provider.newFileChannel`, which forces JDK ZipFileSystem to extract deflated entries to a temp filebeside the jar — fails when that directory isn't writable (non-root in a Kubernetes pod).- The fix in `metabase.sql-parsing.pool/nio-polyglot-fs` overrides `newByteChannel` to call+ The fix in `metabase.sql-parsing.graal/nio-polyglot-fs` overrides `newByteChannel` to call`Files/newByteChannel`, which returns an in-memory `ByteArrayChannel` for compressed entries."(:require[clojure.test :refer :all]- [metabase.sql-parsing.pool :as pool]+ [metabase.sql-parsing.graal :as graal][metabase.util.files :as u.files])(:import(java.io FileOutputStream)@@ -37,7 +37,7 @@(.write zos (.getBytes "print('hello')\n" "UTF-8"))(.closeEntry zos)))(with-open [^java.nio.file.FileSystem nio-fs (u.files/nio-fs (str zip-file))]- (let [^FileSystem polyfs (#'pool/read-only-polyglot-fs nio-fs)+ (let [^FileSystem polyfs (#'graal/read-only-polyglot-fs nio-fs)path (.parsePath polyfs "/python-sources/sql_tools.py")](with-open [ch (.newByteChannel polyfs path(EnumSet/of StandardOpenOption/READ)diff --git a/test/metabase/sql_parsing/pool_test.clj b/test/metabase/sql_parsing/pool_test.cljdeleted file mode 100644index 30fa0a8d009..00000000000--- a/test/metabase/sql_parsing/pool_test.clj+++ /dev/null@@ -1,161 +0,0 @@-(ns metabase.sql-parsing.pool-test- (:require- [clojure.test :refer :all]- [metabase.sql-parsing.pool :as pool])- (:import- (io.aleph.dirigiste Pool)- (java.util.concurrent CountDownLatch)))--(set! *warn-on-reflection* true)--(defn- mock-context- "Create a mock 'context' object. Since Context is final, we just return a map with metadata.- The pool doesn't care what type the objects are - it just stores and returns them."- [id]- {:context-id id- :created-at (System/currentTimeMillis)})--(defn- counting-generator- "Returns a generator function that counts how many contexts it has created."- [counter-atom]- (fn []- (let [id (swap! counter-atom inc)]- (mock-context id))))--(defn- failing-generator- "Returns a generator that fails after creating n contexts."- [counter-atom fail-after]- (fn []- (let [id (swap! counter-atom inc)]- (when (> id fail-after)- (throw (ex-info "Generator limit reached" {:count id :limit fail-after})))- (mock-context id))))--;;; ------------------------------------------------- Test Helpers -----------------------------------------------------(defn- test-pool- "Create a pool for testing with sensible defaults.- Returns [pool created-count-atom]."- ([]- (test-pool {}))- ([config]- (let [counter (atom 0)- generator (counting-generator counter)- pool (#'pool/make-python-context-pool generator config)]- [pool counter])))--(defn- with-pool- "Execute f with a context from pool. Uses with-open for proper resource management."- [pool f]- (with-open [^java.io.Closeable ctx (#'pool/acquire-context pool)]- (f ctx)))--(defn- run-concurrent- "Run n threads concurrently, each executing f with a context from pool.- Returns vector of results."- [pool n f]- (let [latch (CountDownLatch. n)- results (atom [])- futures (doall- (for [i (range n)]- (future- (.countDown latch)- (.await latch)- (let [result (with-pool pool (fn [ctx] (f i ctx)))]- (swap! results conj result)))))]- (doseq [fut futures] @fut)- @results))--;;; ------------------------------------------------ Pool Size Tests ---------------------------------------------------(deftest pool-respects-maximum-size-test- (testing "Pool never creates more than max contexts even under concurrent load"- (let [[pool created-count] (test-pool {:max-size 3})- num-threads 20- results (run-concurrent pool num-threads- (fn [_i ctx]- (Thread/sleep (long (+ 10 (rand-int 50))))- ctx))]- (is (<= @created-count 3)- (str "Pool created " @created-count " contexts but max is 3"))- (is (= num-threads (count results))- "All threads should have received a context"))))--(deftest pool-maintains-minimum-size-test- (testing "Pool keeps at least 1 context available"- (let [[pool created-count] (test-pool)]- (with-pool pool (fn [_ctx] (is (= 1 @created-count) "First use creates one context")))- (Thread/sleep (long 100))- (with-pool pool (fn [_ctx] (is (= 1 @created-count) "Second use reuses the same context"))))))--;;; ----------------------------------------------- Context Expiry Tests -----------------------------------------------(deftest pool-expires-old-contexts-test- (testing "Pool disposes of contexts that have exceeded their TTL"- (let [[pool _created-count] (test-pool {:ttl-minutes 10})- [context expiry-ts :as tuple] (.acquire ^Pool pool :python)]- (try- (is (some? context) "Context should be created")- (is (number? expiry-ts) "Expiry timestamp should be set")- (is (< (System/nanoTime) expiry-ts) "Expiry should be in the future")- (let [expiry-in-minutes (/ (- expiry-ts (System/nanoTime)) (* 1000000000 60))]- (is (< 9 expiry-in-minutes 11) "Expiry should be approximately 10 minutes"))- (finally- (.release ^Pool pool :python tuple))))))--(deftest with-pooled-context-handles-expired-contexts-test- (testing "with-pooled-context properly handles and replaces expired contexts"- (let [[pool created-count] (test-pool)]- (with-pool pool (fn [_ctx] (is (= 1 @created-count))))- ;; Manually expire by disposing- (let [tuple (.acquire ^Pool pool :python)]- (.dispose ^Pool pool :python tuple))- (with-pool pool (fn [_ctx] (is (= 2 @created-count) "Expired context should be replaced"))))))--;;; --------------------------------------------- Generator Failure Tests ----------------------------------------------(deftest pool-handles-generator-failures-test- (testing "Pool gracefully handles when generator fails"- (let [counter (atom 0)- generator (failing-generator counter 1)- pool (#'pool/make-python-context-pool generator)]- (is (some? (with-pool pool identity)) "First context creation succeeds")- ;; Dispose to force new creation- (let [tuple (.acquire ^Pool pool :python)]- (.dispose ^Pool pool :python tuple))- (is (thrown? Exception (with-pool pool identity)) "Generator failure should propagate"))))--;;; ------------------------------------------- Concurrent Access Tests ------------------------------------------------(deftest pool-handles-concurrent-access-safely-test- (testing "Pool handles concurrent access without race conditions"- (let [[pool created-count] (test-pool {:max-size 3})- num-threads 50- results (run-concurrent pool num-threads- (fn [i _ctx]- (Thread/sleep (long (rand-int 20)))- i))]- (is (= num-threads (count results)) "All threads should complete successfully")- (is (<= @created-count 3) (str "Created " @created-count " contexts, expected <= 3")))))--;;; -------------------------------------------- Pool Behavior Tests ---------------------------------------------------(deftest pool-stores-and-returns-objects-test- (testing "Pool correctly stores and returns arbitrary objects"- (let [[pool created-count] (test-pool)- result1 (with-pool pool :context)- result2 (with-pool pool :context)]- (is (map? result1))- (is (= 1 (:context-id result1)))- (is (= 1 @created-count))- (is (map? result2))- (is (= 1 (:context-id result2))) ;; Same context ID- (is (= 1 @created-count)))))--;;; ----------------------------------------- Version Parsing ----------------------------------------------------------(deftest expected-sqlglot-version-test- (testing "Sqlglot version is properly parsed from pyproject.toml"- (let [version (#'pool/expected-sqlglot-version)]- (is (string? version))- (is (re-matches #"\d+\.\d+\.\d+" version)))))diff --git a/test/metabase/sql_parsing/python_test.clj b/test/metabase/sql_parsing/python_test.cljnew file mode 100644index 00000000000..2cebd795ecf--- /dev/null+++ b/test/metabase/sql_parsing/python_test.clj@@ -0,0 +1,45 @@+(ns metabase.sql-parsing.python-test+ "Tests for the `:python` sqlglot parser: the same assertions the graal parser is trusted for, through+ the external-process path. Requires a `python3` binary on the PATH (sqlglot itself is installed into+ `resources/python-sources` on first use, like the graal tests)."+ (:require+ [clojure.test :refer :all]+ [metabase.sql-parsing.core :as sql-parsing]+ [metabase.sql-parsing.protocol :as protocol]+ [metabase.sql-parsing.python :as python]))++(set! *warn-on-reflection* true)++(deftest referenced-tables-test+ (testing "extracts table references through the external CPython process"+ (is (= [[nil nil "transactions"] [nil "public" "orders"]]+ (protocol/referenced-tables+ (python/parser)+ "postgres"+ "SELECT * FROM transactions t JOIN public.orders o ON t.id = o.tid")))))++(deftest validate-query-test+ (testing "validates syntax through the external CPython process"+ (is (= "ok"+ (:status (protocol/validate-query (python/parser) "postgres" "SELECT 1" nil nil))))+ (is (= "error"+ (:status (protocol/validate-query (python/parser) "postgres" "SELECT FROM WHERE" nil nil))))))++(deftest call-error-keeps-process-healthy-test+ (testing "a failed call surfaces the Python error and the process still answers the next call"+ (is (thrown-with-msg? Exception #"sqlglot call failed"+ (protocol/add-into-clause (python/parser) nil nil "t")))+ (is (= [[nil nil "users"]]+ (protocol/referenced-tables (python/parser) "postgres" "SELECT * FROM users")))))++(deftest parse-error-test+ (testing "unparseable SQL surfaces as the same transport-agnostic parse error the graal parser throws"+ (let [e (try+ (protocol/referenced-tables (python/parser) "postgres" "SELECT !!!")+ (catch Exception e e))]+ (is (sql-parsing/parse-error? e))))+ (testing "other Python-side failures are not parse errors"+ (let [e (try+ (protocol/add-into-clause (python/parser) nil nil "t")+ (catch Exception e e))]+ (is (not (sql-parsing/parse-error? e))))))