Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove unused code (#77613)

Alexander Polyankin669dc959ff65
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 110c7599f1b..6f6f9771d31 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2187,8 +2187,8 @@
sql-parsing
{:team "DevEx"
- :api #{metabase.sql-parsing.core metabase.sql-parsing.init}
- :uses #{analytics-interface settings util}}
+ :api #{metabase.sql-parsing.core}
+ :uses #{analytics-interface util}}
sql-tools
{:team "DevEx"
diff --git a/Dockerfile b/Dockerfile
index a791686b026..02f0ac26170 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,7 @@ ENV FC_LANG=en-US LC_CTYPE=en_US.UTF-8
COPY bin/docker/DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem
# dependencies
-RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk java-cacerts nodejs && \
+RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk java-cacerts && \
apk upgrade && \
rm -rf /var/cache/apk/* && \
mkdir -p /app/certs && \
diff --git a/bin/docker/Dockerfile b/bin/docker/Dockerfile
index a741764aeb1..e5a8483116d 100644
--- a/bin/docker/Dockerfile
+++ b/bin/docker/Dockerfile
@@ -9,7 +9,7 @@ ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA}
COPY DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem
# dependencies
-RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts nodejs python3 && \
+RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts && \
apk upgrade && \
rm -rf /var/cache/apk/* && \
mkdir -p /app/certs && \
diff --git a/bin/docker/Dockerfile_ubuntu b/bin/docker/Dockerfile_ubuntu
index 56c1513c4ed..c013bb51dfd 100644
--- a/bin/docker/Dockerfile_ubuntu
+++ b/bin/docker/Dockerfile_ubuntu
@@ -11,7 +11,7 @@ COPY DigiCertGlobalRootG2.crt.pem /usr/local/share/ca-certificates/DigiCertGloba
# Dependencies
RUN apt-get update && \
apt-get upgrade -y && \
- apt-get install -y ca-certificates ca-certificates-java fonts-noto nodejs python3 && \
+ apt-get install -y ca-certificates ca-certificates-java fonts-noto && \
apt-get clean && \
curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /usr/local/share/ca-certificates/rds-combined-ca-bundle.pem && \
update-ca-certificates && \
diff --git a/dev/src/dev/readme.md b/dev/src/dev/readme.md
index 161de30bda3..d860100950a 100644
--- a/dev/src/dev/readme.md
+++ b/dev/src/dev/readme.md
@@ -6,7 +6,7 @@ In this namespace, you can run `(help)` to get a bit of help. It's principle usa
### NOTE!
-You must build the js bundle used to create the svgs with visx. Run `bun run build-static-viz` to ensure that this bundle is created and the file `resources/frontend_client/app/dist/app-static-viz.bundle.js` exists
+You must build the js bundle used to create the svgs with visx. Run `bun run build-static-viz` to ensure that this bundle is created and the file `resources/frontend_client/app/dist/lib-static-viz.bundle.js` exists
Example usage below:
diff --git a/frontend/lint/module-boundaries.mjs b/frontend/lint/module-boundaries.mjs
index e20d099eec7..0829d0f0161 100644
--- a/frontend/lint/module-boundaries.mjs
+++ b/frontend/lint/module-boundaries.mjs
@@ -250,7 +250,6 @@ const elements = [
"frontend/src/metabase/app-embed.ts",
"frontend/src/metabase/app-public.ts",
"frontend/src/metabase/app-static-viz.ts",
- "frontend/src/metabase/app-static-viz-cli.ts",
"frontend/src/metabase/AppComponent.tsx",
"frontend/src/metabase/App.styled.tsx",
"frontend/src/metabase/AppKBarProvider.tsx",
diff --git a/frontend/src/metabase/app-static-viz-cli.ts b/frontend/src/metabase/app-static-viz-cli.ts
deleted file mode 100644
index d447ea188ca..00000000000
--- a/frontend/src/metabase/app-static-viz-cli.ts
+++ /dev/null
@@ -1,60 +0,0 @@
-// Entry point of the self-contained Node bundle (`app-static-viz-cli.bundle.js`) run by the `:node`
-// static-viz renderer (see `metabase.channel.render.js.node`). It answers newline-delimited JSON render
-// requests over stdin/stdout.
-//
-// ESM imports are hoisted above statements but execute in import order, so the console redirect lives in
-// its own dependency-free module imported first: it must run *before* the viz code loads — stdout carries
-// the line protocol, so anything the bundle logs during init has to go to stderr instead.
-import "metabase/static-viz/console";
-import "metabase/static-viz/polyfill";
-
-import readline from "readline";
-
-import {
- type CellBackgroundColorsInput,
- type RenderChartInput,
- getCellBackgroundColors,
- renderChart,
-} from "metabase/static-viz";
-
-type RenderChartData = {
- fn: "renderChart";
- arg: RenderChartInput;
-};
-
-type GetCellBackgroundColorsData = {
- fn: "getCellBackgroundColors";
- arg: CellBackgroundColorsInput;
-};
-
-type LineData = RenderChartData | GetCellBackgroundColorsData;
-
-const rl = readline.createInterface({ input: process.stdin });
-rl.on("line", (line: string) => {
- const lineClean = line.trim();
- if (!lineClean) {
- return;
- }
- let out;
- try {
- const lineData: LineData = JSON.parse(lineClean);
- const { fn, arg } = lineData;
- if (fn === "renderChart") {
- out = JSON.stringify({ ok: true, result: renderChart(arg) });
- } else if (fn === "getCellBackgroundColors") {
- out = JSON.stringify({ ok: true, result: getCellBackgroundColors(arg) });
- } else {
- throw new Error(`unknown function: ${fn}`);
- }
- } catch (e) {
- out = JSON.stringify({
- ok: false,
- error: String(e instanceof Error ? (e.stack ?? e) : e),
- });
- }
- process.stdout.write(out + "\n");
-});
-
-rl.on("close", () => process.exit(0));
-
-process.stdout.write(JSON.stringify({ ready: true }) + "\n");
diff --git a/frontend/src/metabase/static-viz/console.ts b/frontend/src/metabase/static-viz/console.ts
deleted file mode 100644
index af3b5db1a6a..00000000000
--- a/frontend/src/metabase/static-viz/console.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-// Imported FIRST by the node CLI entry (`app-static-viz-cli.ts`): stdout carries the render line
-// protocol, so console output must be routed to stderr before any static-viz module gets a chance to
-// log during initialization. This module has no imports of its own, so its body is guaranteed to run
-// before every other module in the bundle (ESM imports are hoisted, but execute in import order).
-/* eslint-disable no-console */
-
-for (const method of [
- "log",
- "info",
- "warn",
- "error",
- "debug",
- "trace",
-] as const) {
- console[method] = (...args: unknown[]) =>
- process.stderr.write(args.map((a) => String(a)).join(" ") + "\n");
-}
diff --git a/resources/python-sources/sql_tools_cli.py b/resources/python-sources/sql_tools_cli.py
deleted file mode 100644
index 406894a7f25..00000000000
--- a/resources/python-sources/sql_tools_cli.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""Newline-delimited JSON stdin/stdout server around sql_tools, for running sqlglot on a native
-CPython instead of GraalPy. Spawned by metabase.sql-parsing.python with this directory (or a zip of
-it) on PYTHONPATH; prints one ready line, then answers one response line per request line:
-
- {"fn": "referenced_tables", "args": ["SELECT 1", "postgres"]}
- -> {"ok": true, "result": "..."} | {"ok": false, "error": "..."}
-"""
-
-import json
-import sys
-
-import sql_tools
-
-
-def _resolve(fn_name):
- if fn_name.startswith("_"):
- raise ValueError(f"unknown sql_tools function: {fn_name}")
- fn = getattr(sql_tools, fn_name, None)
- if not callable(fn):
- raise ValueError(f"unknown sql_tools function: {fn_name}")
- return fn
-
-
-def main():
- out = sys.stdout
- out.write(json.dumps({"ready": True}) + "\n")
- out.flush()
- for line in sys.stdin:
- line = line.strip()
- if not line:
- continue
- try:
- request = json.loads(line)
- result = _resolve(request["fn"])(*request["args"])
- response = {"ok": True, "result": result}
- except Exception as e: # noqa: BLE001 -- any failure is reported to the JVM side
- response = {"ok": False, "error": f"{type(e).__name__}: {e}"}
- out.write(json.dumps(response) + "\n")
- out.flush()
-
-
-if __name__ == "__main__":
- main()
diff --git a/rspack.static-viz.config.js b/rspack.static-viz.config.js
index 0bd5da16c07..74b09fc6d1c 100644
--- a/rspack.static-viz.config.js
+++ b/rspack.static-viz.config.js
@@ -20,94 +20,8 @@ const ENTERPRISE_SRC_PATH =
const devMode = WEBPACK_BUNDLE !== "production";
-const moduleConfig = {
- rules: [
- {
- test: /\.css$/i,
- use: "null-loader",
- },
- {
- test: /\.(tsx?|jsx?)$/,
- exclude: /node_modules|cljs|css\/core\/fonts\.styled\.ts/,
- use: [
- {
- loader: "builtin:swc-loader",
- options: {
- jsc: {
- loose: true,
- transform: {
- react: {
- runtime: "automatic",
- refresh: false,
- },
- },
- parser: {
- syntax: "typescript",
- tsx: true,
- },
- experimental: {
- plugins: [["@swc/plugin-emotion", { sourceMap: devMode }]],
- },
- },
-
- sourceMaps: false,
- minify: true,
- env: {
- targets: ["defaults"],
- },
- },
- },
- ],
- },
- {
- test: /\.svg/,
- type: "asset/source",
- resourceQuery: /source/, // *.svg?source
- },
- {
- test: /\.svg$/i,
- issuer: /\.[jt]sx?$/,
- resourceQuery: /component/, // *.svg?component
- use: [
- {
- loader: "@svgr/webpack",
- options: {
- ref: true,
- svgoConfig: SVGO_CONFIG,
- },
- },
- ],
- },
- {
- test: /\.svg$/,
- type: "asset/resource",
- resourceQuery: { not: [/component|source/] },
- },
- ],
-};
-
-const resolveAlias = {
- assets: ASSETS_PATH,
- metabase: SRC_PATH,
- "metabase-enterprise": ENTERPRISE_SRC_PATH,
- cljs: devMode ? CLJS_SRC_PATH_DEV : CLJS_SRC_PATH,
- "metabase-lib": LIB_SRC_PATH,
- "metabase-types": TYPES_SRC_PATH,
- embedding: EMBEDDING_SRC_PATH,
- "embedding-sdk-bundle": SDK_BUNDLE_SRC_PATH,
- "embedding-sdk-shared": SDK_SHARED_SRC_PATH,
- "process/browser": require.resolve("process/browser"),
- "ee-overrides":
- process.env.MB_EDITION === "ee"
- ? ENTERPRISE_SRC_PATH + "/static-viz-overrides"
- : SRC_PATH + "/utils/noop",
-};
-
-const resolveExtensions = [".web.js", ".js", ".jsx", ".ts", ".tsx"];
-
module.exports = (env) => {
- const graalConfig = {
- name: "graal",
+ return {
mode: "production",
context: SRC_PATH,
@@ -116,7 +30,7 @@ module.exports = (env) => {
},
entry: {
- "app-static-viz": {
+ "lib-static-viz": {
import: "./app-static-viz.ts",
library: {
name: "MetabaseStaticViz",
@@ -132,10 +46,89 @@ module.exports = (env) => {
globalObject: "globalThis",
},
- module: moduleConfig,
+ module: {
+ rules: [
+ {
+ test: /\.css$/i,
+ use: "null-loader",
+ },
+ {
+ test: /\.(tsx?|jsx?)$/,
+ exclude: /node_modules|cljs|css\/core\/fonts\.styled\.ts/,
+ use: [
+ {
+ loader: "builtin:swc-loader",
+ options: {
+ jsc: {
+ loose: true,
+ transform: {
+ react: {
+ runtime: "automatic",
+ refresh: false,
+ },
+ },
+ parser: {
+ syntax: "typescript",
+ tsx: true,
+ },
+ experimental: {
+ plugins: [["@swc/plugin-emotion", { sourceMap: devMode }]],
+ },
+ },
+
+ sourceMaps: false,
+ minify: true,
+ env: {
+ targets: ["defaults"],
+ },
+ },
+ },
+ ],
+ },
+ {
+ test: /\.svg/,
+ type: "asset/source",
+ resourceQuery: /source/, // *.svg?source
+ },
+ {
+ test: /\.svg$/i,
+ issuer: /\.[jt]sx?$/,
+ resourceQuery: /component/, // *.svg?component
+ use: [
+ {
+ loader: "@svgr/webpack",
+ options: {
+ ref: true,
+ svgoConfig: SVGO_CONFIG,
+ },
+ },
+ ],
+ },
+ {
+ test: /\.svg$/,
+ type: "asset/resource",
+ resourceQuery: { not: [/component|source/] },
+ },
+ ],
+ },
resolve: {
- extensions: resolveExtensions,
- alias: resolveAlias,
+ extensions: [".web.js", ".js", ".jsx", ".ts", ".tsx"],
+ alias: {
+ assets: ASSETS_PATH,
+ metabase: SRC_PATH,
+ "metabase-enterprise": ENTERPRISE_SRC_PATH,
+ cljs: devMode ? CLJS_SRC_PATH_DEV : CLJS_SRC_PATH,
+ "metabase-lib": LIB_SRC_PATH,
+ "metabase-types": TYPES_SRC_PATH,
+ embedding: EMBEDDING_SRC_PATH,
+ "embedding-sdk-bundle": SDK_BUNDLE_SRC_PATH,
+ "embedding-sdk-shared": SDK_SHARED_SRC_PATH,
+ "process/browser": require.resolve("process/browser"),
+ "ee-overrides":
+ process.env.MB_EDITION === "ee"
+ ? ENTERPRISE_SRC_PATH + "/static-viz-overrides"
+ : SRC_PATH + "/utils/noop",
+ },
fallback: {
crypto: require.resolve("crypto-browserify"),
stream: require.resolve("stream-browserify"),
@@ -179,46 +172,9 @@ module.exports = (env) => {
)
.map((module) =>
module.nameForCondition.replace(`${__dirname}/`, ""),
- )
- .concat(["frontend/src/metabase/app-static-viz-cli.ts"]),
+ ),
}),
}),
],
};
-
- const nodeCliConfig = {
- name: "node-cli",
- mode: "production",
- context: SRC_PATH,
- target: "node",
-
- performance: {
- hints: false,
- },
-
- entry: {
- "app-static-viz-cli": "./app-static-viz-cli.ts",
- },
-
- output: {
- path: BUILD_PATH + "/app/dist",
- filename: "[name].bundle.js",
- },
-
- module: moduleConfig,
- resolve: {
- extensions: resolveExtensions,
- alias: resolveAlias,
- },
- optimization: {
- minimize: true,
- },
- plugins: [
- new rspack.EnvironmentPlugin({
- IS_EMBEDDING_SDK_BUILD: false,
- }),
- ],
- };
-
- return [graalConfig, nodeCliConfig];
};
diff --git a/src/metabase/channel/render/js/common.clj b/src/metabase/channel/render/js/common.clj
index 810ccfaeae9..41b318f6fce 100644
--- a/src/metabase/channel/render/js/common.clj
+++ b/src/metabase/channel/render/js/common.clj
@@ -1,7 +1,6 @@
(ns metabase.channel.render.js.common
- "Shared helpers for the pooled static-viz renderers ([[metabase.channel.render.js.graal]] and
- [[metabase.channel.render.js.node]]): the graal bundle's classpath path, the test-init guard, and the
- dirigiste worker-pool factory they both use."
+ "Shared helpers for pooled static-viz renderers (see [[metabase.channel.render.js.graal]]): the graal
+ bundle's classpath path, the test-init guard, and the dirigiste worker-pool factory."
(:require
[metabase.config.core :as config])
(:import
@@ -11,10 +10,8 @@
(set! *warn-on-reflection* true)
(def bundle-resource-path
- "Classpath path of the built static-viz bundle the graal renderer evaluates in-process. (The node
- renderer runs its own self-contained bundle from disk instead — see
- [[metabase.channel.render.js.node]].)"
- "frontend_client/app/dist/app-static-viz.bundle.js")
+ "Classpath path of the built static-viz bundle the graal renderer evaluates in-process."
+ "frontend_client/app/dist/lib-static-viz.bundle.js")
(defn assert-tests-not-initializing!
"Guard against loading the static-viz bundle as a side effect of loading namespaces: it might not have
diff --git a/src/metabase/channel/render/js/node.clj b/src/metabase/channel/render/js/node.clj
deleted file mode 100644
index 021855fecdd..00000000000
--- a/src/metabase/channel/render/js/node.clj
+++ /dev/null
@@ -1,182 +0,0 @@
-(ns metabase.channel.render.js.node
- "The `:node` [[metabase.channel.render.js.protocol/StaticVizRenderer]]: runs the static-viz JS in a pool
- of external Node.js child processes rather than in-process on GraalVM. Each process runs the
- self-contained node entrypoint (`app-static-viz-cli.bundle.js`, built by `rspack.static-viz.config.js`),
- which answers newline-delimited JSON render requests over stdin/stdout.
-
- The script is written to a fresh temp file from the classpath at every process spawn, executed
- immediately, and deleted when the process is destroyed. So what `node` executes always comes straight
- out of the jar — there is no long-lived on-disk copy to drift out of sync or be tampered with, and no
- deploy-time provisioning.
-
- The pool holds a single process, held exclusively per render (so renders serialize onto it); with a min
- of 0 it shrinks to 0 when idle, killing the process after 1 minute with no work. We cap it at one
- process — spawning a fresh `node` (runtime init + bundle load) is costly, and a warm process renders
- fast enough that queuing a second render behind it beats paying that startup cost for concurrency.
-
- Requires a `node` binary on the host's PATH."
- (:require
- [clojure.java.io :as io]
- [metabase.channel.render.js.common :as common]
- [metabase.channel.render.js.protocol :as js.protocol]
- [metabase.util.json :as json])
- (:import
- (io.aleph.dirigiste Pool)
- (java.io BufferedReader BufferedWriter File IOException InputStreamReader OutputStreamWriter)
- (java.lang ProcessBuilder$Redirect)
- (java.nio.charset StandardCharsets)
- (java.nio.file Files)
- (java.nio.file.attribute FileAttribute)))
-
-(set! *warn-on-reflection* true)
-
-;;; ------------------------------------------------ process ----------------------------------------------
-
-(def ^:private render-timeout-ms
- "How long to wait for a render process to answer before assuming it has wedged: we kill it and fail the
- render, so a stuck process can't hold its pool slot forever."
- (* 60 1000))
-
-(defrecord ^:private NodeProcess [^Process process ^BufferedWriter writer ^BufferedReader reader ^File script-file])
-
-(defn- read-response-line
- "Read one line from the process, killing it and throwing if it doesn't answer within
- [[render-timeout-ms]] (so a wedged process can't hold its pool slot forever). `destroyForcibly`
- (SIGKILL) rather than the polite `destroy`: a process wedged hard enough to miss the deadline may
- ignore SIGTERM, and killing it also closes its stdout, which unblocks the reader future."
- ^String [^BufferedReader reader ^Process process]
- (let [fut (future (.readLine reader))
- line (deref fut render-timeout-ms ::timeout)]
- (if (identical? ::timeout line)
- (do (.destroyForcibly process)
- (future-cancel fut)
- (throw (ex-info "static-viz node render timed out" {})))
- line)))
-
-(def ^:private script-resource-path
- "Classpath path of the node entrypoint bundle written to a temp file at each process spawn."
- "frontend_client/app/dist/app-static-viz-cli.bundle.js")
-
-(defn- create-script-file!
- "Write the node entrypoint bundle from the classpath to a fresh temp file, to be executed immediately
- and deleted when its process is destroyed (`deleteOnExit` is only a backstop for JVM death)."
- ^File []
- (let [resource (io/resource script-resource-path)]
- (when (nil? resource)
- (throw (ex-info (str "static-viz resource not found: " script-resource-path
- " — build it with `bun run build-static-viz`")
- {:resource script-resource-path})))
- (let [file (.toFile (Files/createTempFile "mb-static-viz" ".js" (into-array FileAttribute [])))]
- (.deleteOnExit file)
- (with-open [in (io/input-stream resource)]
- (io/copy in file))
- file)))
-
-(defn- start-process!
- "Write the script to a temp file, spawn `node` on it, and block until it reports it has loaded. The
- process's stderr is inherited (for debugging); stdout carries the line protocol. On any failure to
- reach a ready state the process is destroyed and the file deleted, so nothing leaks."
- ^NodeProcess []
- (common/assert-tests-not-initializing!)
- (let [script (create-script-file!)
- ^Process process (try
- (.. (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String ["node" (.getAbsolutePath script)]))
- (redirectError ProcessBuilder$Redirect/INHERIT)
- (start))
- (catch Throwable t
- (.delete script)
- (if (instance? IOException t)
- (throw (ex-info (str "could not start `node` for static-viz rendering — is node"
- " installed and on PATH? (otherwise set MB_STATIC_VIZ_MODE"
- " to graalvm)")
- {} t))
- (throw t))))]
- (try
- (let [writer (BufferedWriter. (OutputStreamWriter. (.getOutputStream process) StandardCharsets/UTF_8))
- reader (BufferedReader. (InputStreamReader. (.getInputStream process) StandardCharsets/UTF_8))
- ready (read-response-line reader process)]
- (when-not (some-> ready json/decode+kw :ready)
- (throw (ex-info "static-viz node process failed to start" {:line ready})))
- (->NodeProcess process writer reader script))
- (catch Throwable t
- (.destroy process)
- (.delete script)
- (throw t)))))
-
-(defn- stop-process!
- [{:keys [^Process process ^BufferedWriter writer ^BufferedReader reader ^File script-file]}]
- (try (.close writer) (catch Exception _))
- (try (.close reader) (catch Exception _))
- (try (.destroy process) (catch Exception _))
- (try (.delete script-file) (catch Exception _)))
-
-(defn- render-on-process
- "Send one render request to `node-process` and read its response. `arg` is a Clojure data structure sent
- as a JS object (not a JSON string), so the bundle serializes only once — at this process boundary rather
- than again inside the bundle. Returns the render result parsed into Clojure data. Not thread-safe on its
- own — the pool hands a process to one render at a time. A write failure (broken pipe) means the pooled
- process had already died; it is thrown as `:retryable` so [[call-node]] can get a fresh one."
- [{:keys [^BufferedWriter writer ^BufferedReader reader ^Process process]} fn-name arg]
- (try
- (.write writer ^String (str (json/encode {:fn fn-name :arg arg}) "\n"))
- (.flush writer)
- (catch IOException e
- (throw (ex-info "static-viz node process is not writable" {:retryable true} e))))
- (let [line (read-response-line reader process)]
- (when (nil? line)
- (throw (ex-info "static-viz node process closed unexpectedly" {:retryable true})))
- (let [{:keys [ok result error]} (json/decode+kw line)]
- (if ok
- result
- ;; :render-error — the process answered the protocol correctly, so it is healthy; only this
- ;; render failed. [[call-node]] releases (rather than disposes) the process on these.
- (throw (ex-info (str "static-viz node render failed: " error) {:render-error true}))))))
-
-;;; ------------------------------------------------ pool -------------------------------------------------
-
-(def ^:private pool-key
- "Dirigiste pools are keyed; the key is arbitrary, it just has to be the same for every operation."
- :static-viz-node)
-
-(def ^:private ^Pool node-process-pool
- "A pool of a single static-viz Node.js process, held exclusively per render (so renders serialize onto
- it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because a
- warm process is fast and spawning another is costly. See
- [[metabase.channel.render.js.common/create-pool]]."
- (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 1}))
-
-(defn- call-node
- "Run static-viz bundle function `fn-name` with `arg` (a Clojure data structure) on a pooled Node process,
- returning the render result parsed into Clojure data. A `:render-error` (the process answered `ok:
- false` — it is healthy, only the render failed) releases the process back for reuse, like the graal
- renderer does with its context; any other failure may have left the process in a bad state, so it is
- disposed (dropped from the pool). Retries once if the acquired process turned out to be already dead
- (e.g. reaped or killed while parked in the pool), since a fresh one should succeed."
- [fn-name arg]
- (loop [retries 1]
- (let [node-process (.acquire node-process-pool pool-key)
- outcome (try
- (let [result (render-on-process node-process fn-name arg)]
- (.release node-process-pool pool-key node-process)
- {:result result})
- (catch Throwable t
- (if (:render-error (ex-data t))
- (.release node-process-pool pool-key node-process)
- (.dispose node-process-pool pool-key node-process))
- (if (and (:retryable (ex-data t)) (pos? retries))
- {:retry true}
- (throw t))))]
- (if (:retry outcome)
- (recur (dec retries))
- (:result outcome)))))
-
-(defn renderer
- "The `:node` [[metabase.channel.render.js.protocol/StaticVizRenderer]] — runs the static-viz JS in a pool
- of external Node.js processes. Passes each `input` map to the bundle's object-in/object-out functions, so
- it is serialized once (at the process boundary) rather than JSON-encoded again inside the bundle."
- []
- (reify js.protocol/StaticVizRenderer
- (chart [_ input]
- (call-node "renderChart" input))
- (cell-background-colors [_ input]
- (call-node "getCellBackgroundColors" input))))
diff --git a/src/metabase/channel/render/js/renderer.clj b/src/metabase/channel/render/js/renderer.clj
index 655a46d8f9f..4e290248b00 100644
--- a/src/metabase/channel/render/js/renderer.clj
+++ b/src/metabase/channel/render/js/renderer.clj
@@ -1,19 +1,14 @@
(ns metabase.channel.render.js.renderer
- "Hands callers the static-viz [[metabase.channel.render.js.protocol/StaticVizRenderer]] implementation
- for the configured [[metabase.channel.settings/static-viz-mode]]: the in-process GraalVM renderer
- (default) or a pool of external Node.js processes. Kept behind the protocol so the callers
+ "Hands callers the static-viz [[metabase.channel.render.js.protocol/StaticVizRenderer]] implementation:
+ the in-process GraalVM renderer. Kept behind the protocol so the callers
([[metabase.channel.render.js.svg]], [[metabase.channel.render.js.color]]) don't care which backend
runs."
(:require
- [metabase.channel.render.js.graal :as graal]
- [metabase.channel.render.js.node :as node]
- [metabase.channel.settings :as channel.settings]))
+ [metabase.channel.render.js.graal :as graal]))
(set! *warn-on-reflection* true)
(defn renderer
- "The static-viz renderer for the configured mode."
+ "The static-viz renderer."
[]
- (if (= :node (channel.settings/static-viz-mode))
- (node/renderer)
- (graal/renderer)))
+ (graal/renderer))
diff --git a/src/metabase/channel/settings.clj b/src/metabase/channel/settings.clj
index 6adbc7edd82..b09b0bc6943 100644
--- a/src/metabase/channel/settings.clj
+++ b/src/metabase/channel/settings.clj
@@ -337,17 +337,3 @@
:getter (fn [] (boolean (slack-app-token)))
:export? false
:setter :none)
-
-(defsetting static-viz-mode
- "How static visualizations (subscription/alert charts and pulse table cell colors) are rendered:
- `graalvm` runs the JavaScript in-process on a pooled GraalVM context (default), or `node` runs it in a
- pool of external Node.js child processes (requires a `node` binary on the host's PATH)."
- :type :keyword
- :visibility :internal
- :default :graalvm
- :export? false
- :setter (fn [new-value]
- (when (some? new-value)
- (assert (#{:graalvm :node} (keyword new-value))
- (tru "Invalid static-viz-mode! Only values of graalvm and node are allowed.")))
- (setting/set-value-of-type! :keyword :static-viz-mode new-value)))
diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.clj
index c50465fd643..5d7a3752330 100644
--- a/src/metabase/core/init.clj
+++ b/src/metabase/core/init.clj
@@ -64,7 +64,6 @@
[metabase.settings.init]
[metabase.setup.init]
[metabase.slackbot.init]
- [metabase.sql-parsing.init]
[metabase.sql-tools.init]
[metabase.sso.init]
[metabase.store-api.init]
diff --git a/src/metabase/sql_parsing/common.clj b/src/metabase/sql_parsing/common.clj
index faf2cd0383f..af213aa2725 100644
--- a/src/metabase/sql_parsing/common.clj
+++ b/src/metabase/sql_parsing/common.clj
@@ -1,8 +1,7 @@
(ns metabase.sql-parsing.common
- "Shared helpers for the pooled sqlglot parsers ([[metabase.sql-parsing.graal]] and
- [[metabase.sql-parsing.python]]): the python-sources locations, the lazy dev sqlglot install, the
- protocol reify over a transport function, and the dirigiste worker-pool factory (copied from
- [[metabase.channel.render.js.common]])."
+ "Shared helpers for pooled sqlglot parsers (see [[metabase.sql-parsing.graal]]): the python-sources
+ locations, the lazy dev sqlglot install, the protocol reify over a transport function, and the
+ dirigiste worker-pool factory (copied from [[metabase.channel.render.js.common]])."
(:require
[clojure.java.io :as io]
[clojure.java.shell :as shell]
diff --git a/src/metabase/sql_parsing/init.clj b/src/metabase/sql_parsing/init.clj
deleted file mode 100644
index 88bded2543e..00000000000
--- a/src/metabase/sql_parsing/init.clj
+++ /dev/null
@@ -1,3 +0,0 @@
-(ns metabase.sql-parsing.init
- (:require
- [metabase.sql-parsing.settings]))
diff --git a/src/metabase/sql_parsing/parser.clj b/src/metabase/sql_parsing/parser.clj
index cad9dbe356e..02bfa71ff36 100644
--- a/src/metabase/sql_parsing/parser.clj
+++ b/src/metabase/sql_parsing/parser.clj
@@ -1,18 +1,13 @@
(ns metabase.sql-parsing.parser
- "Hands callers the [[metabase.sql-parsing.protocol/SqlParser]] implementation for the configured
- [[metabase.sql-parsing.settings/sql-parsing-mode]]: the in-process GraalPy parser (default) or a pool
- of external native CPython processes. Kept behind the protocol so the caller
- ([[metabase.sql-parsing.core]]) doesn't care which backend runs."
+ "Hands callers the [[metabase.sql-parsing.protocol/SqlParser]] implementation: the in-process GraalPy
+ parser. Kept behind the protocol so the caller ([[metabase.sql-parsing.core]]) doesn't care which
+ backend runs."
(:require
- [metabase.sql-parsing.graal :as graal]
- [metabase.sql-parsing.python :as python]
- [metabase.sql-parsing.settings :as settings]))
+ [metabase.sql-parsing.graal :as graal]))
(set! *warn-on-reflection* true)
(defn parser
- "The sqlglot parser for the configured mode."
+ "The sqlglot parser."
[]
- (if (= :python (settings/sql-parsing-mode))
- (python/parser)
- (graal/parser)))
+ (graal/parser))
diff --git a/src/metabase/sql_parsing/python.clj b/src/metabase/sql_parsing/python.clj
deleted file mode 100644
index 0776da7e92b..00000000000
--- a/src/metabase/sql_parsing/python.clj
+++ /dev/null
@@ -1,199 +0,0 @@
-(ns metabase.sql-parsing.python
- "The `:python` [[metabase.sql-parsing.protocol/SqlParser]]: runs sqlglot in a pool of external native
- CPython child processes rather than in-process on GraalPy. Each process runs `sql_tools_cli.py`, which
- answers newline-delimited JSON requests over stdin/stdout.
-
- The Python sources come straight out of the classpath: in dev the child imports directly from
- `resources/python-sources`; from the jar they are written to a fresh temp zip at every process spawn
- (CPython imports from zips via zipimport), put on the child's PYTHONPATH, and deleted when the process
- is destroyed — so there is no long-lived on-disk copy to drift out of sync or be tampered with, and no
- deploy-time provisioning.
-
- The pool holds a single process, held exclusively per call (so calls serialize onto it); with a min of
- 0 it shrinks to 0 when idle, killing the process after 10 minutes with no work. We cap it at one
- process — spawning a fresh `python3` (interpreter init + sqlglot import) is costly, and a warm process
- parses fast enough that queuing a second call behind it beats paying that startup cost for
- concurrency.
-
- Requires a `python3` binary on the host's PATH."
- (:require
- [clojure.java.io :as io]
- [clojure.string :as str]
- [metabase.sql-parsing.common :as common]
- [metabase.util.files :as u.files]
- [metabase.util.json :as json])
- (:import
- (io.aleph.dirigiste Pool)
- (java.io BufferedReader BufferedWriter File IOException InputStreamReader OutputStreamWriter)
- (java.lang ProcessBuilder$Redirect)
- (java.nio.charset StandardCharsets)
- (java.nio.file FileVisitOption Files LinkOption)
- (java.nio.file.attribute FileAttribute)
- (java.util.concurrent TimeoutException)
- (java.util.zip ZipEntry ZipOutputStream)))
-
-(set! *warn-on-reflection* true)
-
-;;; ------------------------------------------------ process ----------------------------------------------
-
-(def ^:private ^:const call-timeout-ms
- "How long to wait for the process to answer before assuming it has wedged: we kill it and fail the
- call, so a stuck process can't hold its pool slot forever."
- 30000)
-
-(defrecord ^:private PythonProcess [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip])
-
-(defn- read-response-line
- "Read one line from the process, killing it and throwing if it doesn't answer within
- [[call-timeout-ms]] (so a wedged process can't hold its pool slot forever). `destroyForcibly`
- (SIGKILL) rather than the polite `destroy`: a process wedged hard enough to miss the deadline may
- ignore SIGTERM, and killing it also closes its stdout, which unblocks the reader future."
- ^String [^BufferedReader reader ^Process process]
- (let [fut (future (.readLine reader))
- line (deref fut call-timeout-ms ::timeout)]
- (if (identical? ::timeout line)
- (do (.destroyForcibly process)
- (future-cancel fut)
- (throw (TimeoutException. (str "Python execution timed out after " call-timeout-ms "ms"))))
- line)))
-
-(defn- create-sources-zip!
- "Write the python-sources tree from the jar to a fresh temp zip, importable by the child via zipimport,
- to be put on its PYTHONPATH and deleted when its process is destroyed (`deleteOnExit` is only a
- backstop for JVM death)."
- ^File []
- (let [file (.toFile (Files/createTempFile "mb-sql-tools" ".zip" (into-array FileAttribute [])))]
- (.deleteOnExit file)
- (try
- (with-open [jar-fs (u.files/nio-fs (u.files/get-jar-path))
- zos (ZipOutputStream. (io/output-stream file))]
- (let [root (.getPath jar-fs (str "/" common/python-sources-resource) (into-array String []))]
- (with-open [paths (Files/walk root (into-array FileVisitOption []))]
- (doseq [^java.nio.file.Path path (iterator-seq (.iterator paths))
- :when (and (not (Files/isDirectory path (into-array LinkOption [])))
- (not (str/includes? (str path) "__pycache__")))]
- (.putNextEntry zos (ZipEntry. (str (.relativize root path))))
- (Files/copy path zos)
- (.closeEntry zos)))))
- file
- (catch Throwable t
- (.delete file)
- (throw t)))))
-
-(defn- python-path
- "The PYTHONPATH entry for a new child process, plus the temp zip to delete with it (nil in dev, where
- the child imports straight from `resources/python-sources`)."
- []
- (if (common/jar-resource? common/python-sources-resource)
- (let [zip (create-sources-zip!)]
- {:path (.getAbsolutePath zip) :sources-zip zip})
- (do
- (common/ensure-sqlglot-installed!)
- {:path common/dev-python-sources-dir :sources-zip nil})))
-
-(defn- start-process!
- "Spawn `python3 -m sql_tools_cli` with the Python sources on PYTHONPATH and block until it reports it
- has loaded. The process's stderr is inherited (for debugging); stdout carries the line protocol. On
- any failure to reach a ready state the process is destroyed and the temp zip deleted, so nothing
- leaks."
- ^PythonProcess []
- (let [{:keys [path ^File sources-zip]} (python-path)
- ^Process process (try
- (let [builder (.. (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String ["python3" "-m" "sql_tools_cli"]))
- (redirectError ProcessBuilder$Redirect/INHERIT))]
- (doto (.environment builder)
- (.put "PYTHONPATH" path)
- (.put "PYTHONUNBUFFERED" "1"))
- (.start builder))
- (catch Throwable t
- (some-> sources-zip .delete)
- (if (instance? IOException t)
- (throw (ex-info (str "could not start `python3` for SQL parsing — is python3"
- " installed and on PATH? (otherwise set MB_SQL_PARSING_MODE"
- " to graalvm)")
- {} t))
- (throw t))))]
- (try
- (let [writer (BufferedWriter. (OutputStreamWriter. (.getOutputStream process) StandardCharsets/UTF_8))
- reader (BufferedReader. (InputStreamReader. (.getInputStream process) StandardCharsets/UTF_8))
- ready (read-response-line reader process)]
- (when-not (some-> ready json/decode+kw :ready)
- (throw (ex-info "sql-parsing python process failed to start" {:line ready})))
- (->PythonProcess process writer reader sources-zip))
- (catch Throwable t
- (.destroy process)
- (some-> sources-zip .delete)
- (throw t)))))
-
-(defn- stop-process!
- [{:keys [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip]}]
- (try (.close writer) (catch Exception _))
- (try (.close reader) (catch Exception _))
- (try (.destroy process) (catch Exception _))
- (some-> sources-zip .delete))
-
-(defn- call-on-process
- "Send one request to `python-process` and read its response. Returns the `sql_tools` function's result
- string. Not thread-safe on its own — the pool hands a process to one call at a time. A write failure
- (broken pipe) means the pooled process had already died; it is thrown as `:retryable` so [[call-python]]
- can get a fresh one."
- ^String [{:keys [^BufferedWriter writer ^BufferedReader reader ^Process process]} fn-name args]
- (try
- (.write writer ^String (str (json/encode {:fn fn-name :args args}) "\n"))
- (.flush writer)
- (catch IOException e
- (throw (ex-info "sql-parsing python process is not writable" {:retryable true} e))))
- (let [line (read-response-line reader process)]
- (when (nil? line)
- (throw (ex-info "sql-parsing python process closed unexpectedly" {:retryable true})))
- (let [{:keys [ok result error]} (json/decode+kw line)]
- (if ok
- result
- ;; The process answered the protocol correctly, so it is healthy; only this call failed.
- ;; [[call-python]] releases (rather than disposes) the process on these.
- (throw (common/call-failed-ex error))))))
-
-;;; ------------------------------------------------ pool -------------------------------------------------
-
-(def ^:private pool-key
- "Dirigiste pools are keyed; the key is arbitrary, it just has to be the same for every operation."
- :sql-parsing-python)
-
-(def ^:private ^Pool python-process-pool
- "A pool of a single sql-parsing CPython process, held exclusively per call (so calls serialize onto
- it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because
- a warm process is fast and spawning another is costly. See
- [[metabase.sql-parsing.common/create-pool]]."
- (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 10}))
-
-(defn- call-python
- "Execute `sql_tools.<fn-name>` with `args` on a pooled CPython process and return the result as a
- string. A Python-side failure (the process answered `ok: false` — it is healthy, only the call
- failed, thrown as the transport-agnostic [[metabase.sql-parsing.common/call-failed-ex]]) releases the
- process back for reuse, like the graal parser does with its context; any other failure may have left
- the process in a bad state, so it is disposed (dropped from the pool). Retries once if the acquired
- process turned out to be already dead (e.g. reaped or killed while parked in the pool), since a fresh
- one should succeed."
- ^String [fn-name & args]
- (loop [retries 1]
- (let [python-process (.acquire python-process-pool pool-key)
- outcome (try
- (let [result (call-on-process python-process fn-name args)]
- (.release python-process-pool pool-key python-process)
- {:result result})
- (catch Throwable t
- (if (:sql-parsing/error (ex-data t))
- (.release python-process-pool pool-key python-process)
- (.dispose python-process-pool pool-key python-process))
- (if (and (:retryable (ex-data t)) (pos? retries))
- {:retry true}
- (throw t))))]
- (if (:retry outcome)
- (recur (dec retries))
- (:result outcome)))))
-
-(defn parser
- "The `:python` [[metabase.sql-parsing.protocol/SqlParser]] — runs sqlglot in a pool of external native
- CPython processes."
- []
- (common/make-parser call-python))
diff --git a/src/metabase/sql_parsing/settings.clj b/src/metabase/sql_parsing/settings.clj
deleted file mode 100644
index 476e8bbb435..00000000000
--- a/src/metabase/sql_parsing/settings.clj
+++ /dev/null
@@ -1,18 +0,0 @@
-(ns metabase.sql-parsing.settings
- (:require
- [metabase.settings.core :as setting :refer [defsetting]]
- [metabase.util.i18n :refer [deferred-tru tru]]))
-
-(set! *warn-on-reflection* true)
-
-(defsetting sql-parsing-mode
- (deferred-tru "How SQL parsing (sqlglot) runs: `graalvm` runs it in-process on pooled GraalPy contexts (default), or `python` runs it in a pool of external native CPython child processes (requires a `python3` binary on the host PATH).")
- :type :keyword
- :visibility :internal
- :default :graalvm
- :export? false
- :setter (fn [new-value]
- (when (some? new-value)
- (assert (#{:graalvm :python} (keyword new-value))
- (tru "Invalid sql-parsing-mode! Only values of graalvm and python are allowed.")))
- (setting/set-value-of-type! :keyword :sql-parsing-mode new-value)))
diff --git a/test/metabase/channel/render/js/node_test.clj b/test/metabase/channel/render/js/node_test.clj
deleted file mode 100644
index 5012b26d6d0..00000000000
--- a/test/metabase/channel/render/js/node_test.clj
+++ /dev/null
@@ -1,46 +0,0 @@
-(ns metabase.channel.render.js.node-test
- "Tests for the `:node` static-viz renderer: the same assertions the graal renderer is trusted for,
- through the external-process path. Requires a built static-viz bundle (like the graal tests) plus a
- `node` binary on the PATH."
- (:require
- [clojure.string :as str]
- [clojure.test :refer :all]
- [metabase.channel.render.js.node :as node]
- [metabase.channel.render.js.protocol :as js.protocol]))
-
-(set! *warn-on-reflection* true)
-
-(use-fixtures :each
- (fn warn-possible-rebuild
- [thunk]
- (testing "[PRO TIP] If this test fails, you may need to rebuild the bundle with `bun run build-static-viz`\n"
- (thunk))))
-
-(deftest render-chart-test
- (testing "renders a simple funnel chart to an SVG"
- (let [{:keys [type content]} (js.protocol/chart
- (node/renderer)
- {:kind "funnel"
- :data [["Visit" 1000] ["Sign up" 700] ["Trial" 300]]
- :settings {:step {:name "Step" :format {}}
- :measure {:format {}}
- :visualization_settings {}}
- :tokenFeatures {}})]
- (is (= "svg" type))
- (is (str/starts-with? content "<svg"))
- (is (str/includes? content "Visit")))))
-
-(deftest cell-background-colors-test
- (testing "computes cell background colors positionally from column formatting rules"
- (is (= ["rgba(255, 0, 0, 0.65)" nil]
- (js.protocol/cell-background-colors
- (node/renderer)
- {:rows [[1] [5]]
- :cols [{:name "test"}]
- :settings {:table.column_formatting [{:columns ["test"]
- :type :single
- :operator "="
- :value 1
- :color "#ff0000"
- :highlight_row false}]}
- :cells [[1 0 "test"] [5 1 "test"]]})))))
diff --git a/test/metabase/sql_parsing/python_test.clj b/test/metabase/sql_parsing/python_test.clj
deleted file mode 100644
index 2cebd795ecf..00000000000
--- a/test/metabase/sql_parsing/python_test.clj
+++ /dev/null
@@ -1,45 +0,0 @@
-(ns metabase.sql-parsing.python-test
- "Tests for the `:python` sqlglot parser: the same assertions the graal parser is trusted for, through
- the external-process path. Requires a `python3` binary on the PATH (sqlglot itself is installed into
- `resources/python-sources` on first use, like the graal tests)."
- (:require
- [clojure.test :refer :all]
- [metabase.sql-parsing.core :as sql-parsing]
- [metabase.sql-parsing.protocol :as protocol]
- [metabase.sql-parsing.python :as python]))
-
-(set! *warn-on-reflection* true)
-
-(deftest referenced-tables-test
- (testing "extracts table references through the external CPython process"
- (is (= [[nil nil "transactions"] [nil "public" "orders"]]
- (protocol/referenced-tables
- (python/parser)
- "postgres"
- "SELECT * FROM transactions t JOIN public.orders o ON t.id = o.tid")))))
-
-(deftest validate-query-test
- (testing "validates syntax through the external CPython process"
- (is (= "ok"
- (:status (protocol/validate-query (python/parser) "postgres" "SELECT 1" nil nil))))
- (is (= "error"
- (:status (protocol/validate-query (python/parser) "postgres" "SELECT FROM WHERE" nil nil))))))
-
-(deftest call-error-keeps-process-healthy-test
- (testing "a failed call surfaces the Python error and the process still answers the next call"
- (is (thrown-with-msg? Exception #"sqlglot call failed"
- (protocol/add-into-clause (python/parser) nil nil "t")))
- (is (= [[nil nil "users"]]
- (protocol/referenced-tables (python/parser) "postgres" "SELECT * FROM users")))))
-
-(deftest parse-error-test
- (testing "unparseable SQL surfaces as the same transport-agnostic parse error the graal parser throws"
- (let [e (try
- (protocol/referenced-tables (python/parser) "postgres" "SELECT !!!")
- (catch Exception e e))]
- (is (sql-parsing/parse-error? e))))
- (testing "other Python-side failures are not parse errors"
- (let [e (try
- (protocol/add-into-clause (python/parser) nil nil "t")
- (catch Exception e e))]
- (is (not (sql-parsing/parse-error? e))))))