The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove unused code (#77613)
Alexander Polyankin669dc959ff65
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 110c7599f1b..6f6f9771d31 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2187,8 +2187,8 @@sql-parsing{:team "DevEx"- :api #{metabase.sql-parsing.core metabase.sql-parsing.init}- :uses #{analytics-interface settings util}}+ :api #{metabase.sql-parsing.core}+ :uses #{analytics-interface util}}sql-tools{:team "DevEx"diff --git a/Dockerfile b/Dockerfileindex a791686b026..02f0ac26170 100644--- a/Dockerfile+++ b/Dockerfile@@ -49,7 +49,7 @@ ENV FC_LANG=en-US LC_CTYPE=en_US.UTF-8COPY bin/docker/DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem# dependencies-RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk java-cacerts nodejs && \+RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk java-cacerts && \apk upgrade && \rm -rf /var/cache/apk/* && \mkdir -p /app/certs && \diff --git a/bin/docker/Dockerfile b/bin/docker/Dockerfileindex a741764aeb1..e5a8483116d 100644--- a/bin/docker/Dockerfile+++ b/bin/docker/Dockerfile@@ -9,7 +9,7 @@ ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA}COPY DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem# dependencies-RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts nodejs python3 && \+RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts && \apk upgrade && \rm -rf /var/cache/apk/* && \mkdir -p /app/certs && \diff --git a/bin/docker/Dockerfile_ubuntu b/bin/docker/Dockerfile_ubuntuindex 56c1513c4ed..c013bb51dfd 100644--- a/bin/docker/Dockerfile_ubuntu+++ b/bin/docker/Dockerfile_ubuntu@@ -11,7 +11,7 @@ COPY DigiCertGlobalRootG2.crt.pem /usr/local/share/ca-certificates/DigiCertGloba# DependenciesRUN apt-get update && \apt-get upgrade -y && \- apt-get install -y ca-certificates ca-certificates-java fonts-noto nodejs python3 && \+ apt-get install -y ca-certificates ca-certificates-java fonts-noto && \apt-get clean && \curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /usr/local/share/ca-certificates/rds-combined-ca-bundle.pem && \update-ca-certificates && \diff --git a/dev/src/dev/readme.md b/dev/src/dev/readme.mdindex 161de30bda3..d860100950a 100644--- a/dev/src/dev/readme.md+++ b/dev/src/dev/readme.md@@ -6,7 +6,7 @@ In this namespace, you can run `(help)` to get a bit of help. It's principle usa### NOTE!-You must build the js bundle used to create the svgs with visx. Run `bun run build-static-viz` to ensure that this bundle is created and the file `resources/frontend_client/app/dist/app-static-viz.bundle.js` exists+You must build the js bundle used to create the svgs with visx. Run `bun run build-static-viz` to ensure that this bundle is created and the file `resources/frontend_client/app/dist/lib-static-viz.bundle.js` existsExample usage below:diff --git a/frontend/lint/module-boundaries.mjs b/frontend/lint/module-boundaries.mjsindex e20d099eec7..0829d0f0161 100644--- a/frontend/lint/module-boundaries.mjs+++ b/frontend/lint/module-boundaries.mjs@@ -250,7 +250,6 @@ const elements = ["frontend/src/metabase/app-embed.ts","frontend/src/metabase/app-public.ts","frontend/src/metabase/app-static-viz.ts",- "frontend/src/metabase/app-static-viz-cli.ts","frontend/src/metabase/AppComponent.tsx","frontend/src/metabase/App.styled.tsx","frontend/src/metabase/AppKBarProvider.tsx",diff --git a/frontend/src/metabase/app-static-viz-cli.ts b/frontend/src/metabase/app-static-viz-cli.tsdeleted file mode 100644index d447ea188ca..00000000000--- a/frontend/src/metabase/app-static-viz-cli.ts+++ /dev/null@@ -1,60 +0,0 @@-// Entry point of the self-contained Node bundle (`app-static-viz-cli.bundle.js`) run by the `:node`-// static-viz renderer (see `metabase.channel.render.js.node`). It answers newline-delimited JSON render-// requests over stdin/stdout.-//-// ESM imports are hoisted above statements but execute in import order, so the console redirect lives in-// its own dependency-free module imported first: it must run *before* the viz code loads — stdout carries-// the line protocol, so anything the bundle logs during init has to go to stderr instead.-import "metabase/static-viz/console";-import "metabase/static-viz/polyfill";--import readline from "readline";--import {- type CellBackgroundColorsInput,- type RenderChartInput,- getCellBackgroundColors,- renderChart,-} from "metabase/static-viz";--type RenderChartData = {- fn: "renderChart";- arg: RenderChartInput;-};--type GetCellBackgroundColorsData = {- fn: "getCellBackgroundColors";- arg: CellBackgroundColorsInput;-};--type LineData = RenderChartData | GetCellBackgroundColorsData;--const rl = readline.createInterface({ input: process.stdin });-rl.on("line", (line: string) => {- const lineClean = line.trim();- if (!lineClean) {- return;- }- let out;- try {- const lineData: LineData = JSON.parse(lineClean);- const { fn, arg } = lineData;- if (fn === "renderChart") {- out = JSON.stringify({ ok: true, result: renderChart(arg) });- } else if (fn === "getCellBackgroundColors") {- out = JSON.stringify({ ok: true, result: getCellBackgroundColors(arg) });- } else {- throw new Error(`unknown function: ${fn}`);- }- } catch (e) {- out = JSON.stringify({- ok: false,- error: String(e instanceof Error ? (e.stack ?? e) : e),- });- }- process.stdout.write(out + "\n");-});--rl.on("close", () => process.exit(0));--process.stdout.write(JSON.stringify({ ready: true }) + "\n");diff --git a/frontend/src/metabase/static-viz/console.ts b/frontend/src/metabase/static-viz/console.tsdeleted file mode 100644index af3b5db1a6a..00000000000--- a/frontend/src/metabase/static-viz/console.ts+++ /dev/null@@ -1,17 +0,0 @@-// Imported FIRST by the node CLI entry (`app-static-viz-cli.ts`): stdout carries the render line-// protocol, so console output must be routed to stderr before any static-viz module gets a chance to-// log during initialization. This module has no imports of its own, so its body is guaranteed to run-// before every other module in the bundle (ESM imports are hoisted, but execute in import order).-/* eslint-disable no-console */--for (const method of [- "log",- "info",- "warn",- "error",- "debug",- "trace",-] as const) {- console[method] = (...args: unknown[]) =>- process.stderr.write(args.map((a) => String(a)).join(" ") + "\n");-}diff --git a/resources/python-sources/sql_tools_cli.py b/resources/python-sources/sql_tools_cli.pydeleted file mode 100644index 406894a7f25..00000000000--- a/resources/python-sources/sql_tools_cli.py+++ /dev/null@@ -1,43 +0,0 @@-"""Newline-delimited JSON stdin/stdout server around sql_tools, for running sqlglot on a native-CPython instead of GraalPy. Spawned by metabase.sql-parsing.python with this directory (or a zip of-it) on PYTHONPATH; prints one ready line, then answers one response line per request line:-- {"fn": "referenced_tables", "args": ["SELECT 1", "postgres"]}- -> {"ok": true, "result": "..."} | {"ok": false, "error": "..."}-"""--import json-import sys--import sql_tools---def _resolve(fn_name):- if fn_name.startswith("_"):- raise ValueError(f"unknown sql_tools function: {fn_name}")- fn = getattr(sql_tools, fn_name, None)- if not callable(fn):- raise ValueError(f"unknown sql_tools function: {fn_name}")- return fn---def main():- out = sys.stdout- out.write(json.dumps({"ready": True}) + "\n")- out.flush()- for line in sys.stdin:- line = line.strip()- if not line:- continue- try:- request = json.loads(line)- result = _resolve(request["fn"])(*request["args"])- response = {"ok": True, "result": result}- except Exception as e: # noqa: BLE001 -- any failure is reported to the JVM side- response = {"ok": False, "error": f"{type(e).__name__}: {e}"}- out.write(json.dumps(response) + "\n")- out.flush()---if __name__ == "__main__":- main()diff --git a/rspack.static-viz.config.js b/rspack.static-viz.config.jsindex 0bd5da16c07..74b09fc6d1c 100644--- a/rspack.static-viz.config.js+++ b/rspack.static-viz.config.js@@ -20,94 +20,8 @@ const ENTERPRISE_SRC_PATH =const devMode = WEBPACK_BUNDLE !== "production";-const moduleConfig = {- rules: [- {- test: /\.css$/i,- use: "null-loader",- },- {- test: /\.(tsx?|jsx?)$/,- exclude: /node_modules|cljs|css\/core\/fonts\.styled\.ts/,- use: [- {- loader: "builtin:swc-loader",- options: {- jsc: {- loose: true,- transform: {- react: {- runtime: "automatic",- refresh: false,- },- },- parser: {- syntax: "typescript",- tsx: true,- },- experimental: {- plugins: [["@swc/plugin-emotion", { sourceMap: devMode }]],- },- },-- sourceMaps: false,- minify: true,- env: {- targets: ["defaults"],- },- },- },- ],- },- {- test: /\.svg/,- type: "asset/source",- resourceQuery: /source/, // *.svg?source- },- {- test: /\.svg$/i,- issuer: /\.[jt]sx?$/,- resourceQuery: /component/, // *.svg?component- use: [- {- loader: "@svgr/webpack",- options: {- ref: true,- svgoConfig: SVGO_CONFIG,- },- },- ],- },- {- test: /\.svg$/,- type: "asset/resource",- resourceQuery: { not: [/component|source/] },- },- ],-};--const resolveAlias = {- assets: ASSETS_PATH,- metabase: SRC_PATH,- "metabase-enterprise": ENTERPRISE_SRC_PATH,- cljs: devMode ? CLJS_SRC_PATH_DEV : CLJS_SRC_PATH,- "metabase-lib": LIB_SRC_PATH,- "metabase-types": TYPES_SRC_PATH,- embedding: EMBEDDING_SRC_PATH,- "embedding-sdk-bundle": SDK_BUNDLE_SRC_PATH,- "embedding-sdk-shared": SDK_SHARED_SRC_PATH,- "process/browser": require.resolve("process/browser"),- "ee-overrides":- process.env.MB_EDITION === "ee"- ? ENTERPRISE_SRC_PATH + "/static-viz-overrides"- : SRC_PATH + "/utils/noop",-};--const resolveExtensions = [".web.js", ".js", ".jsx", ".ts", ".tsx"];-module.exports = (env) => {- const graalConfig = {- name: "graal",+ return {mode: "production",context: SRC_PATH,@@ -116,7 +30,7 @@ module.exports = (env) => {},entry: {- "app-static-viz": {+ "lib-static-viz": {import: "./app-static-viz.ts",library: {name: "MetabaseStaticViz",@@ -132,10 +46,89 @@ module.exports = (env) => {globalObject: "globalThis",},- module: moduleConfig,+ module: {+ rules: [+ {+ test: /\.css$/i,+ use: "null-loader",+ },+ {+ test: /\.(tsx?|jsx?)$/,+ exclude: /node_modules|cljs|css\/core\/fonts\.styled\.ts/,+ use: [+ {+ loader: "builtin:swc-loader",+ options: {+ jsc: {+ loose: true,+ transform: {+ react: {+ runtime: "automatic",+ refresh: false,+ },+ },+ parser: {+ syntax: "typescript",+ tsx: true,+ },+ experimental: {+ plugins: [["@swc/plugin-emotion", { sourceMap: devMode }]],+ },+ },++ sourceMaps: false,+ minify: true,+ env: {+ targets: ["defaults"],+ },+ },+ },+ ],+ },+ {+ test: /\.svg/,+ type: "asset/source",+ resourceQuery: /source/, // *.svg?source+ },+ {+ test: /\.svg$/i,+ issuer: /\.[jt]sx?$/,+ resourceQuery: /component/, // *.svg?component+ use: [+ {+ loader: "@svgr/webpack",+ options: {+ ref: true,+ svgoConfig: SVGO_CONFIG,+ },+ },+ ],+ },+ {+ test: /\.svg$/,+ type: "asset/resource",+ resourceQuery: { not: [/component|source/] },+ },+ ],+ },resolve: {- extensions: resolveExtensions,- alias: resolveAlias,+ extensions: [".web.js", ".js", ".jsx", ".ts", ".tsx"],+ alias: {+ assets: ASSETS_PATH,+ metabase: SRC_PATH,+ "metabase-enterprise": ENTERPRISE_SRC_PATH,+ cljs: devMode ? CLJS_SRC_PATH_DEV : CLJS_SRC_PATH,+ "metabase-lib": LIB_SRC_PATH,+ "metabase-types": TYPES_SRC_PATH,+ embedding: EMBEDDING_SRC_PATH,+ "embedding-sdk-bundle": SDK_BUNDLE_SRC_PATH,+ "embedding-sdk-shared": SDK_SHARED_SRC_PATH,+ "process/browser": require.resolve("process/browser"),+ "ee-overrides":+ process.env.MB_EDITION === "ee"+ ? ENTERPRISE_SRC_PATH + "/static-viz-overrides"+ : SRC_PATH + "/utils/noop",+ },fallback: {crypto: require.resolve("crypto-browserify"),stream: require.resolve("stream-browserify"),@@ -179,46 +172,9 @@ module.exports = (env) => {).map((module) =>module.nameForCondition.replace(`${__dirname}/`, ""),- )- .concat(["frontend/src/metabase/app-static-viz-cli.ts"]),+ ),}),}),],};-- const nodeCliConfig = {- name: "node-cli",- mode: "production",- context: SRC_PATH,- target: "node",-- performance: {- hints: false,- },-- entry: {- "app-static-viz-cli": "./app-static-viz-cli.ts",- },-- output: {- path: BUILD_PATH + "/app/dist",- filename: "[name].bundle.js",- },-- module: moduleConfig,- resolve: {- extensions: resolveExtensions,- alias: resolveAlias,- },- optimization: {- minimize: true,- },- plugins: [- new rspack.EnvironmentPlugin({- IS_EMBEDDING_SDK_BUILD: false,- }),- ],- };-- return [graalConfig, nodeCliConfig];};diff --git a/src/metabase/channel/render/js/common.clj b/src/metabase/channel/render/js/common.cljindex 810ccfaeae9..41b318f6fce 100644--- a/src/metabase/channel/render/js/common.clj+++ b/src/metabase/channel/render/js/common.clj@@ -1,7 +1,6 @@(ns metabase.channel.render.js.common- "Shared helpers for the pooled static-viz renderers ([[metabase.channel.render.js.graal]] and- [[metabase.channel.render.js.node]]): the graal bundle's classpath path, the test-init guard, and the- dirigiste worker-pool factory they both use."+ "Shared helpers for pooled static-viz renderers (see [[metabase.channel.render.js.graal]]): the graal+ bundle's classpath path, the test-init guard, and the dirigiste worker-pool factory."(:require[metabase.config.core :as config])(:import@@ -11,10 +10,8 @@(set! *warn-on-reflection* true)(def bundle-resource-path- "Classpath path of the built static-viz bundle the graal renderer evaluates in-process. (The node- renderer runs its own self-contained bundle from disk instead — see- [[metabase.channel.render.js.node]].)"- "frontend_client/app/dist/app-static-viz.bundle.js")+ "Classpath path of the built static-viz bundle the graal renderer evaluates in-process."+ "frontend_client/app/dist/lib-static-viz.bundle.js")(defn assert-tests-not-initializing!"Guard against loading the static-viz bundle as a side effect of loading namespaces: it might not havediff --git a/src/metabase/channel/render/js/node.clj b/src/metabase/channel/render/js/node.cljdeleted file mode 100644index 021855fecdd..00000000000--- a/src/metabase/channel/render/js/node.clj+++ /dev/null@@ -1,182 +0,0 @@-(ns metabase.channel.render.js.node- "The `:node` [[metabase.channel.render.js.protocol/StaticVizRenderer]]: runs the static-viz JS in a pool- of external Node.js child processes rather than in-process on GraalVM. Each process runs the- self-contained node entrypoint (`app-static-viz-cli.bundle.js`, built by `rspack.static-viz.config.js`),- which answers newline-delimited JSON render requests over stdin/stdout.-- The script is written to a fresh temp file from the classpath at every process spawn, executed- immediately, and deleted when the process is destroyed. So what `node` executes always comes straight- out of the jar — there is no long-lived on-disk copy to drift out of sync or be tampered with, and no- deploy-time provisioning.-- The pool holds a single process, held exclusively per render (so renders serialize onto it); with a min- of 0 it shrinks to 0 when idle, killing the process after 1 minute with no work. We cap it at one- process — spawning a fresh `node` (runtime init + bundle load) is costly, and a warm process renders- fast enough that queuing a second render behind it beats paying that startup cost for concurrency.-- Requires a `node` binary on the host's PATH."- (:require- [clojure.java.io :as io]- [metabase.channel.render.js.common :as common]- [metabase.channel.render.js.protocol :as js.protocol]- [metabase.util.json :as json])- (:import- (io.aleph.dirigiste Pool)- (java.io BufferedReader BufferedWriter File IOException InputStreamReader OutputStreamWriter)- (java.lang ProcessBuilder$Redirect)- (java.nio.charset StandardCharsets)- (java.nio.file Files)- (java.nio.file.attribute FileAttribute)))--(set! *warn-on-reflection* true)--;;; ------------------------------------------------ process ------------------------------------------------(def ^:private render-timeout-ms- "How long to wait for a render process to answer before assuming it has wedged: we kill it and fail the- render, so a stuck process can't hold its pool slot forever."- (* 60 1000))--(defrecord ^:private NodeProcess [^Process process ^BufferedWriter writer ^BufferedReader reader ^File script-file])--(defn- read-response-line- "Read one line from the process, killing it and throwing if it doesn't answer within- [[render-timeout-ms]] (so a wedged process can't hold its pool slot forever). `destroyForcibly`- (SIGKILL) rather than the polite `destroy`: a process wedged hard enough to miss the deadline may- ignore SIGTERM, and killing it also closes its stdout, which unblocks the reader future."- ^String [^BufferedReader reader ^Process process]- (let [fut (future (.readLine reader))- line (deref fut render-timeout-ms ::timeout)]- (if (identical? ::timeout line)- (do (.destroyForcibly process)- (future-cancel fut)- (throw (ex-info "static-viz node render timed out" {})))- line)))--(def ^:private script-resource-path- "Classpath path of the node entrypoint bundle written to a temp file at each process spawn."- "frontend_client/app/dist/app-static-viz-cli.bundle.js")--(defn- create-script-file!- "Write the node entrypoint bundle from the classpath to a fresh temp file, to be executed immediately- and deleted when its process is destroyed (`deleteOnExit` is only a backstop for JVM death)."- ^File []- (let [resource (io/resource script-resource-path)]- (when (nil? resource)- (throw (ex-info (str "static-viz resource not found: " script-resource-path- " — build it with `bun run build-static-viz`")- {:resource script-resource-path})))- (let [file (.toFile (Files/createTempFile "mb-static-viz" ".js" (into-array FileAttribute [])))]- (.deleteOnExit file)- (with-open [in (io/input-stream resource)]- (io/copy in file))- file)))--(defn- start-process!- "Write the script to a temp file, spawn `node` on it, and block until it reports it has loaded. The- process's stderr is inherited (for debugging); stdout carries the line protocol. On any failure to- reach a ready state the process is destroyed and the file deleted, so nothing leaks."- ^NodeProcess []- (common/assert-tests-not-initializing!)- (let [script (create-script-file!)- ^Process process (try- (.. (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String ["node" (.getAbsolutePath script)]))- (redirectError ProcessBuilder$Redirect/INHERIT)- (start))- (catch Throwable t- (.delete script)- (if (instance? IOException t)- (throw (ex-info (str "could not start `node` for static-viz rendering — is node"- " installed and on PATH? (otherwise set MB_STATIC_VIZ_MODE"- " to graalvm)")- {} t))- (throw t))))]- (try- (let [writer (BufferedWriter. (OutputStreamWriter. (.getOutputStream process) StandardCharsets/UTF_8))- reader (BufferedReader. (InputStreamReader. (.getInputStream process) StandardCharsets/UTF_8))- ready (read-response-line reader process)]- (when-not (some-> ready json/decode+kw :ready)- (throw (ex-info "static-viz node process failed to start" {:line ready})))- (->NodeProcess process writer reader script))- (catch Throwable t- (.destroy process)- (.delete script)- (throw t)))))--(defn- stop-process!- [{:keys [^Process process ^BufferedWriter writer ^BufferedReader reader ^File script-file]}]- (try (.close writer) (catch Exception _))- (try (.close reader) (catch Exception _))- (try (.destroy process) (catch Exception _))- (try (.delete script-file) (catch Exception _)))--(defn- render-on-process- "Send one render request to `node-process` and read its response. `arg` is a Clojure data structure sent- as a JS object (not a JSON string), so the bundle serializes only once — at this process boundary rather- than again inside the bundle. Returns the render result parsed into Clojure data. Not thread-safe on its- own — the pool hands a process to one render at a time. A write failure (broken pipe) means the pooled- process had already died; it is thrown as `:retryable` so [[call-node]] can get a fresh one."- [{:keys [^BufferedWriter writer ^BufferedReader reader ^Process process]} fn-name arg]- (try- (.write writer ^String (str (json/encode {:fn fn-name :arg arg}) "\n"))- (.flush writer)- (catch IOException e- (throw (ex-info "static-viz node process is not writable" {:retryable true} e))))- (let [line (read-response-line reader process)]- (when (nil? line)- (throw (ex-info "static-viz node process closed unexpectedly" {:retryable true})))- (let [{:keys [ok result error]} (json/decode+kw line)]- (if ok- result- ;; :render-error — the process answered the protocol correctly, so it is healthy; only this- ;; render failed. [[call-node]] releases (rather than disposes) the process on these.- (throw (ex-info (str "static-viz node render failed: " error) {:render-error true}))))))--;;; ------------------------------------------------ pool ---------------------------------------------------(def ^:private pool-key- "Dirigiste pools are keyed; the key is arbitrary, it just has to be the same for every operation."- :static-viz-node)--(def ^:private ^Pool node-process-pool- "A pool of a single static-viz Node.js process, held exclusively per render (so renders serialize onto- it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because a- warm process is fast and spawning another is costly. See- [[metabase.channel.render.js.common/create-pool]]."- (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 1}))--(defn- call-node- "Run static-viz bundle function `fn-name` with `arg` (a Clojure data structure) on a pooled Node process,- returning the render result parsed into Clojure data. A `:render-error` (the process answered `ok:- false` — it is healthy, only the render failed) releases the process back for reuse, like the graal- renderer does with its context; any other failure may have left the process in a bad state, so it is- disposed (dropped from the pool). Retries once if the acquired process turned out to be already dead- (e.g. reaped or killed while parked in the pool), since a fresh one should succeed."- [fn-name arg]- (loop [retries 1]- (let [node-process (.acquire node-process-pool pool-key)- outcome (try- (let [result (render-on-process node-process fn-name arg)]- (.release node-process-pool pool-key node-process)- {:result result})- (catch Throwable t- (if (:render-error (ex-data t))- (.release node-process-pool pool-key node-process)- (.dispose node-process-pool pool-key node-process))- (if (and (:retryable (ex-data t)) (pos? retries))- {:retry true}- (throw t))))]- (if (:retry outcome)- (recur (dec retries))- (:result outcome)))))--(defn renderer- "The `:node` [[metabase.channel.render.js.protocol/StaticVizRenderer]] — runs the static-viz JS in a pool- of external Node.js processes. Passes each `input` map to the bundle's object-in/object-out functions, so- it is serialized once (at the process boundary) rather than JSON-encoded again inside the bundle."- []- (reify js.protocol/StaticVizRenderer- (chart [_ input]- (call-node "renderChart" input))- (cell-background-colors [_ input]- (call-node "getCellBackgroundColors" input))))diff --git a/src/metabase/channel/render/js/renderer.clj b/src/metabase/channel/render/js/renderer.cljindex 655a46d8f9f..4e290248b00 100644--- a/src/metabase/channel/render/js/renderer.clj+++ b/src/metabase/channel/render/js/renderer.clj@@ -1,19 +1,14 @@(ns metabase.channel.render.js.renderer- "Hands callers the static-viz [[metabase.channel.render.js.protocol/StaticVizRenderer]] implementation- for the configured [[metabase.channel.settings/static-viz-mode]]: the in-process GraalVM renderer- (default) or a pool of external Node.js processes. Kept behind the protocol so the callers+ "Hands callers the static-viz [[metabase.channel.render.js.protocol/StaticVizRenderer]] implementation:+ the in-process GraalVM renderer. Kept behind the protocol so the callers([[metabase.channel.render.js.svg]], [[metabase.channel.render.js.color]]) don't care which backendruns."(:require- [metabase.channel.render.js.graal :as graal]- [metabase.channel.render.js.node :as node]- [metabase.channel.settings :as channel.settings]))+ [metabase.channel.render.js.graal :as graal]))(set! *warn-on-reflection* true)(defn renderer- "The static-viz renderer for the configured mode."+ "The static-viz renderer."[]- (if (= :node (channel.settings/static-viz-mode))- (node/renderer)- (graal/renderer)))+ (graal/renderer))diff --git a/src/metabase/channel/settings.clj b/src/metabase/channel/settings.cljindex 6adbc7edd82..b09b0bc6943 100644--- a/src/metabase/channel/settings.clj+++ b/src/metabase/channel/settings.clj@@ -337,17 +337,3 @@:getter (fn [] (boolean (slack-app-token))):export? false:setter :none)--(defsetting static-viz-mode- "How static visualizations (subscription/alert charts and pulse table cell colors) are rendered:- `graalvm` runs the JavaScript in-process on a pooled GraalVM context (default), or `node` runs it in a- pool of external Node.js child processes (requires a `node` binary on the host's PATH)."- :type :keyword- :visibility :internal- :default :graalvm- :export? false- :setter (fn [new-value]- (when (some? new-value)- (assert (#{:graalvm :node} (keyword new-value))- (tru "Invalid static-viz-mode! Only values of graalvm and node are allowed.")))- (setting/set-value-of-type! :keyword :static-viz-mode new-value)))diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.cljindex c50465fd643..5d7a3752330 100644--- a/src/metabase/core/init.clj+++ b/src/metabase/core/init.clj@@ -64,7 +64,6 @@[metabase.settings.init][metabase.setup.init][metabase.slackbot.init]- [metabase.sql-parsing.init][metabase.sql-tools.init][metabase.sso.init][metabase.store-api.init]diff --git a/src/metabase/sql_parsing/common.clj b/src/metabase/sql_parsing/common.cljindex faf2cd0383f..af213aa2725 100644--- a/src/metabase/sql_parsing/common.clj+++ b/src/metabase/sql_parsing/common.clj@@ -1,8 +1,7 @@(ns metabase.sql-parsing.common- "Shared helpers for the pooled sqlglot parsers ([[metabase.sql-parsing.graal]] and- [[metabase.sql-parsing.python]]): the python-sources locations, the lazy dev sqlglot install, the- protocol reify over a transport function, and the dirigiste worker-pool factory (copied from- [[metabase.channel.render.js.common]])."+ "Shared helpers for pooled sqlglot parsers (see [[metabase.sql-parsing.graal]]): the python-sources+ locations, the lazy dev sqlglot install, the protocol reify over a transport function, and the+ dirigiste worker-pool factory (copied from [[metabase.channel.render.js.common]])."(:require[clojure.java.io :as io][clojure.java.shell :as shell]diff --git a/src/metabase/sql_parsing/init.clj b/src/metabase/sql_parsing/init.cljdeleted file mode 100644index 88bded2543e..00000000000--- a/src/metabase/sql_parsing/init.clj+++ /dev/null@@ -1,3 +0,0 @@-(ns metabase.sql-parsing.init- (:require- [metabase.sql-parsing.settings]))diff --git a/src/metabase/sql_parsing/parser.clj b/src/metabase/sql_parsing/parser.cljindex cad9dbe356e..02bfa71ff36 100644--- a/src/metabase/sql_parsing/parser.clj+++ b/src/metabase/sql_parsing/parser.clj@@ -1,18 +1,13 @@(ns metabase.sql-parsing.parser- "Hands callers the [[metabase.sql-parsing.protocol/SqlParser]] implementation for the configured- [[metabase.sql-parsing.settings/sql-parsing-mode]]: the in-process GraalPy parser (default) or a pool- of external native CPython processes. Kept behind the protocol so the caller- ([[metabase.sql-parsing.core]]) doesn't care which backend runs."+ "Hands callers the [[metabase.sql-parsing.protocol/SqlParser]] implementation: the in-process GraalPy+ parser. Kept behind the protocol so the caller ([[metabase.sql-parsing.core]]) doesn't care which+ backend runs."(:require- [metabase.sql-parsing.graal :as graal]- [metabase.sql-parsing.python :as python]- [metabase.sql-parsing.settings :as settings]))+ [metabase.sql-parsing.graal :as graal]))(set! *warn-on-reflection* true)(defn parser- "The sqlglot parser for the configured mode."+ "The sqlglot parser."[]- (if (= :python (settings/sql-parsing-mode))- (python/parser)- (graal/parser)))+ (graal/parser))diff --git a/src/metabase/sql_parsing/python.clj b/src/metabase/sql_parsing/python.cljdeleted file mode 100644index 0776da7e92b..00000000000--- a/src/metabase/sql_parsing/python.clj+++ /dev/null@@ -1,199 +0,0 @@-(ns metabase.sql-parsing.python- "The `:python` [[metabase.sql-parsing.protocol/SqlParser]]: runs sqlglot in a pool of external native- CPython child processes rather than in-process on GraalPy. Each process runs `sql_tools_cli.py`, which- answers newline-delimited JSON requests over stdin/stdout.-- The Python sources come straight out of the classpath: in dev the child imports directly from- `resources/python-sources`; from the jar they are written to a fresh temp zip at every process spawn- (CPython imports from zips via zipimport), put on the child's PYTHONPATH, and deleted when the process- is destroyed — so there is no long-lived on-disk copy to drift out of sync or be tampered with, and no- deploy-time provisioning.-- The pool holds a single process, held exclusively per call (so calls serialize onto it); with a min of- 0 it shrinks to 0 when idle, killing the process after 10 minutes with no work. We cap it at one- process — spawning a fresh `python3` (interpreter init + sqlglot import) is costly, and a warm process- parses fast enough that queuing a second call behind it beats paying that startup cost for- concurrency.-- Requires a `python3` binary on the host's PATH."- (:require- [clojure.java.io :as io]- [clojure.string :as str]- [metabase.sql-parsing.common :as common]- [metabase.util.files :as u.files]- [metabase.util.json :as json])- (:import- (io.aleph.dirigiste Pool)- (java.io BufferedReader BufferedWriter File IOException InputStreamReader OutputStreamWriter)- (java.lang ProcessBuilder$Redirect)- (java.nio.charset StandardCharsets)- (java.nio.file FileVisitOption Files LinkOption)- (java.nio.file.attribute FileAttribute)- (java.util.concurrent TimeoutException)- (java.util.zip ZipEntry ZipOutputStream)))--(set! *warn-on-reflection* true)--;;; ------------------------------------------------ process ------------------------------------------------(def ^:private ^:const call-timeout-ms- "How long to wait for the process to answer before assuming it has wedged: we kill it and fail the- call, so a stuck process can't hold its pool slot forever."- 30000)--(defrecord ^:private PythonProcess [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip])--(defn- read-response-line- "Read one line from the process, killing it and throwing if it doesn't answer within- [[call-timeout-ms]] (so a wedged process can't hold its pool slot forever). `destroyForcibly`- (SIGKILL) rather than the polite `destroy`: a process wedged hard enough to miss the deadline may- ignore SIGTERM, and killing it also closes its stdout, which unblocks the reader future."- ^String [^BufferedReader reader ^Process process]- (let [fut (future (.readLine reader))- line (deref fut call-timeout-ms ::timeout)]- (if (identical? ::timeout line)- (do (.destroyForcibly process)- (future-cancel fut)- (throw (TimeoutException. (str "Python execution timed out after " call-timeout-ms "ms"))))- line)))--(defn- create-sources-zip!- "Write the python-sources tree from the jar to a fresh temp zip, importable by the child via zipimport,- to be put on its PYTHONPATH and deleted when its process is destroyed (`deleteOnExit` is only a- backstop for JVM death)."- ^File []- (let [file (.toFile (Files/createTempFile "mb-sql-tools" ".zip" (into-array FileAttribute [])))]- (.deleteOnExit file)- (try- (with-open [jar-fs (u.files/nio-fs (u.files/get-jar-path))- zos (ZipOutputStream. (io/output-stream file))]- (let [root (.getPath jar-fs (str "/" common/python-sources-resource) (into-array String []))]- (with-open [paths (Files/walk root (into-array FileVisitOption []))]- (doseq [^java.nio.file.Path path (iterator-seq (.iterator paths))- :when (and (not (Files/isDirectory path (into-array LinkOption [])))- (not (str/includes? (str path) "__pycache__")))]- (.putNextEntry zos (ZipEntry. (str (.relativize root path))))- (Files/copy path zos)- (.closeEntry zos)))))- file- (catch Throwable t- (.delete file)- (throw t)))))--(defn- python-path- "The PYTHONPATH entry for a new child process, plus the temp zip to delete with it (nil in dev, where- the child imports straight from `resources/python-sources`)."- []- (if (common/jar-resource? common/python-sources-resource)- (let [zip (create-sources-zip!)]- {:path (.getAbsolutePath zip) :sources-zip zip})- (do- (common/ensure-sqlglot-installed!)- {:path common/dev-python-sources-dir :sources-zip nil})))--(defn- start-process!- "Spawn `python3 -m sql_tools_cli` with the Python sources on PYTHONPATH and block until it reports it- has loaded. The process's stderr is inherited (for debugging); stdout carries the line protocol. On- any failure to reach a ready state the process is destroyed and the temp zip deleted, so nothing- leaks."- ^PythonProcess []- (let [{:keys [path ^File sources-zip]} (python-path)- ^Process process (try- (let [builder (.. (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String ["python3" "-m" "sql_tools_cli"]))- (redirectError ProcessBuilder$Redirect/INHERIT))]- (doto (.environment builder)- (.put "PYTHONPATH" path)- (.put "PYTHONUNBUFFERED" "1"))- (.start builder))- (catch Throwable t- (some-> sources-zip .delete)- (if (instance? IOException t)- (throw (ex-info (str "could not start `python3` for SQL parsing — is python3"- " installed and on PATH? (otherwise set MB_SQL_PARSING_MODE"- " to graalvm)")- {} t))- (throw t))))]- (try- (let [writer (BufferedWriter. (OutputStreamWriter. (.getOutputStream process) StandardCharsets/UTF_8))- reader (BufferedReader. (InputStreamReader. (.getInputStream process) StandardCharsets/UTF_8))- ready (read-response-line reader process)]- (when-not (some-> ready json/decode+kw :ready)- (throw (ex-info "sql-parsing python process failed to start" {:line ready})))- (->PythonProcess process writer reader sources-zip))- (catch Throwable t- (.destroy process)- (some-> sources-zip .delete)- (throw t)))))--(defn- stop-process!- [{:keys [^Process process ^BufferedWriter writer ^BufferedReader reader ^File sources-zip]}]- (try (.close writer) (catch Exception _))- (try (.close reader) (catch Exception _))- (try (.destroy process) (catch Exception _))- (some-> sources-zip .delete))--(defn- call-on-process- "Send one request to `python-process` and read its response. Returns the `sql_tools` function's result- string. Not thread-safe on its own — the pool hands a process to one call at a time. A write failure- (broken pipe) means the pooled process had already died; it is thrown as `:retryable` so [[call-python]]- can get a fresh one."- ^String [{:keys [^BufferedWriter writer ^BufferedReader reader ^Process process]} fn-name args]- (try- (.write writer ^String (str (json/encode {:fn fn-name :args args}) "\n"))- (.flush writer)- (catch IOException e- (throw (ex-info "sql-parsing python process is not writable" {:retryable true} e))))- (let [line (read-response-line reader process)]- (when (nil? line)- (throw (ex-info "sql-parsing python process closed unexpectedly" {:retryable true})))- (let [{:keys [ok result error]} (json/decode+kw line)]- (if ok- result- ;; The process answered the protocol correctly, so it is healthy; only this call failed.- ;; [[call-python]] releases (rather than disposes) the process on these.- (throw (common/call-failed-ex error))))))--;;; ------------------------------------------------ pool ---------------------------------------------------(def ^:private pool-key- "Dirigiste pools are keyed; the key is arbitrary, it just has to be the same for every operation."- :sql-parsing-python)--(def ^:private ^Pool python-process-pool- "A pool of a single sql-parsing CPython process, held exclusively per call (so calls serialize onto- it); when idle it shrinks to 0 and the generator's `destroy` kills the process. Capped at one because- a warm process is fast and spawning another is costly. See- [[metabase.sql-parsing.common/create-pool]]."- (common/create-pool start-process! stop-process! {:max-size 1, :idle-minutes 10}))--(defn- call-python- "Execute `sql_tools.<fn-name>` with `args` on a pooled CPython process and return the result as a- string. A Python-side failure (the process answered `ok: false` — it is healthy, only the call- failed, thrown as the transport-agnostic [[metabase.sql-parsing.common/call-failed-ex]]) releases the- process back for reuse, like the graal parser does with its context; any other failure may have left- the process in a bad state, so it is disposed (dropped from the pool). Retries once if the acquired- process turned out to be already dead (e.g. reaped or killed while parked in the pool), since a fresh- one should succeed."- ^String [fn-name & args]- (loop [retries 1]- (let [python-process (.acquire python-process-pool pool-key)- outcome (try- (let [result (call-on-process python-process fn-name args)]- (.release python-process-pool pool-key python-process)- {:result result})- (catch Throwable t- (if (:sql-parsing/error (ex-data t))- (.release python-process-pool pool-key python-process)- (.dispose python-process-pool pool-key python-process))- (if (and (:retryable (ex-data t)) (pos? retries))- {:retry true}- (throw t))))]- (if (:retry outcome)- (recur (dec retries))- (:result outcome)))))--(defn parser- "The `:python` [[metabase.sql-parsing.protocol/SqlParser]] — runs sqlglot in a pool of external native- CPython processes."- []- (common/make-parser call-python))diff --git a/src/metabase/sql_parsing/settings.clj b/src/metabase/sql_parsing/settings.cljdeleted file mode 100644index 476e8bbb435..00000000000--- a/src/metabase/sql_parsing/settings.clj+++ /dev/null@@ -1,18 +0,0 @@-(ns metabase.sql-parsing.settings- (:require- [metabase.settings.core :as setting :refer [defsetting]]- [metabase.util.i18n :refer [deferred-tru tru]]))--(set! *warn-on-reflection* true)--(defsetting sql-parsing-mode- (deferred-tru "How SQL parsing (sqlglot) runs: `graalvm` runs it in-process on pooled GraalPy contexts (default), or `python` runs it in a pool of external native CPython child processes (requires a `python3` binary on the host PATH).")- :type :keyword- :visibility :internal- :default :graalvm- :export? false- :setter (fn [new-value]- (when (some? new-value)- (assert (#{:graalvm :python} (keyword new-value))- (tru "Invalid sql-parsing-mode! Only values of graalvm and python are allowed.")))- (setting/set-value-of-type! :keyword :sql-parsing-mode new-value)))diff --git a/test/metabase/channel/render/js/node_test.clj b/test/metabase/channel/render/js/node_test.cljdeleted file mode 100644index 5012b26d6d0..00000000000--- a/test/metabase/channel/render/js/node_test.clj+++ /dev/null@@ -1,46 +0,0 @@-(ns metabase.channel.render.js.node-test- "Tests for the `:node` static-viz renderer: the same assertions the graal renderer is trusted for,- through the external-process path. Requires a built static-viz bundle (like the graal tests) plus a- `node` binary on the PATH."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase.channel.render.js.node :as node]- [metabase.channel.render.js.protocol :as js.protocol]))--(set! *warn-on-reflection* true)--(use-fixtures :each- (fn warn-possible-rebuild- [thunk]- (testing "[PRO TIP] If this test fails, you may need to rebuild the bundle with `bun run build-static-viz`\n"- (thunk))))--(deftest render-chart-test- (testing "renders a simple funnel chart to an SVG"- (let [{:keys [type content]} (js.protocol/chart- (node/renderer)- {:kind "funnel"- :data [["Visit" 1000] ["Sign up" 700] ["Trial" 300]]- :settings {:step {:name "Step" :format {}}- :measure {:format {}}- :visualization_settings {}}- :tokenFeatures {}})]- (is (= "svg" type))- (is (str/starts-with? content "<svg"))- (is (str/includes? content "Visit")))))--(deftest cell-background-colors-test- (testing "computes cell background colors positionally from column formatting rules"- (is (= ["rgba(255, 0, 0, 0.65)" nil]- (js.protocol/cell-background-colors- (node/renderer)- {:rows [[1] [5]]- :cols [{:name "test"}]- :settings {:table.column_formatting [{:columns ["test"]- :type :single- :operator "="- :value 1- :color "#ff0000"- :highlight_row false}]}- :cells [[1 0 "test"] [5 1 "test"]]})))))diff --git a/test/metabase/sql_parsing/python_test.clj b/test/metabase/sql_parsing/python_test.cljdeleted file mode 100644index 2cebd795ecf..00000000000--- a/test/metabase/sql_parsing/python_test.clj+++ /dev/null@@ -1,45 +0,0 @@-(ns metabase.sql-parsing.python-test- "Tests for the `:python` sqlglot parser: the same assertions the graal parser is trusted for, through- the external-process path. Requires a `python3` binary on the PATH (sqlglot itself is installed into- `resources/python-sources` on first use, like the graal tests)."- (:require- [clojure.test :refer :all]- [metabase.sql-parsing.core :as sql-parsing]- [metabase.sql-parsing.protocol :as protocol]- [metabase.sql-parsing.python :as python]))--(set! *warn-on-reflection* true)--(deftest referenced-tables-test- (testing "extracts table references through the external CPython process"- (is (= [[nil nil "transactions"] [nil "public" "orders"]]- (protocol/referenced-tables- (python/parser)- "postgres"- "SELECT * FROM transactions t JOIN public.orders o ON t.id = o.tid")))))--(deftest validate-query-test- (testing "validates syntax through the external CPython process"- (is (= "ok"- (:status (protocol/validate-query (python/parser) "postgres" "SELECT 1" nil nil))))- (is (= "error"- (:status (protocol/validate-query (python/parser) "postgres" "SELECT FROM WHERE" nil nil))))))--(deftest call-error-keeps-process-healthy-test- (testing "a failed call surfaces the Python error and the process still answers the next call"- (is (thrown-with-msg? Exception #"sqlglot call failed"- (protocol/add-into-clause (python/parser) nil nil "t")))- (is (= [[nil nil "users"]]- (protocol/referenced-tables (python/parser) "postgres" "SELECT * FROM users")))))--(deftest parse-error-test- (testing "unparseable SQL surfaces as the same transport-agnostic parse error the graal parser throws"- (let [e (try- (protocol/referenced-tables (python/parser) "postgres" "SELECT !!!")- (catch Exception e e))]- (is (sql-parsing/parse-error? e))))- (testing "other Python-side failures are not parse errors"- (let [e (try- (protocol/add-into-clause (python/parser) nil nil "t")- (catch Exception e e))]- (is (not (sql-parsing/parse-error? e))))))