The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Sqlglot review follow-up (#69788)
lbrdnk2f043ea9d23f
diff --git a/.gitignore b/.gitignoreindex c3da3bd91a1..49691bea2ae 100644--- a/.gitignore+++ b/.gitignore@@ -168,6 +168,5 @@ CLAUDE.local.md# Python dependencies (pip-installed, not committed)/resources/python-sources/sqlglot//resources/python-sources/sqlglot-*.dist-info/-/resources/python-sources/.lock__pycache__/diff --git a/bin/build/src/build/python.clj b/bin/build/src/build/python.cljindex 95d596ad58d..112426d1478 100644--- a/bin/build/src/build/python.clj+++ b/bin/build/src/build/python.clj@@ -2,7 +2,6 @@"Build step to install Python dependencies (sqlglot) for GraalVM Python."(:require[clojure.java.io :as io]- [clojure.string :as str][environ.core :as env][metabuild-common.core :as u]))@@ -14,14 +13,11 @@(def ^:private sqlglot-version"Pinned sqlglot version for reproducible builds.- Read from .sqlglot-version file (single source of truth for both dev and build)."- (->> (io/file u/project-root-directory "resources/python-sources/.sqlglot-version")+ Read from pyproject.toml (single source of truth for both dev and build)."+ (->> (io/file u/project-root-directory "resources/python-sources/pyproject.toml")slurp- ;; handle # comments in the version file- str/split-lines- (remove #(str/starts-with? % "#"))- (str/join "\n")- str/trim))+ (re-find #"\"sqlglot==([^\"]+)\"")+ second))(defn build-python-deps!"Install sqlglot to resources/python-sources for bundling into uberjar.@@ -36,7 +32,7 @@(u/step "Install Python dependencies (sqlglot)"(u/create-directory-unless-exists! python-sources-dir)(u/sh {:dir u/project-root-directory}- "uv" "pip" "install" (str "sqlglot==" sqlglot-version)+ "uv" "pip" "install" "-r" (u/filename python-sources-dir "pyproject.toml")"--target" python-sources-dir"--no-compile")(u/announce "sqlglot %s installed to %s" sqlglot-version python-sources-dir))))diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.cljindex c9f661d12fe..b05def9661a 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj@@ -1,8 +1,8 @@(ns metabase-enterprise.metabot-v3.query-analyzer- "Integration with SQLGlot, which parses native SQL queries. All SQL-specific logic is in SQLGlot, the purpose of this- namespace is to:+ "Integration with `sql-tools`, which parses native SQL queries. All SQL-specific logic is in `sql-tools,+ the purpose of this namespace is to:- 1. Translate Metabase-isms into generic SQL that SQLGlot can understand.+ 1. Translate Metabase-isms into generic SQL that `sql-tools` can understand.2. Encapsulate Metabase-specific business logic."(:require[clojure.string :as str]diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.cljindex 4f847ee6d9e..5066efea514 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj@@ -111,35 +111,25 @@Uses the same matching logic as find-table-or-transform: if parsed schema is nil,it matches against the driver's default schema.Returns a map of [db_id search-schema table] -> table_id where search-schema- is the schema used for lookup (defaulted if nil).- Uses case-insensitive matching for table names to handle dialect differences- (e.g., Snowflake returns uppercase identifiers but DB may store lowercase)."+ is the schema used for lookup (defaulted if nil)."[driver db-id table-refs](when (seq table-refs)(let [default-schema (driver.sql/default-schema driver);; Include default-schema in filter so nil refs can match that toonil-schema? (some (comp nil? :schema) table-refs)schemas-to-query (into #{} (map #(or (:schema %) default-schema)) table-refs)- ;; Use case-insensitive matching: lowercase both the query and the lookup keys- table-names-lower (into #{} (map (comp u/lower-case-en :table)) table-refs)- db-table-lookup (t2/select-fn->fn (fn [row]- [(some-> (:schema row) u/lower-case-en)- (u/lower-case-en (:name row))])- :id+ db-table-lookup (t2/select-fn->fn (juxt :schema :name) :id[:model/Table :id :schema :name]:db_id db-id- {:where [:and- [:in [:lower :name] table-names-lower]- (if nil-schema?- [:or [:= :schema nil] [:in :schema schemas-to-query]]- [:in :schema schemas-to-query])]})]+ :name [:in (map :table table-refs)]+ {:where (if nil-schema?+ [:or [:= :schema nil] [:in :schema schemas-to-query]]+ [:in :schema schemas-to-query])})](into {}(keep (fn [{:keys [schema table]}]- (let [schema-lower (some-> schema u/lower-case-en)- table-lower (u/lower-case-en table)- table-id (or (get db-table-lookup [schema-lower table-lower])+ (let [table-id (or (get db-table-lookup [schema table])(when (nil? schema)- (get db-table-lookup [(some-> default-schema u/lower-case-en) table-lower])))]+ (get db-table-lookup [default-schema table])))](when table-id[[db-id schema table] table-id]))))table-refs))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.cljindex 7a0d08fe19c..b7995fbdb5f 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj@@ -44,7 +44,7 @@(deftest ^:parallel analyze-entity-native-query-existing-table-test(testing "analyze-entity extracts dependencies from native query referencing existing table"(let [mp (mt/metadata-provider)- query (lib/native-query mp "SELECT * FROM ORDERS WHERE total > 100")+ query (lib/native-query mp "SELECT * FROM orders WHERE total > 100")transform {:source {:type "query":query query}:target {:database (mt/id)@@ -56,7 +56,7 @@:schema "analytics":table "high_value_orders"}(:output result))))- (testing "inputs include table_id since ORDERS table exists"+ (testing "inputs include table_id since orders table exists"(let [inputs (:inputs result)];; Native query parsing should detect ORDERS table reference and look up table_id(is (pos? (count inputs)))diff --git a/resources/python-sources/.sqlglot-version b/resources/python-sources/.sqlglot-versiondeleted file mode 100644index da179ff9c5b..00000000000--- a/resources/python-sources/.sqlglot-version+++ /dev/null@@ -1,3 +0,0 @@-# This file keeps the dev and CI versions of sqlglot in sync-# Check here for new versions: https://pypi.org/project/sqlglot/#history-28.6.0diff --git a/resources/python-sources/pyproject.toml b/resources/python-sources/pyproject.tomlnew file mode 100644index 00000000000..61fd7d2790b--- /dev/null+++ b/resources/python-sources/pyproject.toml@@ -0,0 +1,8 @@+[project]+name = "metabase-python-deps"+version = "0.1.0"+requires-python = ">=3.11"+dependencies = [+ # Keep these in the format "library==X.Y.Z", we parse it with a regex.+ "sqlglot==28.6.0",+]diff --git a/resources/python-sources/uv.lock b/resources/python-sources/uv.locknew file mode 100644index 00000000000..8b74d6ab87e--- /dev/null+++ b/resources/python-sources/uv.lock@@ -0,0 +1,23 @@+version = 1+revision = 3+requires-python = ">=3.11"++[[package]]+name = "metabase-python-deps"+version = "0.1.0"+source = { virtual = "." }+dependencies = [+ { name = "sqlglot" },+]++[package.metadata]+requires-dist = [{ name = "sqlglot", specifier = "==28.6.0" }]++[[package]]+name = "sqlglot"+version = "28.6.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/46/b6/f188b9616bef49943353f3622d726af30fdb08acbd081deef28ba43ceb48/sqlglot-28.6.0.tar.gz", hash = "sha256:8c0a432a6745c6c7965bbe99a17667c5a3ca1d524a54b31997cf5422b1727f6a", size = 5676522, upload-time = "2026-01-13T17:39:24.389Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/8f/a6/21b1e19994296ba4a34bc7abaf4fcb40d7e7787477bdfde58cd843594459/sqlglot-28.6.0-py3-none-any.whl", hash = "sha256:8af76e825dc8456a49f8ce049d69bbfcd116655dda3e53051754789e2edf8eba", size = 575186, upload-time = "2026-01-13T17:39:22.327Z" },+]diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.cljindex 58723e15e2c..f6ccd848c57 100644--- a/src/metabase/sql_parsing/pool.clj+++ b/src/metabase/sql_parsing/pool.clj@@ -69,8 +69,8 @@^String [](let [^Path plugins-path (plugins-dir-path)dest-path (.resolve plugins-path "python-sources")- version-file (.resolve dest-path ".sqlglot-version")- jar-version (some-> (io/resource "python-sources/.sqlglot-version") slurp str/trim)+ version-file (.resolve dest-path "pyproject.toml")+ jar-version (some-> (io/resource "python-sources/pyproject.toml") slurp str/trim)dest-version (when (u.files/exists? version-file)(str/trim (slurp (.toFile version-file))))](if (and jar-version (= jar-version dest-version))@@ -98,15 +98,12 @@(.delete f))(defn- expected-sqlglot-version- "Read the expected sqlglot version from .sqlglot-version resource file."+ "Read the expected sqlglot version from pyproject.toml resource file."[]- (some->> (io/resource "python-sources/.sqlglot-version")+ (some->> (io/resource "python-sources/pyproject.toml")slurp- ;; handle # comments in the version file- str/split-lines- (remove #(str/starts-with? % "#"))- (str/join "\n")- str/trim))+ (re-find #"\"sqlglot==([^\"]+)\"")+ second))(defn- package-installer-available?"Check if uv or pip is available for installing Python packages."@@ -131,16 +128,17 @@(log/info "Removing old sqlglot:" (.getName f))(delete-recursive! f)));; Try uv first (fast), fall back to pip- (let [pkg (str "sqlglot==" version)- uv-result (shell/sh "uv" "pip" "install" pkg "--target" target-dir "--no-compile")]+ (let [pyproject-file (str target-dir "/pyproject.toml")+ uv-result (shell/sh "uv" "pip" "install" "-r" pyproject-file "--target" target-dir "--no-compile")](if (zero? (:exit uv-result))(log/info "sqlglot" version "installed via uv")(do(log/info "uv not available, trying pip...")- (let [pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile")]+ (let [pkg (str "sqlglot==" version)+ pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile")](when-not (zero? (:exit pip-result))(throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"- "Manual install: uv pip install " pkg " --target " target-dir "\n"+ "Manual install: uv pip install -r " pyproject-file " --target " target-dir "\n""Install uv: https://docs.astral.sh/uv/getting-started/installation/"){:uv-error (:err uv-result):pip-error (:err pip-result)@@ -154,8 +152,8 @@[](let [expected-ver (expected-sqlglot-version)](when-not expected-ver- (throw (ex-info "Missing .sqlglot-version file in resources/python-sources/"- {:resource "python-sources/.sqlglot-version"})))+ (throw (ex-info "Missing pyproject.toml or unable to parse sqlglot entry in resources/python-sources/"+ {:resource "python-sources/pyproject.toml"})))(when-not (version-installed? dev-python-sources-dir expected-ver)(if (package-installer-available?)(dodiff --git a/test/metabase/sql_parsing/pool_test.clj b/test/metabase/sql_parsing/pool_test.cljindex 8d09a45451c..a1853cb534a 100644--- a/test/metabase/sql_parsing/pool_test.clj+++ b/test/metabase/sql_parsing/pool_test.clj@@ -161,3 +161,11 @@(is (map? result2))(is (= 1 (:context-id result2))) ;; Same context ID(is (= 1 @created-count)))))++;;; ----------------------------------------- Version Parsing --------------------------------------------------------++(deftest expected-sqlglot-version-test+ (testing "Sqlglot version is properly parsed from pyproject.toml"+ (let [version (#'pool/expected-sqlglot-version)]+ (is (string? version))+ (is (re-matches #"\d+\.\d+\.\d+" version)))))