Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Sqlglot review follow-up (#69788)

lbrdnk2f043ea9d23f
diff --git a/.gitignore b/.gitignore
index c3da3bd91a1..49691bea2ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -168,6 +168,5 @@ CLAUDE.local.md
# Python dependencies (pip-installed, not committed)
/resources/python-sources/sqlglot/
/resources/python-sources/sqlglot-*.dist-info/
-/resources/python-sources/.lock
__pycache__/
diff --git a/bin/build/src/build/python.clj b/bin/build/src/build/python.clj
index 95d596ad58d..112426d1478 100644
--- a/bin/build/src/build/python.clj
+++ b/bin/build/src/build/python.clj
@@ -2,7 +2,6 @@
"Build step to install Python dependencies (sqlglot) for GraalVM Python."
(:require
[clojure.java.io :as io]
- [clojure.string :as str]
[environ.core :as env]
[metabuild-common.core :as u]))
@@ -14,14 +13,11 @@
(def ^:private sqlglot-version
"Pinned sqlglot version for reproducible builds.
- Read from .sqlglot-version file (single source of truth for both dev and build)."
- (->> (io/file u/project-root-directory "resources/python-sources/.sqlglot-version")
+ Read from pyproject.toml (single source of truth for both dev and build)."
+ (->> (io/file u/project-root-directory "resources/python-sources/pyproject.toml")
slurp
- ;; handle # comments in the version file
- str/split-lines
- (remove #(str/starts-with? % "#"))
- (str/join "\n")
- str/trim))
+ (re-find #"\"sqlglot==([^\"]+)\"")
+ second))
(defn build-python-deps!
"Install sqlglot to resources/python-sources for bundling into uberjar.
@@ -36,7 +32,7 @@
(u/step "Install Python dependencies (sqlglot)"
(u/create-directory-unless-exists! python-sources-dir)
(u/sh {:dir u/project-root-directory}
- "uv" "pip" "install" (str "sqlglot==" sqlglot-version)
+ "uv" "pip" "install" "-r" (u/filename python-sources-dir "pyproject.toml")
"--target" python-sources-dir
"--no-compile")
(u/announce "sqlglot %s installed to %s" sqlglot-version python-sources-dir))))
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj
index c9f661d12fe..b05def9661a 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj
@@ -1,8 +1,8 @@
(ns metabase-enterprise.metabot-v3.query-analyzer
- "Integration with SQLGlot, which parses native SQL queries. All SQL-specific logic is in SQLGlot, the purpose of this
- namespace is to:
+ "Integration with `sql-tools`, which parses native SQL queries. All SQL-specific logic is in `sql-tools,
+ the purpose of this namespace is to:
- 1. Translate Metabase-isms into generic SQL that SQLGlot can understand.
+ 1. Translate Metabase-isms into generic SQL that `sql-tools` can understand.
2. Encapsulate Metabase-specific business logic."
(:require
[clojure.string :as str]
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
index 4f847ee6d9e..5066efea514 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
@@ -111,35 +111,25 @@
Uses the same matching logic as find-table-or-transform: if parsed schema is nil,
it matches against the driver's default schema.
Returns a map of [db_id search-schema table] -> table_id where search-schema
- is the schema used for lookup (defaulted if nil).
- Uses case-insensitive matching for table names to handle dialect differences
- (e.g., Snowflake returns uppercase identifiers but DB may store lowercase)."
+ is the schema used for lookup (defaulted if nil)."
[driver db-id table-refs]
(when (seq table-refs)
(let [default-schema (driver.sql/default-schema driver)
;; Include default-schema in filter so nil refs can match that too
nil-schema? (some (comp nil? :schema) table-refs)
schemas-to-query (into #{} (map #(or (:schema %) default-schema)) table-refs)
- ;; Use case-insensitive matching: lowercase both the query and the lookup keys
- table-names-lower (into #{} (map (comp u/lower-case-en :table)) table-refs)
- db-table-lookup (t2/select-fn->fn (fn [row]
- [(some-> (:schema row) u/lower-case-en)
- (u/lower-case-en (:name row))])
- :id
+ db-table-lookup (t2/select-fn->fn (juxt :schema :name) :id
[:model/Table :id :schema :name]
:db_id db-id
- {:where [:and
- [:in [:lower :name] table-names-lower]
- (if nil-schema?
- [:or [:= :schema nil] [:in :schema schemas-to-query]]
- [:in :schema schemas-to-query])]})]
+ :name [:in (map :table table-refs)]
+ {:where (if nil-schema?
+ [:or [:= :schema nil] [:in :schema schemas-to-query]]
+ [:in :schema schemas-to-query])})]
(into {}
(keep (fn [{:keys [schema table]}]
- (let [schema-lower (some-> schema u/lower-case-en)
- table-lower (u/lower-case-en table)
- table-id (or (get db-table-lookup [schema-lower table-lower])
+ (let [table-id (or (get db-table-lookup [schema table])
(when (nil? schema)
- (get db-table-lookup [(some-> default-schema u/lower-case-en) table-lower])))]
+ (get db-table-lookup [default-schema table])))]
(when table-id
[[db-id schema table] table-id]))))
table-refs))))
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
index 7a0d08fe19c..b7995fbdb5f 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
@@ -44,7 +44,7 @@
(deftest ^:parallel analyze-entity-native-query-existing-table-test
(testing "analyze-entity extracts dependencies from native query referencing existing table"
(let [mp (mt/metadata-provider)
- query (lib/native-query mp "SELECT * FROM ORDERS WHERE total > 100")
+ query (lib/native-query mp "SELECT * FROM orders WHERE total > 100")
transform {:source {:type "query"
:query query}
:target {:database (mt/id)
@@ -56,7 +56,7 @@
:schema "analytics"
:table "high_value_orders"}
(:output result))))
- (testing "inputs include table_id since ORDERS table exists"
+ (testing "inputs include table_id since orders table exists"
(let [inputs (:inputs result)]
;; Native query parsing should detect ORDERS table reference and look up table_id
(is (pos? (count inputs)))
diff --git a/resources/python-sources/.sqlglot-version b/resources/python-sources/.sqlglot-version
deleted file mode 100644
index da179ff9c5b..00000000000
--- a/resources/python-sources/.sqlglot-version
+++ /dev/null
@@ -1,3 +0,0 @@
-# This file keeps the dev and CI versions of sqlglot in sync
-# Check here for new versions: https://pypi.org/project/sqlglot/#history
-28.6.0
diff --git a/resources/python-sources/pyproject.toml b/resources/python-sources/pyproject.toml
new file mode 100644
index 00000000000..61fd7d2790b
--- /dev/null
+++ b/resources/python-sources/pyproject.toml
@@ -0,0 +1,8 @@
+[project]
+name = "metabase-python-deps"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+ # Keep these in the format "library==X.Y.Z", we parse it with a regex.
+ "sqlglot==28.6.0",
+]
diff --git a/resources/python-sources/uv.lock b/resources/python-sources/uv.lock
new file mode 100644
index 00000000000..8b74d6ab87e
--- /dev/null
+++ b/resources/python-sources/uv.lock
@@ -0,0 +1,23 @@
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[[package]]
+name = "metabase-python-deps"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+ { name = "sqlglot" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "sqlglot", specifier = "==28.6.0" }]
+
+[[package]]
+name = "sqlglot"
+version = "28.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/46/b6/f188b9616bef49943353f3622d726af30fdb08acbd081deef28ba43ceb48/sqlglot-28.6.0.tar.gz", hash = "sha256:8c0a432a6745c6c7965bbe99a17667c5a3ca1d524a54b31997cf5422b1727f6a", size = 5676522, upload-time = "2026-01-13T17:39:24.389Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/8f/a6/21b1e19994296ba4a34bc7abaf4fcb40d7e7787477bdfde58cd843594459/sqlglot-28.6.0-py3-none-any.whl", hash = "sha256:8af76e825dc8456a49f8ce049d69bbfcd116655dda3e53051754789e2edf8eba", size = 575186, upload-time = "2026-01-13T17:39:22.327Z" },
+]
diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.clj
index 58723e15e2c..f6ccd848c57 100644
--- a/src/metabase/sql_parsing/pool.clj
+++ b/src/metabase/sql_parsing/pool.clj
@@ -69,8 +69,8 @@
^String []
(let [^Path plugins-path (plugins-dir-path)
dest-path (.resolve plugins-path "python-sources")
- version-file (.resolve dest-path ".sqlglot-version")
- jar-version (some-> (io/resource "python-sources/.sqlglot-version") slurp str/trim)
+ version-file (.resolve dest-path "pyproject.toml")
+ jar-version (some-> (io/resource "python-sources/pyproject.toml") slurp str/trim)
dest-version (when (u.files/exists? version-file)
(str/trim (slurp (.toFile version-file))))]
(if (and jar-version (= jar-version dest-version))
@@ -98,15 +98,12 @@
(.delete f))
(defn- expected-sqlglot-version
- "Read the expected sqlglot version from .sqlglot-version resource file."
+ "Read the expected sqlglot version from pyproject.toml resource file."
[]
- (some->> (io/resource "python-sources/.sqlglot-version")
+ (some->> (io/resource "python-sources/pyproject.toml")
slurp
- ;; handle # comments in the version file
- str/split-lines
- (remove #(str/starts-with? % "#"))
- (str/join "\n")
- str/trim))
+ (re-find #"\"sqlglot==([^\"]+)\"")
+ second))
(defn- package-installer-available?
"Check if uv or pip is available for installing Python packages."
@@ -131,16 +128,17 @@
(log/info "Removing old sqlglot:" (.getName f))
(delete-recursive! f)))
;; Try uv first (fast), fall back to pip
- (let [pkg (str "sqlglot==" version)
- uv-result (shell/sh "uv" "pip" "install" pkg "--target" target-dir "--no-compile")]
+ (let [pyproject-file (str target-dir "/pyproject.toml")
+ uv-result (shell/sh "uv" "pip" "install" "-r" pyproject-file "--target" target-dir "--no-compile")]
(if (zero? (:exit uv-result))
(log/info "sqlglot" version "installed via uv")
(do
(log/info "uv not available, trying pip...")
- (let [pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile")]
+ (let [pkg (str "sqlglot==" version)
+ pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile")]
(when-not (zero? (:exit pip-result))
(throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"
- "Manual install: uv pip install " pkg " --target " target-dir "\n"
+ "Manual install: uv pip install -r " pyproject-file " --target " target-dir "\n"
"Install uv: https://docs.astral.sh/uv/getting-started/installation/")
{:uv-error (:err uv-result)
:pip-error (:err pip-result)
@@ -154,8 +152,8 @@
[]
(let [expected-ver (expected-sqlglot-version)]
(when-not expected-ver
- (throw (ex-info "Missing .sqlglot-version file in resources/python-sources/"
- {:resource "python-sources/.sqlglot-version"})))
+ (throw (ex-info "Missing pyproject.toml or unable to parse sqlglot entry in resources/python-sources/"
+ {:resource "python-sources/pyproject.toml"})))
(when-not (version-installed? dev-python-sources-dir expected-ver)
(if (package-installer-available?)
(do
diff --git a/test/metabase/sql_parsing/pool_test.clj b/test/metabase/sql_parsing/pool_test.clj
index 8d09a45451c..a1853cb534a 100644
--- a/test/metabase/sql_parsing/pool_test.clj
+++ b/test/metabase/sql_parsing/pool_test.clj
@@ -161,3 +161,11 @@
(is (map? result2))
(is (= 1 (:context-id result2))) ;; Same context ID
(is (= 1 @created-count)))))
+
+;;; ----------------------------------------- Version Parsing --------------------------------------------------------
+
+(deftest expected-sqlglot-version-test
+ (testing "Sqlglot version is properly parsed from pyproject.toml"
+ (let [version (#'pool/expected-sqlglot-version)]
+ (is (string? version))
+ (is (re-matches #"\d+\.\d+\.\d+" version)))))