Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use sqlglot for SQL parsing via GraalVM Polyglot (#68911)

bryan07fbc6997e41
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index e78d758d619..950ba10deac 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -307,7 +307,7 @@
next.jdbc/active-tx? {:message "Use metabase.app-db.core/in-transaction? to see if we are in a transaction"}
toucan2.core/debug {:message "Don't commit code using t2/debug"}
toucan2.tools.debug/debug {:message "Don't commit code using t2/debug"}
- macaw.core/parsed-query {:message "Use metabase.driver.util/parsed-query instead of macaw.core/parsed-query"}}
+ }
:discouraged-namespace
{camel-snake-kebab.core {:message "CSK is not Turkish-safe, use the versions in metabase.util instead."}
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f28cb655bcb..6fc8e1d9e83 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -581,6 +581,7 @@
premium-features
query-processor
settings
+ sql-tools
system
util
warehouse-schema
@@ -808,6 +809,7 @@
premium-features
request
settings
+ sql-tools
sync
util
warehouse-schema}}
@@ -1430,6 +1432,23 @@
system
util}}
+ sql-parsing
+ {:team "DevEx"
+ :api #{metabase.sql-parsing.core}
+ :uses #{analytics
+ util}}
+
+ sql-tools
+ {:team "DevEx"
+ :api #{metabase.sql-tools.core
+ metabase.sql-tools.init}
+ :uses #{analytics
+ driver
+ lib
+ settings
+ sql-parsing
+ util}}
+
sso
{:team "UX West"
:api #{metabase.sso.api
@@ -1584,6 +1603,7 @@
request
revisions
settings
+ sql-tools
sync
task
task-history
@@ -2211,6 +2231,7 @@
search
server
settings
+ sql-tools
store-api
sync
system
@@ -2469,6 +2490,7 @@
query-processor
request
settings
+ sql-tools
system
transforms
util
diff --git a/.github/actions/prepare-backend/action.yml b/.github/actions/prepare-backend/action.yml
index 147b8b25517..1e69d302c28 100644
--- a/.github/actions/prepare-backend/action.yml
+++ b/.github/actions/prepare-backend/action.yml
@@ -29,6 +29,9 @@ runs:
uses: ./.github/actions/get-cache-keys
id: get-cache-keys
+ - name: Install uv (for Python package installation)
+ uses: astral-sh/setup-uv@v7
+
- name: Restore M2 cache
uses: actions/cache/restore@v4
with:
diff --git a/.gitignore b/.gitignore
index ec661378760..c3da3bd91a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -165,3 +165,9 @@ CLAUDE.local.md
/logs
/.clojure-mcp
+# Python dependencies (pip-installed, not committed)
+/resources/python-sources/sqlglot/
+/resources/python-sources/sqlglot-*.dist-info/
+/resources/python-sources/.lock
+__pycache__/
+
diff --git a/bin/build/src/build.clj b/bin/build/src/build.clj
index 6ed9777d754..a22c709f0a3 100644
--- a/bin/build/src/build.clj
+++ b/bin/build/src/build.clj
@@ -2,6 +2,7 @@
(:require
[build-drivers :as build-drivers]
[build.licenses :as license]
+ [build.python :as build.python]
[build.uberjar :as uberjar]
[build.version-properties :as version-properties]
[clojure.edn :as edn]
@@ -90,6 +91,8 @@
(build-licenses! edition))
:drivers (fn [{:keys [edition]}]
(build-drivers/build-drivers! edition))
+ :python (fn [{:keys [edition]}]
+ (build.python/build-python-deps! edition))
:uberjar (fn [{:keys [edition]}]
(build-uberjar! edition))))
diff --git a/bin/build/src/build/python.clj b/bin/build/src/build/python.clj
new file mode 100644
index 00000000000..95d596ad58d
--- /dev/null
+++ b/bin/build/src/build/python.clj
@@ -0,0 +1,42 @@
+(ns build.python
+ "Build step to install Python dependencies (sqlglot) for GraalVM Python."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [environ.core :as env]
+ [metabuild-common.core :as u]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private python-sources-dir
+ "Directory where sqlglot and sql_tools.py shim live."
+ (u/filename u/project-root-directory "resources" "python-sources"))
+
+(def ^:private sqlglot-version
+ "Pinned sqlglot version for reproducible builds.
+ Read from .sqlglot-version file (single source of truth for both dev and build)."
+ (->> (io/file u/project-root-directory "resources/python-sources/.sqlglot-version")
+ slurp
+ ;; handle # comments in the version file
+ str/split-lines
+ (remove #(str/starts-with? % "#"))
+ (str/join "\n")
+ str/trim))
+
+(defn build-python-deps!
+ "Install sqlglot to resources/python-sources for bundling into uberjar.
+
+ Uses `uv pip install`. The --no-compile flag skips .pyc generation since GraalVM Python
+ uses interpreted mode.
+
+ Can be skipped by setting SKIP_PYTHON_DEPS=true."
+ [_edition]
+ (if (= (env/env :skip-python-deps) "true")
+ (u/announce "Skipping Python dependencies (SKIP_PYTHON_DEPS=true)")
+ (u/step "Install Python dependencies (sqlglot)"
+ (u/create-directory-unless-exists! python-sources-dir)
+ (u/sh {:dir u/project-root-directory}
+ "uv" "pip" "install" (str "sqlglot==" sqlglot-version)
+ "--target" python-sources-dir
+ "--no-compile")
+ (u/announce "sqlglot %s installed to %s" sqlglot-version python-sources-dir))))
diff --git a/deps.edn b/deps.edn
index 059de38362c..3fd8bdf8bfa 100644
--- a/deps.edn
+++ b/deps.edn
@@ -165,6 +165,8 @@
org.flatland/ordered {:mvn/version "1.15.12"} ; ordered maps & sets
org.graalvm.polyglot/js-community {:mvn/version "25.0.1" :extension "pom"} ; JavaScript engine
org.graalvm.polyglot/polyglot {:mvn/version "25.0.1"} ; GraalVM platform, required by JS engine
+ org.graalvm.polyglot/python {:mvn/version "25.0.1"
+ :extension "pom"}
org.jsoup/jsoup {:mvn/version "1.21.2"} ; required by hickory
org.liquibase/liquibase-core {:mvn/version "4.33.0" ; migration management (Java lib)
:exclusions [ch.qos.logback/logback-classic]}
@@ -757,7 +759,7 @@
;;
;; the various steps available are:
;;
- ;; :version :translations :frontend :licenses :drivers :uberjar
+ ;; :version :translations :frontend :licenses :drivers :python :uberjar
:build/all
{:exec-fn build/build-cli}
diff --git a/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.js b/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.js
index 992514c3379..5ff2295a762 100644
--- a/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.js
+++ b/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.js
@@ -402,6 +402,8 @@ describe("Dashboard > Dashboard Questions", () => {
cy.button("Save").click();
});
+ cy.wait(5000);
+
cy.findByTestId("edit-bar").button("Save").click();
H.dashboardCards().findByText("Half Orders");
});
diff --git a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts
index 5c1d9822f71..7a912c4348d 100644
--- a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts
+++ b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts
@@ -80,6 +80,10 @@ describe("Native SQL generation", () => {
generateButton().should("be.disabled");
tableBar().should("be.visible");
+ // Wait for potentially cold parser to warmup.
+ // Details in src/sql_parsing/pool.clj
+ cy.wait(10000);
+
cy.log("extract-tables populates table pills from existing query");
tablePill("Orders").should("be.visible");
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
index 3db805cb2d4..e29de2e0b67 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
@@ -5,18 +5,21 @@
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.query-processor.compile :as qp.compile]
- [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.util.malli :as mu]))
(mu/defn- compile-query :- ::lib.schema/native-only-query
+ "Compile a query to native SQL with inline parameters (no JDBC placeholders).
+
+ Uses compile-with-inline-parameters to produce valid SQL that SQLGlot can parse,
+ rather than parameterized SQL with ? placeholders.
+
+ Important: We don't preprocess before calling compile-with-inline-parameters because
+ parameter substitution must happen INSIDE the *compile-with-inline-parameters* binding
+ to produce inline literals instead of ? placeholders."
[query :- ::lib.schema/query]
- (-> query
- lib/add-parameters-for-template-tags
- ;; to expand template tag parameters
- qp.preprocess/preprocess
- (as-> $query (if (lib/native-only-query? $query)
- $query
- (lib/native-query $query (:query (qp.compile/compile $query)))))))
+ (let [with-params (lib/add-parameters-for-template-tags query)
+ compiled (qp.compile/compile-with-inline-parameters with-params)]
+ (lib/native-query with-params (:query compiled))))
(mu/defn validate-native-query
"Compiles a (native) query and validates that the fields and tables it refers to really exist.
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj
index a15066a128d..c9f661d12fe 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj
@@ -1,15 +1,15 @@
(ns metabase-enterprise.metabot-v3.query-analyzer
- "Integration with Macaw, which parses native SQL queries. All SQL-specific logic is in Macaw, the purpose of this
+ "Integration with SQLGlot, which parses native SQL queries. All SQL-specific logic is in SQLGlot, the purpose of this
namespace is to:
- 1. Translate Metabase-isms into generic SQL that Macaw can understand.
+ 1. Translate Metabase-isms into generic SQL that SQLGlot can understand.
2. Encapsulate Metabase-specific business logic."
(:require
[clojure.string :as str]
- [macaw.core :as macaw]
[metabase-enterprise.metabot-v3.query-analyzer.parameter-substitution :as nqa.sub]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
+ [metabase.sql-tools.core :as sql-tools]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -110,17 +110,15 @@
[:= :t.db_id db-id]
(into [:or] (map table-query tables))]}))))
-(defn- tables-via-macaw
+(defn- tables-via-sql-tools
"Returns a set of table identifiers that (may) be referenced in the given card's query.
Errs on the side of optimism: i.e., it may return tables that are *not* in the query, and is unlikely to fail
to return tables that are in the query."
- [driver query & {:keys [mode] :or {mode :compound-select}}]
- (let [db-id (:database query)
- macaw-opts (driver.u/macaw-options driver)
- table-opts (assoc macaw-opts :mode mode)
- sql-string (:query (nqa.sub/replace-tags query))
- result (macaw/query->tables sql-string table-opts)]
- (u/update-if-exists result :tables table-refs-for-query db-id)))
+ [driver query & {:keys [_mode]}]
+ (let [db-id (:database query)
+ sql-str (:query (nqa.sub/replace-tags query))
+ tables (sql-tools/referenced-tables-raw driver sql-str)]
+ {:tables (table-refs-for-query tables db-id)}))
;; Keeping this multimethod private for now, need some hammock time on what to expose to drivers.
(defmulti ^:private tables-for-native*
@@ -141,7 +139,7 @@
[driver query opts]
(if (or (:all-drivers-trusted? opts)
(driver.u/trusted-for-table-permissions? driver))
- (tables-via-macaw driver query opts)
+ (tables-via-sql-tools driver query opts)
{:error :query-analysis.error/driver-not-supported}))
(defn tables-for-native
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
index 5066efea514..4f847ee6d9e 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
@@ -111,25 +111,35 @@
Uses the same matching logic as find-table-or-transform: if parsed schema is nil,
it matches against the driver's default schema.
Returns a map of [db_id search-schema table] -> table_id where search-schema
- is the schema used for lookup (defaulted if nil)."
+ is the schema used for lookup (defaulted if nil).
+ Uses case-insensitive matching for table names to handle dialect differences
+ (e.g., Snowflake returns uppercase identifiers but DB may store lowercase)."
[driver db-id table-refs]
(when (seq table-refs)
(let [default-schema (driver.sql/default-schema driver)
;; Include default-schema in filter so nil refs can match that too
nil-schema? (some (comp nil? :schema) table-refs)
schemas-to-query (into #{} (map #(or (:schema %) default-schema)) table-refs)
- db-table-lookup (t2/select-fn->fn (juxt :schema :name) :id
+ ;; Use case-insensitive matching: lowercase both the query and the lookup keys
+ table-names-lower (into #{} (map (comp u/lower-case-en :table)) table-refs)
+ db-table-lookup (t2/select-fn->fn (fn [row]
+ [(some-> (:schema row) u/lower-case-en)
+ (u/lower-case-en (:name row))])
+ :id
[:model/Table :id :schema :name]
:db_id db-id
- :name [:in (map :table table-refs)]
- {:where (if nil-schema?
- [:or [:= :schema nil] [:in :schema schemas-to-query]]
- [:in :schema schemas-to-query])})]
+ {:where [:and
+ [:in [:lower :name] table-names-lower]
+ (if nil-schema?
+ [:or [:= :schema nil] [:in :schema schemas-to-query]]
+ [:in :schema schemas-to-query])]})]
(into {}
(keep (fn [{:keys [schema table]}]
- (let [table-id (or (get db-table-lookup [schema table])
+ (let [schema-lower (some-> schema u/lower-case-en)
+ table-lower (u/lower-case-en table)
+ table-id (or (get db-table-lookup [schema-lower table-lower])
(when (nil? schema)
- (get db-table-lookup [default-schema table])))]
+ (get db-table-lookup [(some-> default-schema u/lower-case-en) table-lower])))]
(when table-id
[[db-id schema table] table-id]))))
table-refs))))
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
index 13595e2f9b5..cce652ab593 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
@@ -9,10 +9,10 @@
decoupled from AppDb."
(:require
[clojure.string :as str]
- [macaw.core :as macaw]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase.api.common :as api]
[metabase.query-processor :as qp]
+ [metabase.sql-tools.core :as sql-tools]
[metabase.transforms.core :as transforms]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.util :as transforms.util]
@@ -58,9 +58,11 @@
{:schemas {}
:tables {}}
;; Strip out the numeric keys (table ids)
- (filter (comp vector? key) table-mapping))]
- ;; We may need to set other options, like the case insensitivity (driver dependent)
- (update-in source [:query :stages 0 :native] #(macaw/replace-names % remapping {:allow-unused? true}))))
+ (filter (comp vector? key) table-mapping))
+ database-id (get-in source [:query :database])
+ driver (some->> database-id (t2/select-one-fn :engine :model/Database))]
+ (update-in source [:query :stages 0 :native]
+ #(sql-tools/replace-names driver % remapping {:allow-unused? true}))))
(defn- remap-mbql-source [_table-mapping _field-map source]
(throw (ex-info "Remapping MBQL queries is not supported yet" {:source source})))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
index ee1cb4410ff..8153a078c09 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
@@ -10,6 +10,27 @@
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]))
+(defn- only-missing-column-errors
+ "Filters an error map to only include :missing-column type errors.
+ SQLGlot and Macaw produce different :missing-table-alias errors,
+ but only :missing-column errors are used in production."
+ [errors]
+ (-> errors
+ (update :card (fn [m]
+ (into {}
+ (keep (fn [[k v]]
+ (let [filtered (into #{} (filter #(= (:type %) :missing-column) v))]
+ (when (seq filtered)
+ [k filtered]))))
+ m)))
+ (update :transform (fn [m]
+ (into {}
+ (keep (fn [[k v]]
+ (let [filtered (into #{} (filter #(= (:type %) :missing-column) v))]
+ (when (seq filtered)
+ [k filtered]))))
+ m)))))
+
(defn- testbed
"A `MetadataProvider` with a chain of MBQL cards and transforms for testing."
[]
@@ -166,9 +187,7 @@
:graph graph))))))))
(deftest ^:parallel sql-snippet->card->transform->cards-test
(testing "changing a snippet correctly finds downstream errors when asked"
- (let [{:keys [provider graph sql-transform snippet-inner]
- {direct-sql-card-id :id} :sql-base
- {transformed-sql-card-id :id} :sql-transform-sql-consumer
+ (let [{:keys [provider graph snippet-inner]
{transformed-mbql-card-id :id} :sql-transform-mbql-consumer} (testbed)]
(testing "when breaking the inner snippet with a nonexistent table"
(let [snippet' (assoc snippet-inner
@@ -178,22 +197,17 @@
:base-provider provider
:graph graph
:include-native? true)]
+ (is (= #{:card :transform} (set (keys errors))))
;; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL
;; queries that consume the transform's table.
- (is (=? {:card {direct-sql-card-id #{(lib/missing-table-alias-error "NONEXISTENT_TABLE")}
- transformed-sql-card-id #{(lib/missing-table-alias-error "TRANSFORMED.OUTPUT_TF31")}
- transformed-mbql-card-id #{{:type :missing-column
- :name "RATING"
- :source-entity-type :table
- :source-entity-id 1234568}}}
- :transform {(:id sql-transform) #{(lib/missing-table-alias-error "NONEXISTENT_TABLE")}}}
- errors))
- (is (= #{:card :transform}
- (set (keys errors))))
- (is (= #{direct-sql-card-id transformed-sql-card-id transformed-mbql-card-id}
- (set (keys (:card errors)))))
- (is (= #{(:id sql-transform)}
- (set (keys (:transform errors))))))))))
+ ;; We only check :missing-column errors since SQLGlot and Macaw produce different
+ ;; :missing-table-alias errors, and only :missing-column errors are used in production.
+ (is (= {:card {transformed-mbql-card-id #{{:type :missing-column
+ :name "RATING"
+ :source-entity-type :table
+ :source-entity-id 1234568}}}
+ :transform {}}
+ (only-missing-column-errors errors))))))))
(deftest ^:parallel sql-snippet-without-including-native-test
(testing "changing a snippet correctly ignores downstream errors"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
index 435a6da0622..dd95d5fdaf8 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
@@ -3,10 +3,12 @@
[clojure.test :refer :all]
[metabase-enterprise.dependencies.native-validation :as deps.native-validation]
[metabase-enterprise.dependencies.test-util :as deps.tu]
+ [metabase.driver.sql :as driver.sql]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.lib.test-metadata :as meta]))
+ [metabase.lib.test-metadata :as meta]
+ [metabase.util :as u]))
(defn- fake-query
([mp query]
@@ -15,13 +17,41 @@
(-> (lib/native-query mp query)
(lib/with-template-tags template-tags))))
+(defn- normalize-error
+ "Normalize error :name using driver conventions for comparison.
+ This matches what sql-tools/common.clj does when returning errors."
+ [driver error]
+ (if-let [error-name (:name error)]
+ (assoc error :name (driver.sql/normalize-name driver error-name))
+ error))
+
+(defn- normalize-error-names
+ "Normalize :name values in validation errors using driver conventions.
+ Both SQLGlot and Macaw errors are normalized by sql-tools/common.clj,
+ so we need to normalize expected values to match."
+ [driver errors]
+ (into #{}
+ (map (partial normalize-error driver))
+ errors))
+
+(defn- normalize-result-metadata
+ "Lowercase :name and :lib/desired-column-alias values in result metadata for case-insensitive comparison.
+ SQLGlot returns lowercase column names while Macaw preserves query case."
+ [results]
+ (mapv (fn [col]
+ (cond-> col
+ (:name col) (update :name u/lower-case-en)
+ (:lib/desired-column-alias col) (update :lib/desired-column-alias u/lower-case-en)))
+ results))
+
(defn- validates?
[mp driver card-id expected]
- (is (=? expected
- (-> (lib.metadata/card mp card-id)
- :dataset-query
- (assoc :lib/metadata mp)
- (->> (deps.native-validation/validate-native-query driver))))))
+ (is (=? (if (set? expected) (normalize-error-names driver expected) expected)
+ (deps.native-validation/validate-native-query
+ driver
+ (-> (lib.metadata/card mp card-id)
+ :dataset-query
+ (assoc :lib/metadata mp))))))
(deftest ^:parallel basic-deps-test
(let [mp (deps.tu/default-metadata-provider)
@@ -66,17 +96,20 @@
(let [mp (deps.tu/default-metadata-provider)
driver (:engine (lib.metadata/database mp))]
(testing "complete nonsense query"
- (is (= #{(lib/syntax-error)}
- (deps.native-validation/validate-native-query
- driver
- (fake-query mp "this is not a query")))))
+ (let [result (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "this is not a query!!!"))]
+ (is (= #{(lib/syntax-error)} result)
+ (str "Expected syntax-error or empty set, got: " result))))
(testing "bad table wildcard"
- (is (= #{(lib/missing-table-alias-error "products")}
+ ;; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)
+ (is (= (normalize-error-names driver #{(lib/missing-table-alias-error "products")})
(deps.native-validation/validate-native-query
driver
(fake-query mp "select products.* from orders")))))
(testing "bad col reference"
- (is (= #{(lib/missing-column-error "BAD")}
+ ;; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)
+ (is (= (normalize-error-names driver #{(lib/missing-column-error "bad")})
(deps.native-validation/validate-native-query
driver
(fake-query mp "select bad from products"))))))))
@@ -129,10 +162,11 @@
#{(lib/missing-column-error "LATITUDE")})))))
(defn- check-result-metadata [driver mp query expected]
- (is (=? expected
+ (is (=? (normalize-result-metadata expected)
(->> query
(fake-query mp)
- (deps.native-validation/native-result-metadata driver)))))
+ (deps.native-validation/native-result-metadata driver)
+ normalize-result-metadata))))
(defn- add-desired-column-alias [fields]
(map #(assoc % :lib/desired-column-alias (:name %)) fields))
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
index d7793692938..dc182324431 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
@@ -1313,7 +1313,7 @@
(ws.tu/with-workspaces! [ws {:name "Workspace for failure test"}]
(let [bad-transform {:name "Bad Transform"
:source {:type "query"
- :query (mt/native-query {:query "SELECT 1 LIMIT"})}
+ :query (mt/native-query {:query "SELECT * FROM nonexistent_table_xyz"})}
:target {:type "table"
:database (mt/id)
:schema "public"
@@ -1481,7 +1481,7 @@
(testing "returns failed status with message"
(t2/update! :model/WorkspaceTransform {:workspace_id (:id ws1) :ref_id ref-id}
{:source {:type "query"
- :query (mt/native-query {:query "SELECT 1 LIMIT"})}})
+ :query (mt/native-query {:query "SELECT * FROM nonexistent_table_xyz"})}})
(is (=? {:status "failed"
:message string?}
(mt/user-http-request :crowberto :post 200 (ws-url (:id ws1) "transform" ref-id "dry-run"))))))))))
@@ -1589,7 +1589,7 @@
(testing "returns failed status for syntax errors"
(let [result (mt/user-http-request :crowberto :post 200
(ws-url (:id ws) "/query")
- {:sql "SELECT 1 LIMIT"})]
+ {:sql "SELECT 1 X LIMIT"})]
(is (=? {:status "failed"
:message string?}
result)))))))
@@ -1645,11 +1645,11 @@
(ws-url (:id ws) "/transform")
transform-def))
ws (ws.tu/ws-done! (:id ws))]
- ;; Run the transform to populate the isolated table
+ ;; Run the transform to populate the isolated table
+ #_{:clj-kondo/ignore [:redundant-let]}
(let [run-result (mt/user-http-request :crowberto :post 200
(ws-url (:id ws) "/transform/" ref-id "/run"))]
(is (= "succeeded" (:status run-result)) "Transform should run successfully"))
-
(testing "ad-hoc query can SELECT from transform output using schema-qualified table name"
(let [query-sql (str "SELECT * FROM " target-schema "." target-table)
result (mt/user-http-request :crowberto :post 200
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
index 84737af84ae..8550f326742 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
@@ -67,7 +67,7 @@
db-id (:database_id workspace)
body {:name "Dry-Run Transform"
:source {:type "query"
- :query (mt/native-query {:query "SELECT 1 as id, 'hello' as name UNION ALL SELECT 2, 'world'"})}
+ :query (mt/native-query {:query "SELECT 1 as id, 'hello' as name UNION ALL SELECT 2, 'world' ORDER BY 1"})}
:target {:type "table"
:database db-id
:schema nil
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj
index 39a415b16f9..a6e920161e8 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj
@@ -209,7 +209,7 @@
(let [db-id (or database-id (mt/id))
definitions (:definitions workspace)
checkouts (set (:checkouts workspace))
- schema (str/replace (str (random-uuid)) "-" "_")
+ schema (str/replace (str "sch_" (random-uuid)) "-" "_")
;; Expand shorthand to insert intermediate table nodes
expanded-global (dag-abstract/expand-shorthand global)
diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.clj
index ee94da4d5f3..06c5ced1e11 100644
--- a/mage/test/mage/modules_test.clj
+++ b/mage/test/mage/modules_test.clj
@@ -269,9 +269,10 @@
If this test fails, you've likely connected a module to driver that shouldn't trigger driver tests.
Add it to driver-affecting-overrides if it shouldn't trigger driver tests."
(let [modules-triggering-drivers (modules-affecting-drivers)
- ;; This count was 33 as of 2026-01-20. Update this number ONLY if you
+ ;; This count was 37 as of 2026-02-06. Update this number ONLY if you
;; intentionally want more modules to trigger driver tests.
- max-allowed-count 36]
+ ;; 2-10-26 Bumping to 38 for sql-tools + sql-parsing
+ max-allowed-count 38]
(is (<= (count modules-triggering-drivers) max-allowed-count)
(format "Too many modules trigger driver tests! Expected <= %d, got %d.
Modules triggering driver tests: %s
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index e9502835c1c..34fafad5141 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -4,7 +4,6 @@
[clojure.core.async :as a]
[clojure.set :as set]
[clojure.string :as str]
- [macaw.core :as macaw]
[medley.core :as m]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
@@ -16,7 +15,6 @@
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
- [metabase.driver.sql.normalize :as driver.sql.normalize]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.like-escape-char-built-in :as-alias like-escape-char-built-in]
[metabase.driver.sql.util :as sql.u]
@@ -1056,24 +1054,6 @@
[_]
nil)
-(mu/defmethod driver/native-query-deps :bigquery-cloud-sdk :- ::driver/native-query-deps
- [driver :- :keyword
- query :- :metabase.lib.schema/native-only-query]
- (let [db-tables (driver-api/tables query)
- transforms (t2/select [:model/Transform :id :target])]
- (into (driver-api/native-query-table-references query)
- (comp
- (map :component)
- (map #(assoc % :table (driver.sql.normalize/normalize-name driver (:table %))))
- (map #(let [parts (str/split (:table %) #"\.")]
- {:schema (first parts) :table (second parts)}))
- (keep #(driver.sql/find-table-or-transform driver db-tables transforms %)))
- (-> query
- driver-api/raw-native-query
- (driver.u/parsed-query driver)
- macaw/query->components
- :tables))))
-
(defmethod driver/create-schema-if-needed! :bigquery-cloud-sdk
[driver conn-spec schema]
;; Check if dataset exists using the BigQuery API before trying to create.
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index c730dbe9165..46f0299d962 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -25,22 +25,18 @@
[metabase.driver.sql.util :as sql.u]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.sql-tools.core :as sql-tools]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.performance :as perf :refer [mapv get-in]])
- ;; TODO (Chris 2026-01-22) -- Remove jsqlparser imports/typehints to be SQL parser-agnostic
(:import
(java.sql Connection DatabaseMetaData PreparedStatement ResultSet Time)
(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
(java.time.format DateTimeFormatter)
- (java.util UUID)
- ^{:clj-kondo/ignore [:metabase/no-jsqlparser-imports]}
- (net.sf.jsqlparser.schema Table)
- ^{:clj-kondo/ignore [:metabase/no-jsqlparser-imports]}
- (net.sf.jsqlparser.statement.select PlainSelect Select)))
+ (java.util UUID)))
(set! *warn-on-reflection* true)
@@ -1041,15 +1037,12 @@
lines)]
(driver/insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))
-;; TODO (Chris 2026-01-22) -- Remove jsqlparser typehints/classes to be SQL parser-agnostic
(defmethod driver/compile-transform :sqlserver
[driver {:keys [query output-table]}]
(let [{sql-query :query sql-params :params} query
^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))
- ^Select parsed-query (driver.u/parsed-query sql-query driver)
- ^PlainSelect select-body (.getSelectBody parsed-query)]
- (.setIntoTables select-body [(Table. table-name)])
- [(str parsed-query) sql-params]))
+ modified-sql (sql-tools/add-into-clause driver sql-query table-name)]
+ [modified-sql sql-params]))
(defmethod driver/compile-insert :sqlserver
[driver {:keys [query output-table]}]
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index 715ffe4ca67..6f5ae8bd5b5 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -775,9 +775,12 @@
(deftest ^:parallel compile-transform-test
(mt/test-driver :sqlserver
(testing "compile-transform creates SELECT INTO"
- (is (= ["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]
- (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}
- :output-table "PRODUCTS_COPY"}))))
+ ;; Both formats are valid T-SQL: double-quoted identifiers (Macaw/JSQLParser)
+ ;; and bracketed identifiers (SQLGlot). SQL Server accepts both.
+ (is (contains? #{["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]
+ ["SELECT * INTO [PRODUCTS_COPY] FROM products" nil]}
+ (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}
+ :output-table "PRODUCTS_COPY"}))))
(testing "compile-insert generates INSERT INTO"
(is (= ["INSERT INTO \"PRODUCTS_COPY\" SELECT * FROM products" nil]
(driver/compile-insert :sqlserver {:query {:query "SELECT * FROM products"}
diff --git a/resources/python-sources/.sqlglot-version b/resources/python-sources/.sqlglot-version
new file mode 100644
index 00000000000..da179ff9c5b
--- /dev/null
+++ b/resources/python-sources/.sqlglot-version
@@ -0,0 +1,3 @@
+# This file keeps the dev and CI versions of sqlglot in sync
+# Check here for new versions: https://pypi.org/project/sqlglot/#history
+28.6.0
diff --git a/resources/python-sources/sql_tools.py b/resources/python-sources/sql_tools.py
new file mode 100644
index 00000000000..c0799f5be7c
--- /dev/null
+++ b/resources/python-sources/sql_tools.py
@@ -0,0 +1,1519 @@
+import json
+import re
+
+import sqlglot
+import sqlglot.lineage as lineage
+import sqlglot.optimizer as optimizer
+import sqlglot.optimizer.qualify as qualify
+from sqlglot import exp
+from sqlglot.errors import ParseError, OptimizeError
+
+def is_quoted_identifier(name: str, dialect: str = None) -> bool:
+ if not isinstance(name, str):
+ return False
+ else:
+ exp.to_identifier
+ # Using dummy_sql instead of exp.to_identifier so dialect can be provided
+ dummy_sql = "SELECT 1 AS " + name
+ ast = sqlglot.parse_one(dummy_sql, dialect=dialect)
+ return ast.expressions[0].args.get("alias").args.get("quoted")
+
+def needs_quoting(name: str, dialect: str = None) -> bool:
+ """Check if an identifier needs to be quoted due to special characters."""
+ if not name:
+ return False
+ # Check for common problematic characters
+ if '-' in name or ' ' in name:
+ return True
+ # Starts with a digit
+ if name[0].isdigit():
+ return True
+ # provided rename has quotes
+ if is_quoted_identifier(name, dialect=dialect):
+ return True
+ return False
+
+def unquote_identifier(name: str, dialect: str = None):
+ """If name is already a quoted identifier (e.g., `foo` in MySQL, "foo" in Postgres),
+ return (unquoted_name, True). Otherwise return (name, False)."""
+ if not name:
+ return (name, False)
+ try:
+ dummy_sql = "SELECT 1 AS " + name
+ ast = sqlglot.parse_one(dummy_sql, dialect=dialect)
+ ident = ast.expressions[0].args.get("alias")
+ if ident and ident.quoted:
+ return (ident.this, True)
+ except:
+ pass
+ return (name, False)
+
+def table_parts(table):
+ """
+ Extract (catalog, schema, table) 3-tuple from a table expression.
+ Returns None if the table doesn't have a valid name (e.g., UDTFs).
+
+ SQLGlot naming:
+ - table.catalog → SQL catalog (e.g., BigQuery project, Snowflake database)
+ - table.db → SQL schema (e.g., BigQuery dataset, Postgres schema)
+ - table.name → table name
+ """
+ name = table.name
+ if not isinstance(name, str) or not name:
+ # UDTFs and other function-based sources don't have traditional table names
+ return None
+ return (table.catalog or None, table.db or None, name)
+
+def referenced_tables(sql: str, dialect: str = "postgres") -> str:
+ """
+ Extract table references from a SQL query.
+
+ Returns a JSON array of [catalog, schema, table] 3-tuples:
+ [[null, null, "users"], [null, "public", "orders"], ["myproject", "dataset", "events"]]
+
+ Excludes CTEs, subquery aliases, and UDTFs.
+
+ :param sql: SQL query string
+ :param dialect: SQL dialect (postgres, mysql, snowflake, bigquery, redshift, duckdb)
+
+ Examples:
+ referenced_tables("SELECT * FROM users")
+ => '[[null, null, "users"]]'
+
+ referenced_tables("SELECT * FROM public.users")
+ => '[[null, "public", "users"]]'
+
+ referenced_tables("SELECT * FROM myproject.analytics.events", "bigquery")
+ => '[["myproject", "analytics", "events"]]'
+ """
+ ast = sqlglot.parse_one(sql, read=dialect)
+ root_scope = optimizer.build_scope(ast)
+
+ tables = set()
+ for scope in root_scope.traverse():
+ for source in scope.sources.values():
+ if isinstance(source, exp.Table):
+ parts = table_parts(source)
+ if parts is not None:
+ tables.add(parts)
+
+ # Sort for deterministic output (nulls sort first via empty string)
+ return json.dumps(sorted(tables, key=lambda x: (x[0] or "", x[1] or "", x[2])))
+
+def referenced_fields(sql: str, dialect: str = "postgres") -> str:
+ """
+ Extract field references from a SQL query, returning only fields from actual database tables.
+
+ Returns a JSON array of [catalog, schema, table, field] 4-tuples:
+ [[null, null, "users", "id"], [null, "public", "orders", "total"]]
+
+ Includes:
+ - Wildcards as [catalog, schema, table, "*"]
+ - All specific column references
+
+ Excludes:
+ - Fields created in CTEs or subqueries
+ - Aliases (returns actual table names, not their aliases)
+ - Computed/derived columns
+
+ :param sql: SQL query string
+ :param dialect: SQL dialect (postgres, mysql, snowflake, bigquery, redshift, duckdb)
+
+ Examples:
+ referenced_fields("SELECT id FROM users", "postgres")
+ => '[[null, null, "users", "id"]]'
+
+ referenced_fields("SELECT * FROM public.users", "postgres")
+ => '[[null, "public", "users", "*"]]'
+
+ referenced_fields("SELECT * FROM myproject.analytics.events", "bigquery")
+ => '[["myproject", "analytics", "events", "*"]]'
+ """
+ ast = sqlglot.parse_one(sql, read=dialect)
+ root_scope = optimizer.build_scope(ast)
+
+ fields = set()
+
+ # Collect all CTE names to exclude them
+ cte_names = set()
+ for cte in ast.find_all(exp.CTE):
+ if cte.alias:
+ cte_names.add(cte.alias)
+
+ # Track scopes with unqualified wildcards
+ unqualified_wildcard_scopes = []
+
+ # Traverse all scopes to find column references
+ for scope in root_scope.traverse():
+ # Build a mapping of table aliases to table_parts tuples (catalog, schema, table)
+ # Only include actual tables, not CTEs or subqueries
+ alias_to_table_parts = {}
+ for alias, source in scope.sources.items():
+ if isinstance(source, exp.Table):
+ table_name = source.name
+ # Skip if this is actually a CTE reference
+ if table_name and table_name not in cte_names and alias not in cte_names:
+ parts = table_parts(source)
+ if parts is not None:
+ alias_to_table_parts[alias] = parts
+
+ # Check for unqualified wildcards (SELECT * without table qualification)
+ if isinstance(scope.expression, exp.Select):
+ for expr in scope.expression.expressions:
+ if isinstance(expr, exp.Star):
+ unqualified_wildcard_scopes.append((scope, alias_to_table_parts))
+ break
+
+ # Find all column references in this scope
+ for column in scope.expression.find_all(exp.Column):
+ column_name = column.name
+ table_ref = column.table
+
+ # Handle qualified wildcards (e.g., "users.*")
+ if column_name == "*":
+ if table_ref:
+ parts = alias_to_table_parts.get(table_ref)
+ if parts is not None:
+ # 4-tuple: (catalog, schema, table, field)
+ fields.add(parts + ("*",))
+ continue
+
+ # Regular column references
+ if table_ref:
+ parts = alias_to_table_parts.get(table_ref)
+ if parts is not None:
+ fields.add(parts + (column_name,))
+ else:
+ # Column without explicit table qualifier
+ # If there's only one source table, use that
+ if len(alias_to_table_parts) == 1:
+ parts = list(alias_to_table_parts.values())[0]
+ fields.add(parts + (column_name,))
+
+ # For scopes with unqualified wildcards, add wildcard entries for all tables
+ for scope, alias_to_table_parts in unqualified_wildcard_scopes:
+ for parts in alias_to_table_parts.values():
+ fields.add(parts + ("*",))
+
+ # Sort for deterministic output: catalog, schema, table, field
+ return json.dumps(sorted(fields, key=lambda x: (x[0] or "", x[1] or "", x[2], x[3])))
+
+def serialize_error(e):
+ """
+ Convert SQLGlot exceptions to structured error dicts.
+
+ Unrecognized error formats fall through to 'unhandled' type - the Clojure
+ layer can add more specific handling if needed.
+ """
+
+ message = e.args[0]
+ assert isinstance(message, str), "Unexpected error format."
+
+ match = re.search(r'^Unknown table:\s*(?P<table>.*)', message)
+ if match:
+ return {"status": "error",
+ "type": "unknown_table",
+ "message": match.group(),
+ "table": match.group('table')}
+
+ match = re.search(r'''(?P<message>^Column '(?P<column>\S+)' could not be resolved.)(?P<details>.*)''', message)
+ if match:
+ return {"status": "error",
+ "type": "column_not_resolved",
+ "message": match.group('message'),
+ "column": match.group('column'),
+ "details": match.group('details')}
+
+ # ('Invalid expression / Unexpected token. Line 1, Col: 23.\n complete nonsense \x1b[4mquery\x1b[0m',)
+ match = re.search(r'(?P<message>^Invalid expression.*?\.)(?P<details>.*)', message)
+ if match:
+ return {"status": "error",
+ "type": "invalid_expression",
+ "message": match.group('message'),
+ "details": match.group('details')}
+
+ return {"status": "error",
+ "type": "unhandled",
+ "message": message}
+
+def validate_query(dialect, sql, default_table_schema, sqlglot_schema_json):
+ """
+ Validate a SQL query against a schema using sqlglot's qualify optimizer.
+
+ Validation modes:
+ - Strict mode (sqlglot_schema provided): Validates columns and tables exist in schema.
+ Returns errors for unknown tables, unresolved columns, missing table aliases.
+ - Permissive mode (sqlglot_schema is None/empty): Only checks SQL syntax, infers schema
+ from query structure. Useful for UDTFs and queries against unknown tables.
+
+ Returns JSON with:
+ - If valid: {"status": "ok"}
+ - If error: {"status": "error", "type": "...", "message": "...", ...}
+
+ Error types:
+ - unknown_table: Table reference not found in schema (strict mode only)
+ - column_not_resolved: Column not found (includes 'column' and optionally 'details')
+ - invalid_expression: Syntax/parse error
+ - unhandled: Other errors
+
+ :param dialect: SQLGlot dialect string (e.g., "postgres", "mysql")
+ :param sql: SQL query string to validate
+ :param default_table_schema: Default schema for unqualified tables
+ :param sqlglot_schema_json: JSON-encoded schema dict {schema: {table: {column: type}}},
+ or None/empty for permissive mode
+ """
+ status = {"status": "ok"}
+
+ # Decode JSON schema if provided (GraalVM Polyglot passes complex maps as JSON strings)
+ sqlglot_schema = json.loads(sqlglot_schema_json) if sqlglot_schema_json else None
+
+ # Determine validation mode based on whether schema is provided
+ # - Strict mode: schema provided → validate columns/tables exist
+ # - Permissive mode: no schema → only check syntax, infer schema from query
+ strict_mode = sqlglot_schema is not None and len(sqlglot_schema) > 0
+
+ try:
+ ast = sqlglot.parse_one(sql, read=dialect)
+ ast = qualify.qualify(ast,
+ db=default_table_schema,
+ dialect=dialect,
+ schema=sqlglot_schema if strict_mode else None,
+ # In strict mode: don't infer, validate against provided schema
+ # In permissive mode: infer schema from query (handles UDTFs)
+ infer_schema=not strict_mode,
+ # In strict mode: validate that columns can be resolved
+ validate_qualify_columns=strict_mode,
+ sql=sql)
+ except ParseError as e:
+ status |= serialize_error(e)
+ except OptimizeError as e:
+ status |= serialize_error(e)
+
+ return json.dumps(status)
+
+#############################################################################
+# Experimental (using lineage)
+#############################################################################
+
+def is_pure_column(root):
+ """
+ Check whether the lineage graph is a path of columns terminated by table.
+
+ :param root: The root of the lineage graph
+ """
+ for node in root.walk():
+ expression = node.expression
+ downstream = node.downstream
+
+ # We are looking for "path"...
+ if len(downstream) > 1:
+ return False
+
+ examined = None
+ if (isinstance(expression, exp.Alias)):
+ examined = expression.this
+ else:
+ examined = expression
+
+ # ...of `exp.Column`s...
+ if isinstance(examined, exp.Column):
+ continue
+ # ...terminated by the `exp.Table`
+ elif isinstance(examined, exp.Table):
+ continue
+ else:
+ return False
+ return True
+
+def simple_query(sql: str, dialect: str = None) -> str:
+ """
+ Check if SQL is a simple SELECT (no LIMIT, OFFSET, or CTEs).
+
+ Used by Workspaces to determine if automatic checkpoints can be inserted.
+
+ Returns a JSON object with:
+ - If simple: {"is_simple": true}
+ - If not simple: {"is_simple": false, "reason": "..."}
+
+ :param sql: SQL query string to check
+ :param dialect: SQL dialect (postgres, mysql, snowflake, bigquery, etc.)
+
+ Examples:
+ simple_query("SELECT * FROM users")
+ => '{"is_simple": true}'
+
+ simple_query("SELECT * FROM users LIMIT 10")
+ => '{"is_simple": false, "reason": "Contains a LIMIT"}'
+
+ simple_query("WITH cte AS (SELECT 1) SELECT * FROM cte")
+ => '{"is_simple": false, "reason": "Contains a CTE"}'
+ """
+ try:
+ ast = sqlglot.parse_one(sql, read=dialect)
+
+ # Must be a SELECT or Query type (CTE queries parse as Select with 'with' arg)
+ if not isinstance(ast, (exp.Select, exp.Query)):
+ return json.dumps({"is_simple": False, "reason": "Not a simple SELECT"})
+
+ # No CTEs (WITH clause) - SQLGlot uses "with_" (with underscore) as the arg name
+ if ast.args.get("with_"):
+ return json.dumps({"is_simple": False, "reason": "Contains a CTE"})
+
+ # For non-Select Query types (Union, Intersect, etc.), reject
+ if not isinstance(ast, exp.Select):
+ return json.dumps({"is_simple": False, "reason": "Not a simple SELECT"})
+
+ # No LIMIT
+ if ast.args.get("limit"):
+ return json.dumps({"is_simple": False, "reason": "Contains a LIMIT"})
+
+ # No OFFSET
+ if ast.args.get("offset"):
+ return json.dumps({"is_simple": False, "reason": "Contains an OFFSET"})
+
+ return json.dumps({"is_simple": True})
+ except ParseError as e:
+ return json.dumps({"is_simple": False, "reason": str(e)})
+ except Exception as e:
+ return json.dumps({"is_simple": False, "reason": f"Unexpected error: {str(e)}"})
+
+def add_into_clause(sql: str, table_name: str, dialect: str = None) -> str:
+ """
+ Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.
+
+ Transforms: SELECT * FROM products
+ Into: SELECT * INTO "new_table" FROM products
+
+ Used by SQL Server transforms which require SELECT INTO syntax
+ instead of CREATE TABLE AS SELECT.
+
+ :param sql: The SELECT SQL query string
+ :param table_name: The target table name (already formatted/quoted)
+ :param dialect: SQL dialect (e.g., "tsql" for SQL Server)
+ :return: Modified SQL string with INTO clause
+
+ Examples:
+ add_into_clause("SELECT * FROM products", '"PRODUCTS_COPY"', "tsql")
+ => 'SELECT * INTO "PRODUCTS_COPY" FROM products'
+ """
+ ast = sqlglot.parse_one(sql, read=dialect)
+
+ if not isinstance(ast, exp.Select):
+ raise ValueError("SQL must be a SELECT statement")
+
+ # Create the Into expression with the table identifier
+ # The table_name is pre-formatted, so parse it to preserve quotes
+ into = exp.Into(this=exp.to_table(table_name))
+
+ # Set the into clause on the select
+ ast.set("into", into)
+
+ # Generate SQL in the target dialect
+ return ast.sql(dialect=dialect)
+
+
+def returned_columns_lineage(dialect, sql, default_table_schema, sqlglot_schema_json):
+ """
+ Extract column lineage from SQL query.
+
+ Args:
+ dialect: SQL dialect string (e.g., "postgres", "mysql")
+ sql: SQL query string
+ default_table_schema: Default schema name (e.g., "public")
+ sqlglot_schema_json: JSON-encoded schema map (to avoid GraalVM polyglot issues)
+
+ Returns:
+ JSON array of [alias, is_pure, dependencies] tuples
+ """
+ # Decode schema from JSON (passed from Clojure to avoid polyglot map issues)
+ sqlglot_schema = json.loads(sqlglot_schema_json) if sqlglot_schema_json else None
+
+ ast = sqlglot.parse_one(sql, read=dialect)
+ ast = qualify.qualify(ast,
+ db=default_table_schema,
+ dialect=dialect,
+ schema=sqlglot_schema,
+ infer_schema=True,
+ sql=sql)
+
+ assert isinstance(ast, exp.Query), "Ast does not represent a `Query`."
+
+ dependencies = []
+ for select in ast.named_selects:
+
+ # Note: SQL allows duplicate column aliases (e.g., SELECT 1 AS x, 2 AS x).
+ # We track columns by position, not by name, so duplicates are fine.
+
+ select_elm_lineage = lineage.lineage(select, ast.sql(dialect), dialect=dialect)
+ is_pure_select = is_pure_column(select_elm_lineage)
+
+ leaves = set()
+ for node in select_elm_lineage.walk():
+ if (# Leaf nodes of `.walk` are `exp.Table`s. We are interested
+ # in the columns selected from those tables (i.e. penultimate
+ # level). The condition is intended to examine those columns.
+ len(node.downstream) == 1
+ and isinstance(node.downstream[0].expression, exp.Table)
+ and not node.downstream[0].downstream):
+ # For UDTFs/table functions, the penultimate node may not be a Column
+ # (e.g., FLATTEN results). Skip these gracefully.
+ if not isinstance(node.expression.this, exp.Column):
+ continue
+ # Extract table parts (catalog, schema, table) from the source.
+ # Note: Schema may be missing when table is aliased to its own name.
+ table = table_parts(node.downstream[0].expression)
+ # Skip UDTFs and other function-based sources that don't have real table names
+ if table is not None:
+ namespaced_column = table + (node.expression.this.name, )
+ leaves.add(namespaced_column)
+
+ dependencies.append((select, is_pure_select, tuple(leaves), ))
+
+ return json.dumps(dependencies)
+
+
+def replace_names(sql: str, replacements_json: str, dialect: str = None) -> str:
+ """
+ Replace schema, table, and column names in a SQL query.
+
+ SECURITY NOTE: Replacement values are injected into the SQL AST without sanitization.
+ This is safe because the only caller (workspaces/execute.clj) passes system-generated
+ values: isolation schema names (mb__isolation_<slug>_<id>) and table names from
+ database metadata. User input never flows into replacement values.
+
+ Args:
+ sql: SQL query string
+ replacements_json: JSON-encoded map with keys:
+ - schemas: {old_schema: new_schema}
+ - tables: list of [[{schema?, table}, new_name], ...]
+ - columns: list of [[{schema?, table?, column}, new_name], ...]
+ dialect: SQL dialect string
+
+ Returns:
+ Modified SQL string with replacements applied.
+
+ Examples:
+ replace_names("SELECT * FROM people", '{"tables": [[{"table": "people"}, "users"]]}', "postgres")
+ => 'SELECT * FROM users'
+
+ replace_names("SELECT id FROM orders", '{"columns": [[{"table": "orders", "column": "id"}, "user_id"]]}')
+ => 'SELECT user_id FROM orders'
+ """
+ replacements = json.loads(replacements_json)
+ schemas = replacements.get("schemas") or {}
+ tables = replacements.get("tables") or [] # List of [key, value] pairs
+ columns = replacements.get("columns") or [] # List of [key, value] pairs
+
+ # Convert list-of-pairs to lookup dicts for O(1) matching
+ # Tables: (schema, table) -> new_name
+ table_map = {}
+ for item in tables:
+ key, new_name = item
+ schema = key.get("schema") # may be None
+ table = key["table"]
+ table_map[(schema, table)] = new_name
+
+ # Columns: (schema, table, column) -> new_name
+ column_map = {}
+ for item in columns:
+ key, new_name = item
+ schema = key.get("schema")
+ table = key.get("table") # may be None for unqualified
+ column = key["column"]
+ column_map[(schema, table, column)] = new_name
+
+ ast = sqlglot.parse_one(sql, read=dialect)
+
+ def rename_fn(node):
+ # Schema rename (appears in Table.db)
+ if isinstance(node, exp.Table):
+ # Capture original values BEFORE any modifications (important for lookup)
+ original_schema = node.db
+ original_table = node.name
+ # Preserve original quoting status for renamed identifiers
+ original_schema_quoted = node.args.get("db").quoted if node.args.get("db") else False
+ original_table_quoted = node.this.quoted if node.this else False
+
+ # Rename schema if present
+ if original_schema and original_schema in schemas:
+ raw_schema, was_quoted = unquote_identifier(schemas[original_schema], dialect)
+ schema_quoted = original_schema_quoted or was_quoted or needs_quoting(raw_schema, dialect)
+ node.set("db", exp.Identifier(this=raw_schema, quoted=schema_quoted))
+
+ # Rename table - try exact match first (with original schema), then without schema
+ new_table = (table_map.get((original_schema, original_table)) or
+ table_map.get((None, original_table)))
+ if new_table:
+ if isinstance(new_table, dict):
+ # New format: {schema: x, table: y}
+ if new_table.get("schema"):
+ # When injecting a new schema, quote if it contains special characters
+ raw_schema, was_quoted = unquote_identifier(new_table["schema"], dialect)
+ schema_quoted = original_schema_quoted or was_quoted or needs_quoting(raw_schema, dialect)
+ node.set("db", exp.Identifier(this=raw_schema, quoted=schema_quoted))
+ if new_table.get("table"):
+ raw_table, was_quoted = unquote_identifier(new_table["table"], dialect)
+ table_quoted = original_table_quoted or was_quoted or needs_quoting(raw_table, dialect)
+ node.set("this", exp.Identifier(this=raw_table, quoted=table_quoted))
+ else:
+ # String: just the table name
+ raw_name, was_quoted = unquote_identifier(new_table, dialect)
+ table_quoted = original_table_quoted or was_quoted or needs_quoting(raw_name, dialect)
+ node.set("this", exp.Identifier(this=raw_name, quoted=table_quoted))
+
+ # Column rename
+ elif isinstance(node, exp.Column):
+ col_name = node.name
+ col_table = node.table # May be None if column is unqualified (e.g., "SELECT id" not "SELECT t.id")
+ # Preserve original quoting status
+ original_col_quoted = node.this.quoted if node.this else False
+
+ # Try to find a matching column rename.
+ # The challenge: replacement key might be {:table "orders" :column "id"}
+ # but the SQL column ref might just be "id" (unqualified).
+ # We need to match flexibly:
+ # - Exact match: (schema, table, column) all match
+ # - Table match: column and table match, schema is None in key
+ # - Column-only match: just column matches (when no table qualifier in SQL)
+ new_col = None
+
+ # Iterate through all column mappings and find best match
+ for (key_schema, key_table, key_col), new_name in column_map.items():
+ if key_col != col_name:
+ continue
+ # Column name matches, now check table qualifier
+ # Note: SQLGlot uses empty string (not None) for missing table qualifier
+ if col_table:
+ # SQL has table qualifier - match if tables are equal
+ if key_table == col_table:
+ new_col = new_name
+ break
+ else:
+ # SQL has no table qualifier - accept any table in key
+ # (this is the common case: "SELECT id FROM orders" with key {:table "orders" :column "id"})
+ new_col = new_name
+ break
+
+ if new_col:
+ node.set("this", exp.Identifier(this=new_col, quoted=original_col_quoted))
+
+ return node
+
+ transformed = ast.transform(rename_fn)
+ return transformed.sql(dialect=dialect)
+
+
+#############################################################################
+# Field References (Macaw-compatible output)
+#############################################################################
+
+def field_references(sql: str, dialect: str = "postgres") -> str:
+ """
+ Extract field references from SQL, returning used and returned fields.
+
+ This is the SQLGlot equivalent of Macaw's field-references function.
+ Returns a JSON object with:
+ - used_fields: list of field specs from WHERE, JOIN ON, GROUP BY, ORDER BY
+ - returned_fields: list of field specs from SELECT clause (ordered)
+ - errors: list of validation errors
+
+ Each field spec has:
+ - type: single_column, all_columns, custom_field, composite_field, or unknown_columns
+ - column: column name (for single_column)
+ - alias: column alias (null if none)
+ - source_columns: nested list of possible source columns
+ - table: table info (for all_columns)
+ - used_fields: list of fields used (for custom_field)
+ - member_fields: list of fields (for composite_field)
+ """
+ try:
+ ast = sqlglot.parse_one(sql, read=dialect)
+ except ParseError:
+ return json.dumps({
+ "used_fields": [],
+ "returned_fields": [],
+ "errors": [{"type": "syntax_error"}]
+ })
+
+ walker = FieldReferenceWalker(ast, dialect)
+ result = walker.walk()
+
+ # Convert sets to lists for JSON serialization
+ result["used_fields"] = list(result["used_fields"])
+ result["errors"] = list(result["errors"])
+
+ return json.dumps(result, default=_json_default)
+
+
+def _json_default(obj):
+ """JSON serializer for objects not serializable by default json code"""
+ if isinstance(obj, set):
+ return list(obj)
+ if isinstance(obj, frozenset):
+ return list(obj)
+ raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+
+class FieldReferenceWalker:
+ """
+ Walks SQLGlot AST to extract field references matching Macaw's output format.
+
+ Key concepts:
+ - sources: Nested list of source tables/subqueries. Outer list = scope levels,
+ inner list = same-scope sources (FROM + JOINs).
+ - withs: Set of CTE definitions, progressively built.
+ """
+
+ def __init__(self, ast, dialect):
+ self.ast = ast
+ self.dialect = dialect
+
+ def walk(self):
+ """Main entry point - returns {used_fields, returned_fields, errors}"""
+ return self._process_query(self.ast, outside_sources=[], withs={})
+
+ def _process_query(self, expr, outside_sources, withs):
+ """Process a query node (SELECT, UNION, etc.)"""
+ if isinstance(expr, exp.Select):
+ return self._process_select(expr, outside_sources, withs)
+ elif isinstance(expr, exp.Union):
+ return self._process_union(expr, outside_sources, withs)
+ else:
+ # Unknown query type
+ return {
+ "used_fields": set(),
+ "returned_fields": [],
+ "errors": {self._syntax_error()}
+ }
+
+ def _process_select(self, select_expr, outside_sources, withs):
+ """Process a SELECT statement."""
+ errors = set() # Set of frozen errors for deduplication
+
+ # Handle CTEs (WITH clause)
+ local_withs = dict(withs)
+ with_clause = select_expr.args.get("with_")
+ if with_clause:
+ is_recursive = with_clause.args.get("recursive")
+ for cte in with_clause.expressions:
+ cte_name = cte.alias
+ cte_body = cte.this
+
+ if is_recursive and isinstance(cte_body, exp.Union):
+ # For recursive CTEs with UNION, process the base case first
+ # to get the returned_fields structure, then process the full body.
+ # The base case is the left side of the UNION.
+ base_result = self._process_query(cte_body.left, outside_sources, local_withs)
+
+ # Now add the CTE to local_withs with the base case's returned_fields
+ # so the recursive part can reference it.
+ # Mark it as the "current CTE" so we can detect self-references.
+ local_withs[cte_name] = {
+ "names": {"table": cte_name, "table_alias": cte_name},
+ "returned_fields": base_result["returned_fields"],
+ "used_fields": base_result["used_fields"],
+ "errors": base_result["errors"],
+ "is_recursive_cte": True # Marker survives JOIN aliasing
+ }
+
+ # Process the full CTE body (UNION) with the CTE now available
+ cte_result = self._process_query(cte_body, outside_sources, local_withs)
+
+ # Keep is_recursive_cte flag in final entry
+ local_withs[cte_name] = {
+ "names": {"table": cte_name},
+ "returned_fields": cte_result["returned_fields"],
+ "used_fields": cte_result["used_fields"],
+ "errors": cte_result["errors"],
+ "is_recursive_cte": True
+ }
+ else:
+ # Non-recursive CTE - just process normally
+ cte_result = self._process_query(cte_body, outside_sources, local_withs)
+
+ local_withs[cte_name] = {
+ "names": {"table": cte_name},
+ "returned_fields": cte_result["returned_fields"],
+ "used_fields": cte_result["used_fields"],
+ "errors": cte_result["errors"]
+ }
+ # cte_result["errors"] is a list of dicts, freeze them for set membership
+ for e in cte_result["errors"]:
+ errors.add(self._freeze_field(e) if isinstance(e, dict) else e)
+
+ # Build local sources from FROM + JOINs
+ local_sources = self._build_sources(select_expr, outside_sources, local_withs)
+ combined_sources = [local_sources] + outside_sources
+
+ # Process returned fields (SELECT clause)
+ returned_fields = []
+ for select_item in select_expr.expressions:
+ fields = self._find_returned_fields(select_item, combined_sources, local_withs)
+ for field_result in fields:
+ if "col" in field_result:
+ returned_fields.append(field_result["col"])
+ if "errors" in field_result:
+ errors.update(field_result["errors"])
+
+ # Build sources including SELECT aliases for ORDER BY resolution
+ # SELECT aliases should be a separate scope level from table sources,
+ # so that source_columns properly nests [select_aliases] [table_sources]
+ select_source = {
+ "names": None,
+ "returned_fields": [f for f in returned_fields if f.get("alias")],
+ "used_fields": set(),
+ "errors": set()
+ }
+ sources_with_select = [[select_source]] + [local_sources] + outside_sources
+
+ # Process used fields
+ used_fields = set()
+
+ # SELECT clause (references in expressions, not the returned columns themselves)
+ for select_item in select_expr.expressions:
+ for field_result in self._find_used_fields(select_item, combined_sources, local_withs):
+ if "col" in field_result:
+ used_fields.add(self._freeze_field(field_result["col"]))
+ if "errors" in field_result:
+ errors.update(field_result["errors"])
+
+ # WHERE clause (include select aliases for lineage tracking, matching Macaw)
+ where = select_expr.args.get("where")
+ if where:
+ for field_result in self._find_used_fields(where, sources_with_select, local_withs):
+ if "col" in field_result:
+ used_fields.add(self._freeze_field(field_result["col"]))
+ if "errors" in field_result:
+ errors.update(field_result["errors"])
+
+ # JOIN ON conditions (include select aliases for lineage tracking, matching Macaw)
+ for join in select_expr.args.get("joins", []):
+ on_clause = join.args.get("on")
+ if on_clause:
+ for field_result in self._find_used_fields(on_clause, sources_with_select, local_withs):
+ if "col" in field_result:
+ used_fields.add(self._freeze_field(field_result["col"]))
+ if "errors" in field_result:
+ errors.update(field_result["errors"])
+
+ # GROUP BY
+ group = select_expr.args.get("group")
+ if group:
+ for field_result in self._find_used_fields(group, sources_with_select, local_withs):
+ if "col" in field_result:
+ used_fields.add(self._freeze_field(field_result["col"]))
+ if "errors" in field_result:
+ errors.update(field_result["errors"])
+
+ # ORDER BY (can reference SELECT aliases)
+ order = select_expr.args.get("order")
+ if order:
+ for field_result in self._find_used_fields(order, sources_with_select, local_withs):
+ if "col" in field_result:
+ used_fields.add(self._freeze_field(field_result["col"]))
+ if "errors" in field_result:
+ errors.update(field_result["errors"])
+
+ # HAVING clause
+ having = select_expr.args.get("having")
+ if having:
+ for field_result in self._find_used_fields(having, sources_with_select, local_withs):
+ if "col" in field_result:
+ used_fields.add(self._freeze_field(field_result["col"]))
+ if "errors" in field_result:
+ errors.update(field_result["errors"])
+
+ # Include used_fields from local sources (CTEs)
+ for source in local_sources:
+ for field in source.get("used_fields", set()):
+ if isinstance(field, dict):
+ used_fields.add(self._freeze_field(field))
+ else:
+ used_fields.add(field)
+
+ # Include used_fields from CTEs
+ for cte_source in local_withs.values():
+ for field in cte_source.get("used_fields", set()):
+ if isinstance(field, dict):
+ used_fields.add(self._freeze_field(field))
+ else:
+ used_fields.add(field)
+
+ return {
+ "used_fields": [self._unfreeze_field(f) for f in used_fields],
+ "returned_fields": returned_fields,
+ "errors": [self._unfreeze_field(e) for e in errors]
+ }
+
+ def _process_union(self, union_expr, outside_sources, withs):
+ """Process UNION/INTERSECT/EXCEPT - creates composite_field for returned columns."""
+ left_result = self._process_query(union_expr.left, outside_sources, withs)
+ right_result = self._process_query(union_expr.right, outside_sources, withs)
+
+ # Combine used_fields
+ used_fields = set()
+ for f in left_result["used_fields"]:
+ used_fields.add(self._freeze_field(f) if isinstance(f, dict) else f)
+ for f in right_result["used_fields"]:
+ used_fields.add(self._freeze_field(f) if isinstance(f, dict) else f)
+
+ # Create composite returned_fields
+ returned_fields = []
+ left_fields = left_result["returned_fields"]
+ right_fields = right_result["returned_fields"]
+
+ for i in range(max(len(left_fields), len(right_fields))):
+ left_field = left_fields[i] if i < len(left_fields) else None
+ right_field = right_fields[i] if i < len(right_fields) else None
+
+ if left_field:
+ alias = left_field.get("alias") or left_field.get("column")
+ elif right_field:
+ alias = right_field.get("alias") or right_field.get("column")
+ else:
+ alias = None
+
+ member_fields = []
+ if left_field:
+ member_fields.append(left_field)
+ if right_field:
+ member_fields.append(right_field)
+
+ returned_fields.append({
+ "type": "composite_field",
+ "alias": alias,
+ "member_fields": member_fields
+ })
+
+ all_errors = set()
+ for e in left_result["errors"]:
+ if isinstance(e, dict):
+ all_errors.add(self._freeze_field(e))
+ else:
+ all_errors.add(e)
+ for e in right_result["errors"]:
+ if isinstance(e, dict):
+ all_errors.add(self._freeze_field(e))
+ else:
+ all_errors.add(e)
+
+ return {
+ "used_fields": [self._unfreeze_field(f) for f in used_fields],
+ "returned_fields": returned_fields,
+ "errors": [self._unfreeze_field(e) for e in all_errors]
+ }
+
+ def _build_sources(self, select_expr, outside_sources, withs):
+ """Build source list from FROM clause and JOINs.
+
+ Note: Macaw processes JOINs first, then appends FROM at the end.
+ This affects the order when expanding wildcards.
+ """
+ sources = []
+
+ # Process JOINs first (matching Macaw's order)
+ for join in select_expr.args.get("joins", []):
+ source = self._process_source(join.this, outside_sources, withs)
+ if source:
+ sources.append(source)
+
+ # Then append FROM clause at the end
+ from_clause = select_expr.args.get("from_")
+ if from_clause:
+ source = self._process_source(from_clause.this, outside_sources, withs)
+ if source:
+ sources.append(source)
+
+ return sources
+
+ def _process_source(self, source_expr, outside_sources, withs):
+ """Process a source (table, subquery, or CTE reference)."""
+ # Check for table functions first (UDTF or any UDTF subclass like GenerateSeries)
+ # Note: isinstance checks for subclasses too, so exp.UDTF catches GenerateSeries
+ if isinstance(source_expr, exp.UDTF):
+ # Table function (GENERATE_SERIES, etc.)
+ alias = source_expr.alias
+ return {
+ "names": {"table_alias": alias} if alias else None,
+ "returned_fields": [{"type": "unknown_columns"}],
+ "used_fields": set(),
+ "errors": set()
+ }
+
+ if isinstance(source_expr, exp.Table):
+ table_name = source_expr.name
+ table_alias = source_expr.alias
+
+ # Check if it's a CTE reference
+ if table_name in withs:
+ cte_data = withs[table_name]
+ cte_table_info = {"table": table_name}
+ if table_alias:
+ cte_table_info["table_alias"] = table_alias
+ cte_source = dict(cte_data)
+ cte_source["names"] = cte_table_info
+ return cte_source
+
+ # A table with empty name but an alias might be a table function result
+ # that SQLGlot didn't recognize. Treat as unknown_columns.
+ if not table_name and table_alias:
+ return {
+ "names": {"table_alias": table_alias},
+ "returned_fields": [{"type": "unknown_columns"}],
+ "used_fields": set(),
+ "errors": set()
+ }
+
+ # A table with empty name whose 'this' is a function is a table function
+ # that SQLGlot parsed as Table instead of UDTF (e.g., my_function(1, 100))
+ if not table_name and isinstance(source_expr.this, exp.Func):
+ alias = source_expr.alias
+ return {
+ "names": {"table_alias": alias} if alias else None,
+ "returned_fields": [{"type": "unknown_columns"}],
+ "used_fields": set(),
+ "errors": set()
+ }
+
+ # Real table
+ table_info = {"table": table_name}
+ if source_expr.db:
+ table_info["schema"] = source_expr.db
+ if source_expr.catalog:
+ table_info["database"] = source_expr.catalog
+ if table_alias:
+ table_info["table_alias"] = table_alias
+
+ return {
+ "names": table_info,
+ "returned_fields": [{"type": "all_columns", "table": table_info}],
+ "used_fields": set(),
+ "errors": set()
+ }
+
+ elif isinstance(source_expr, exp.Subquery):
+ subquery_result = self._process_query(source_expr.this, outside_sources, withs)
+ alias = source_expr.alias
+ subquery_result["names"] = {"table_alias": alias} if alias else None
+ return subquery_result
+
+ return None
+
+ def _find_returned_fields(self, expr, sources, withs):
+ """Find returned fields from a SELECT expression."""
+ # Handle aliases
+ if isinstance(expr, exp.Alias):
+ inner_results = self._find_returned_fields(expr.this, sources, withs)
+ alias = expr.alias
+ for result in inner_results:
+ if "col" in result:
+ result["col"]["alias"] = alias
+ return inner_results
+
+ # Unqualified wildcard: SELECT *
+ if isinstance(expr, exp.Star):
+ results = []
+ if sources and sources[0]:
+ for source in sources[0]:
+ for field in source.get("returned_fields", []):
+ results.append({"col": field})
+ return results
+
+ # Column reference (including table.*)
+ if isinstance(expr, exp.Column):
+ column_name = expr.name
+ table_ref = expr.table
+
+ # Table wildcard: SELECT t.*
+ if column_name == "*":
+ if table_ref:
+ source = self._find_source({"table": table_ref}, sources)
+ if source:
+ return [{"col": field} for field in source.get("returned_fields", [])]
+ else:
+ return [{"errors": {self._missing_table_alias_error(table_ref)}}]
+ return []
+
+ # Regular column
+ return self._get_column(sources, expr, return_table_matches=True)
+
+ # Subquery in SELECT
+ if isinstance(expr, exp.Subquery):
+ subquery_result = self._process_query(expr.this, sources, withs)
+ # Single-column subquery returns that column
+ if subquery_result["returned_fields"]:
+ return [{"col": subquery_result["returned_fields"][0]}]
+ return []
+
+ # Everything else is a custom field (function, case, binary expression, etc.)
+ used_fields = []
+ for result in self._find_used_fields(expr, sources, withs):
+ if "col" in result:
+ used_fields.append(result["col"])
+
+ alias = expr.alias if hasattr(expr, "alias") and expr.alias else None
+ return [{
+ "col": {
+ "type": "custom_field",
+ "alias": alias,
+ # Don't freeze - these go directly to returned_fields which isn't unfrozen
+ "used_fields": used_fields
+ }
+ }]
+
+ def _find_used_fields(self, expr, sources, withs):
+ """Find fields used in an expression (WHERE, JOIN ON, etc.)."""
+ if expr is None:
+ return []
+
+ # Column reference
+ if isinstance(expr, exp.Column):
+ column_name = expr.name
+ table_ref = expr.table
+
+ # Table wildcard: t.* - expand to actual fields from source
+ if column_name == "*" and table_ref:
+ source = self._find_source({"table": table_ref}, sources)
+ if source:
+ # Expand wildcard to the actual fields from this source
+ # But skip all_columns and unknown_columns markers -
+ # they don't represent specific used columns
+ results = []
+ for field in source.get("returned_fields", []):
+ ftype = field.get("type")
+ if ftype not in ("all_columns", "unknown_columns"):
+ results.append({"col": field})
+ return results
+ # Bad table ref - return empty, error is tracked in _find_returned_fields
+ # Don't create a column entry for "*"
+ return []
+
+ return self._get_column(sources, expr, return_table_matches=False)
+
+ # Star/wildcard - no used fields
+ if isinstance(expr, exp.Star):
+ return []
+
+ # Subquery
+ if isinstance(expr, exp.Subquery):
+ subquery_result = self._process_query(expr.this, sources, withs)
+ results = []
+ for field in subquery_result["used_fields"]:
+ results.append({"col": field})
+ return results
+
+ # SELECT in expression context
+ if isinstance(expr, exp.Select):
+ select_result = self._process_query(expr, sources, withs)
+ results = []
+ for field in select_result["used_fields"]:
+ results.append({"col": field})
+ return results
+
+ # Alias - recurse into the aliased expression
+ # Only propagate alias if the Alias directly wraps a Column.
+ # For complex expressions (functions, etc.), the alias belongs to the
+ # expression result, not to the columns used inside.
+ if isinstance(expr, exp.Alias):
+ results = self._find_used_fields(expr.this, sources, withs)
+ # Only add alias if this Alias directly wraps a Column
+ if isinstance(expr.this, exp.Column):
+ alias = expr.alias
+ for result in results:
+ if "col" in result:
+ result["col"]["alias"] = alias
+ return results
+
+ # Binary expressions (AND, OR, =, <, >, etc.)
+ if isinstance(expr, exp.Binary):
+ left = self._find_used_fields(expr.left, sources, withs)
+ right = self._find_used_fields(expr.right, sources, withs)
+ return left + right
+
+ # Unary expressions (NOT, -, etc.)
+ if isinstance(expr, exp.Unary):
+ return self._find_used_fields(expr.this, sources, withs)
+
+ # CASE expression
+ if isinstance(expr, exp.Case):
+ results = []
+ # CASE <expr> WHEN...
+ if expr.this:
+ results.extend(self._find_used_fields(expr.this, sources, withs))
+ # WHEN conditions and THEN values
+ for ifs in expr.args.get("ifs", []):
+ results.extend(self._find_used_fields(ifs.this, sources, withs))
+ results.extend(self._find_used_fields(ifs.args.get("true"), sources, withs))
+ # ELSE
+ if expr.args.get("default"):
+ results.extend(self._find_used_fields(expr.args["default"], sources, withs))
+ return results
+
+ # BETWEEN
+ if isinstance(expr, exp.Between):
+ results = []
+ results.extend(self._find_used_fields(expr.this, sources, withs))
+ results.extend(self._find_used_fields(expr.args.get("low"), sources, withs))
+ results.extend(self._find_used_fields(expr.args.get("high"), sources, withs))
+ return results
+
+ # IS NULL / IS NOT NULL
+ if isinstance(expr, (exp.Is,)):
+ return self._find_used_fields(expr.this, sources, withs)
+
+ # IN expression
+ if isinstance(expr, exp.In):
+ results = self._find_used_fields(expr.this, sources, withs)
+ for item in expr.expressions:
+ results.extend(self._find_used_fields(item, sources, withs))
+ return results
+
+ # EXISTS
+ if isinstance(expr, exp.Exists):
+ return self._find_used_fields(expr.this, sources, withs)
+
+ # Functions
+ if isinstance(expr, exp.Func):
+ results = []
+ for arg in expr.args.values():
+ if isinstance(arg, list):
+ for item in arg:
+ results.extend(self._find_used_fields(item, sources, withs))
+ elif isinstance(arg, exp.Expression):
+ results.extend(self._find_used_fields(arg, sources, withs))
+ return results
+
+ # Window functions
+ if isinstance(expr, exp.Window):
+ results = []
+ results.extend(self._find_used_fields(expr.this, sources, withs))
+ # PARTITION BY
+ partition = expr.args.get("partition_by")
+ if partition:
+ for part in partition:
+ results.extend(self._find_used_fields(part, sources, withs))
+ # ORDER BY within window
+ order = expr.args.get("order")
+ if order:
+ results.extend(self._find_used_fields(order, sources, withs))
+ return results
+
+ # ORDER BY clause
+ if isinstance(expr, exp.Order):
+ results = []
+ for ordered in expr.expressions:
+ results.extend(self._find_used_fields(ordered.this, sources, withs))
+ return results
+
+ # GROUP BY clause
+ if isinstance(expr, exp.Group):
+ results = []
+ for item in expr.expressions:
+ results.extend(self._find_used_fields(item, sources, withs))
+ return results
+
+ # Ordered (for ORDER BY items)
+ if isinstance(expr, exp.Ordered):
+ return self._find_used_fields(expr.this, sources, withs)
+
+ # Cast
+ if isinstance(expr, exp.Cast):
+ return self._find_used_fields(expr.this, sources, withs)
+
+ # Paren (parenthesized expression)
+ if isinstance(expr, exp.Paren):
+ return self._find_used_fields(expr.this, sources, withs)
+
+ # Interval, Literal, etc. - no columns
+ if isinstance(expr, (exp.Literal, exp.Interval, exp.DataType, exp.Boolean)):
+ return []
+
+ # Identifier (column name without table) - treat as column
+ if isinstance(expr, exp.Identifier):
+ # Create a fake column to reuse column resolution
+ fake_col = exp.Column(this=expr)
+ return self._get_column(sources, fake_col, return_table_matches=False)
+
+ # Fallback: try to walk children
+ results = []
+ for child in expr.iter_expressions():
+ results.extend(self._find_used_fields(child, sources, withs))
+ return results
+
+ def _get_column(self, sources, column_expr, return_table_matches):
+ """Get column reference with source resolution."""
+ column_name = column_expr.name
+ table_ref = column_expr.table
+
+ # Find valid sources
+ if table_ref:
+ source = self._find_source({"table": table_ref}, sources)
+ valid_sources = [[source]] if source else [[]]
+ else:
+ valid_sources = sources
+
+ # Build source_columns (nested list of possible sources)
+ # Pass through returned_fields from all sources (matching Macaw behavior).
+ # This preserves lineage back to original tables through CTEs/subqueries.
+ source_columns = []
+ for scope_sources in valid_sources:
+ if isinstance(scope_sources, list):
+ scope_fields = []
+ for source in scope_sources:
+ if source and self._source_might_have_column(source, column_name, table_ref):
+ returned_fields = source.get("returned_fields", [])
+ scope_fields.extend(returned_fields)
+ if scope_fields:
+ source_columns.append(scope_fields)
+
+ # Check for direct column match in source_columns (matching Macaw)
+ # Search in (first source_columns), not in the raw returned_fields
+ source_column = None
+ if source_columns:
+ for field in source_columns[0]:
+ field_name = field.get("alias") or field.get("column")
+ if field_name and field_name.lower() == column_name.lower():
+ source_column = field
+ break
+
+ # Check for table alias match (SELECT o FROM (SELECT * FROM orders) AS o)
+ if not source_column and not table_ref:
+ for scope_sources in valid_sources:
+ if isinstance(scope_sources, list):
+ for source in scope_sources:
+ if source:
+ names = source.get("names", {})
+ table_alias = names.get("table_alias") if names else None
+ table_name = names.get("table") if names else None
+ if table_alias and table_alias.lower() == column_name.lower():
+ if return_table_matches:
+ return [{
+ "col": {
+ "type": "custom_field",
+ "alias": column_name,
+ # Don't freeze - goes directly to returned_fields
+ "used_fields": list(source.get("returned_fields", []))
+ }
+ }]
+ else:
+ # Table alias match but not returning table matches - empty result
+ return []
+ if table_name and table_name.lower() == column_name.lower():
+ if return_table_matches:
+ return [{
+ "col": {
+ "type": "custom_field",
+ "alias": column_name,
+ # Don't freeze - goes directly to returned_fields
+ "used_fields": list(source.get("returned_fields", []))
+ }
+ }]
+ else:
+ # Table name match but not returning table matches - empty result
+ return []
+
+ # Direct column match handling
+ # The source_columns in single_column entries already point to the correct source.
+ # For custom_field (computed columns like "0 AS level"), we need to create a
+ # single_column reference pointing to the source (e.g., the CTE).
+ if source_column:
+ col_type = source_column.get("type")
+ if col_type == "single_column":
+ # For recursive CTE self-references (like h.id in recursive CTE),
+ # create a CTE reference. For non-recursive CTEs, pass through the
+ # matched column (which already has correct source_columns).
+ if table_ref and not return_table_matches:
+ source = self._find_source({"table": table_ref}, valid_sources)
+ # Only create CTE reference for RECURSIVE CTEs
+ if source and source.get("is_recursive_cte"):
+ names = source.get("names") or {}
+ table_info = {"table": names.get("table", table_ref)}
+ if names.get("table_alias"):
+ table_info["table_alias"] = names["table_alias"]
+ return [{
+ "col": {
+ "type": "single_column",
+ "column": column_name,
+ "alias": None,
+ "source_columns": [[{"type": "all_columns", "table": table_info}]]
+ }
+ }]
+ # For non-recursive CTEs and other cases: pass through matched column
+ result_col = dict(source_column)
+ # For unqualified reference with multiple scopes: use multi-level source_columns
+ if not table_ref and len(source_columns) > 1:
+ result_col["source_columns"] = source_columns
+ return [{"col": result_col}]
+ elif col_type == "custom_field":
+ # Custom field has no source_columns (it's a computed expression).
+ # For recursive CTE self-references (like h.level in recursive CTE):
+ # Create single_column with CTE reference
+ # For other CTE references (like m.month referencing non-recursive CTE):
+ # Pass through custom_field to preserve structure
+ #
+ # Detection: is_recursive_cte flag is set during CTE processing
+ if table_ref:
+ source = self._find_source({"table": table_ref}, valid_sources)
+ if source and source.get("is_recursive_cte"):
+ names = source.get("names") or {}
+ table_info = {"table": names.get("table", table_ref)}
+ if names.get("table_alias"):
+ table_info["table_alias"] = names["table_alias"]
+ source_ref = [[{"type": "all_columns", "table": table_info}]]
+ return [{
+ "col": {
+ "type": "single_column",
+ "column": column_name,
+ "alias": None,
+ "source_columns": source_ref
+ }
+ }]
+ # Return custom_field as-is to preserve structure
+ return [{"col": source_column}]
+ # For composite_field, etc.: return as-is to preserve structure
+ else:
+ return [{"col": source_column}]
+
+ # No direct match - return single_column with source_columns
+ errors = set()
+ if not source_columns or not any(source_columns):
+ # No potential sources found
+ if table_ref:
+ # Table qualifier doesn't exist - return error only, no col spec
+ # (Returning a col with empty source_columns would cause Clojure's
+ # resolve-field to generate a redundant missing-column error)
+ return [{"errors": {self._missing_table_alias_error(table_ref)}}]
+ else:
+ errors.add(self._missing_column_error(column_name))
+
+ return [{
+ "col": {
+ "type": "single_column",
+ "column": column_name,
+ "alias": None,
+ "source_columns": source_columns
+ },
+ "errors": errors
+ }]
+
+ def _find_source(self, search, sources):
+ """Find a source matching the search criteria."""
+ for scope_sources in sources:
+ if isinstance(scope_sources, list):
+ for source in scope_sources:
+ if source and self._table_matches(search, source.get("names", {})):
+ return source
+ return None
+
+ def _table_matches(self, search, table_names):
+ """Check if search criteria match table names."""
+ if not table_names:
+ return False
+ search_table = search.get("table")
+ search_schema = search.get("schema")
+ search_database = search.get("database")
+
+ # Match by alias first (if no schema/database in search)
+ if search_table and not search_schema and not search_database:
+ if table_names.get("table_alias") == search_table:
+ return True
+
+ # Match by table/schema/database
+ if search_table and search_table != table_names.get("table"):
+ return False
+ if search_schema and search_schema != table_names.get("schema"):
+ return False
+ if search_database and search_database != table_names.get("database"):
+ return False
+
+ return search_table == table_names.get("table")
+
+ def _source_might_have_column(self, source, column_name, table_ref):
+ """Check if a source might contain a column."""
+ if not source:
+ return False
+
+ # If table ref specified, check if it matches (case-insensitive since
+ # SQLGlot may lowercase identifiers)
+ if table_ref:
+ names = source.get("names", {})
+ if names:
+ table_alias = names.get("table_alias")
+ table_name = names.get("table")
+ if table_alias and table_alias.lower() == table_ref.lower():
+ return True
+ if table_name and table_name.lower() == table_ref.lower():
+ return True
+ return False
+
+ # No table ref - source might have column
+ return True
+
+ def _freeze_field(self, field):
+ """Convert a field dict to a hashable frozenset for use in sets."""
+ if field is None:
+ return None
+ if isinstance(field, (str, int, float, bool)):
+ return field
+ if isinstance(field, frozenset):
+ return field
+ if isinstance(field, set):
+ return frozenset(self._freeze_field(item) for item in field)
+ if isinstance(field, (list, tuple)):
+ return tuple(self._freeze_field(item) for item in field)
+ if isinstance(field, dict):
+ items = []
+ for k, v in field.items():
+ items.append((k, self._freeze_field(v)))
+ return frozenset(items)
+ return field
+
+ def _unfreeze_field(self, frozen):
+ """Convert a frozen field back to a dict."""
+ if frozen is None:
+ return None
+ if isinstance(frozen, (str, int, float, bool)):
+ return frozen
+
+ # frozenset of tuples -> dict
+ if isinstance(frozen, frozenset):
+ # Check if it looks like a dict (frozenset of 2-tuples)
+ try:
+ items = list(frozen)
+ if items and all(isinstance(item, tuple) and len(item) == 2 for item in items):
+ # It's a frozen dict
+ result = {}
+ for k, v in items:
+ result[k] = self._unfreeze_field(v)
+ return result
+ else:
+ # It's a frozen set of items
+ return [self._unfreeze_field(item) for item in frozen]
+ except:
+ return [self._unfreeze_field(item) for item in frozen]
+
+ # tuple -> list
+ if isinstance(frozen, tuple):
+ return [self._unfreeze_field(item) for item in frozen]
+
+ # list -> list (recursively unfreeze items)
+ if isinstance(frozen, list):
+ return [self._unfreeze_field(item) for item in frozen]
+
+ # dict -> dict (recursively unfreeze values)
+ if isinstance(frozen, dict):
+ return {k: self._unfreeze_field(v) for k, v in frozen.items()}
+
+ return frozen
+
+ def _syntax_error(self):
+ """Create a syntax error."""
+ return frozenset([("type", "syntax_error")])
+
+ def _missing_table_alias_error(self, table_name):
+ """Create a missing table alias error."""
+ return frozenset([("type", "missing_table_alias"), ("table", table_name)])
+
+ def _missing_column_error(self, column_name):
+ """Create a missing column error."""
+ return frozenset([("type", "missing_column"), ("column", column_name)])
diff --git a/src/metabase/actions/scope.clj b/src/metabase/actions/scope.clj
index b1f7afaa85b..a3132305f0f 100644
--- a/src/metabase/actions/scope.clj
+++ b/src/metabase/actions/scope.clj
@@ -1,6 +1,5 @@
(ns metabase.actions.scope
(:require
- [macaw.util :as u]
[metabase.actions.types :as types]
[metabase.lib.core :as lib]
[metabase.util.malli :as mu]
@@ -48,12 +47,17 @@
(:table-id scope)
(update :database-id #(or % (t2/select-one-fn :db_id [:model/Table :db_id] (:table-id scope)) missing-id))))
+(defn- strip-nils
+ "Remove any keys corresponding to nil values from the given map."
+ [m]
+ (into {} (filter (comp some? val) m)))
+
(mu/defn hydrate-scope :- ::types/scope.hydrated
"Add the implicit keys that can be derived from the existing ones in a scope. Idempotent."
[scope :- ::types/scope.raw]
;; Infer type if FE is not passing it (yet)
(let [scope-type (or (:type scope) (scope-type scope))]
- (assoc (u/strip-nils
+ (assoc (strip-nils
;; Rerun until it converges.
(ffirst (filter (partial apply =) (partition 2 1 (iterate hydrate-scope* scope)))))
:type scope-type)))
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index e8c547576bd..de28d960b96 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -500,7 +500,30 @@
:labels [:status]})
(prometheus/counter :metabase-token-check/attempt
{:description "Total number of token checks. Includes a status label."
- :labels [:status]})])
+ :labels [:status]})
+ ;; SQL parsing metrics
+ (prometheus/counter :metabase-sql-parsing/context-timeouts
+ {:description "Number of Python/GraalVM SQL parsing execution timeouts."})
+ (prometheus/counter :metabase-sql-parsing/context-acquisitions
+ {:description "Number of Python contexts acquired from the pool."})
+ (prometheus/counter :metabase-sql-parsing/context-creations
+ {:description "Number of new Python contexts created."})
+ (prometheus/counter :metabase-sql-parsing/context-disposals-expired
+ {:description "Number of Python contexts disposed due to TTL expiry."})
+ ;; SQL tools operation metrics
+ (prometheus/counter :metabase-sql-tools/operations-total
+ {:description "Total number of sql-tools operations started."
+ :labels [:parser :operation]})
+ (prometheus/counter :metabase-sql-tools/operations-completed
+ {:description "Number of sql-tools operations completed successfully."
+ :labels [:parser :operation]})
+ (prometheus/counter :metabase-sql-tools/operations-failed
+ {:description "Number of sql-tools operations that threw an exception."
+ :labels [:parser :operation]})
+ (prometheus/histogram :metabase-sql-tools/operation-duration-ms
+ {:description "Duration in milliseconds of sql-tools operations."
+ :labels [:parser :operation]
+ :buckets [1 5 10 25 50 100 250 500 1000 2500 5000 10000 30000]})])
(defn- quartz-collectors
[]
diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.clj
index 87b588c5bb0..3fc57750572 100644
--- a/src/metabase/core/init.clj
+++ b/src/metabase/core/init.clj
@@ -57,6 +57,7 @@
[metabase.session.init]
[metabase.settings.init]
[metabase.setup.init]
+ [metabase.sql-tools.init]
[metabase.sso.init]
[metabase.store-api.init]
[metabase.sync.init]
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index 990008aace7..66ec7de2e0c 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -3,9 +3,6 @@
(:refer-clojure :exclude [some])
(:require
[clojure.set :as set]
- ;; TODO (Cam 10/1/25) -- Isn't having drivers use Macaw directly against the spirt of all the work we did to make a
- ;; Driver API namespace?
- [macaw.core :as macaw]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse]
@@ -14,13 +11,11 @@
[metabase.driver.sql.parameters.substitute :as sql.params.substitute]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
- [metabase.driver.sql.references :as sql.references]
[metabase.driver.sql.util :as sql.u]
[metabase.driver.util :as driver.u]
- [metabase.util.humanization :as u.humanization]
+ [metabase.sql-tools.core :as sql-tools]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [some]]
[potemkin :as p]))
(comment sql.params.substitution/keep-me) ; this is so `cljr-clean-ns` and the linter don't remove the `:require`
@@ -226,184 +221,35 @@
[_]
"public")
-(defn find-table-or-transform
- "Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.
- It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."
- [driver tables transforms {search-table :table raw-schema :schema}]
- (let [search-schema (or raw-schema
- (default-schema driver))
- matches? (fn [db-table db-schema]
- (and (= search-table db-table)
- (= search-schema db-schema)))]
- (or (some (fn [{:keys [name schema id]}]
- (when (matches? name schema)
- {:table id}))
- tables)
- (some (fn [{:keys [id] {:keys [name schema]} :target}]
- (when (matches? name schema)
- {:transform id}))
- transforms))))
-
-(defn- normalize-table-spec
- [driver {:keys [table schema]}]
- {:table (sql.normalize/normalize-name driver table)
- :schema (some->> schema (sql.normalize/normalize-name driver))})
-
-(defn- parsed-table-refs
- "Parse a native query and return a sequence of normalized table specs {:table ... :schema ...}."
- [driver query]
- (-> query
- driver-api/raw-native-query
- (driver.u/parsed-query driver)
- (macaw/query->components {:strip-contexts? true})
- :tables
- (->> (map :component)
- (map #(normalize-table-spec driver %)))))
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Dependencies |
+;;; +----------------------------------------------------------------------------------------------------------------+
(mu/defmethod driver/native-query-table-refs :sql :- ::driver/native-query-table-refs
[driver :- :keyword
query :- :metabase.lib.schema/native-only-query]
- (into #{} (parsed-table-refs driver query)))
+ (into #{} (sql-tools/referenced-tables-raw driver (driver-api/raw-native-query query))))
(mu/defmethod driver/native-query-deps :sql :- ::driver/native-query-deps
[driver :- :keyword
query :- :metabase.lib.schema/native-only-query]
- (let [db-tables (driver-api/tables query)
- db-transforms (driver-api/transforms query)]
- (into (driver-api/native-query-table-references query)
- (keep #(find-table-or-transform driver db-tables db-transforms %)
- (parsed-table-refs driver query)))))
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | Dependencies |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
-(defmulti ^:private resolve-field
- "Resolves a field reference to one or more actual database fields.
-
- This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field
- references returned by sql.references/field-references.
-
- Note: this currently sets :lib/desired-column-alias but no other :lib/* fields, because the callers of this function
- don't need the other fields. If we care about other :lib/* fields in the future, we can add them then."
- {:added "0.57.0" :arglists '([driver metadata-provider col-spec])}
- (fn [_driver _metadata-provider col-spec]
- (:type col-spec)))
-
-(defmethod resolve-field :all-columns
- [driver metadata-provider col-spec]
- (or (some->> (:table col-spec)
- (find-table-or-transform driver (driver-api/tables metadata-provider) (driver-api/transforms metadata-provider))
- :table
- (driver-api/active-fields metadata-provider)
- (map #(-> (assoc % :lib/desired-column-alias (:name %))
- sql.references/wrap-col)))
- [{:error (driver-api/missing-table-alias-error
- (sql.references/table-name (:table col-spec)))}]))
-
-(defmethod resolve-field :single-column
- [driver metadata-provider {:keys [alias] :as col-spec}]
- [(if-let [{:keys [name] :as found}
- (->> (:source-columns col-spec)
- (some (fn [source-col-set]
- ;; in cases like `select (select blah from ...) from ...`, if blah refers to a
- ;; column in both the inner query and the outer query, the column from the inner
- ;; query will be preferred. However, if blah doesn't refer to something in the
- ;; inner query, it can also refer to something in the outer query.
- ;; sql.references/field-references organizes source-cols into a list of lists
- ;; to account for this.
- (->> (mapcat (fn [current-col]
- ;; :unknown-columns is a placeholder for "we know there are columns being
- ;; returned, but have no way of knowing what those are -- this is primarily
- ;; used for table-functions like `select * from my_func()`. If we encounter
- ;; something like that, assume that the query is valid and make up a matching
- ;; column to avoid false positives.
- (if (= (:type current-col) :unknown-columns)
- (let [name (:column col-spec)]
- [{:base-type :type/*
- :name name
- :display-name (->> name (u.humanization/name->human-readable-name :simple))
- :effective-type :type/*
- :semantic-type :Semantic/*}])
- (keep :col (resolve-field driver metadata-provider current-col))))
- source-col-set)
- (some #(when (= (:name %) (:column col-spec))
- %))))))]
- {:col (assoc found :lib/desired-column-alias (or alias name))}
- {:error (driver-api/missing-column-error (:column col-spec))})])
-
-(defn- get-name [m]
- (or (:alias m) (str (gensym "new-col"))))
-
-(defn- get-display-name [m]
- (->> (get-name m)
- (u.humanization/name->human-readable-name :simple)))
-
-(defmethod resolve-field :custom-field
- [_driver _metadata-provider col-spec]
- [{:col {:base-type :type/*
- :name (get-name col-spec)
- :lib/desired-column-alias (get-name col-spec)
- :display-name (get-display-name col-spec)
- :effective-type :type/*
- :semantic-type :Semantic/*}}])
-
-(defn- lca [default-type & types]
- (let [ancestor-sets (for [t types
- :when t]
- (conj (set (ancestors t)) t))
- common-ancestors (when (seq ancestor-sets)
- (apply set/intersection ancestor-sets))]
- (if (seq common-ancestors)
- (apply (partial max-key (comp count ancestors)) common-ancestors)
- default-type)))
-
-(defmethod resolve-field :composite-field
- [driver metadata-provider col-spec]
- (let [member-fields (mapcat #(->> (resolve-field driver metadata-provider %)
- (keep :col))
- (:member-fields col-spec))]
- [{:col {:name (get-name col-spec)
- :lib/desired-column-alias (get-name col-spec)
- :display-name (get-display-name col-spec)
- :base-type (apply lca :type/* (map :base-type member-fields))
- :effective-type (apply lca :type/* (map :effective-type member-fields))
- :semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}}]))
-
-(defmethod resolve-field :unknown-columns
- [_driver _metadata-provider _col-spec]
- [])
+ (into (driver-api/native-query-table-references query)
+ (sql-tools/referenced-tables driver query)))
(mu/defmethod driver/native-result-metadata :sql
[driver :- :keyword
native-query :- :metabase.lib.schema/native-only-query]
- (let [{:keys [returned-fields]} (-> native-query
- driver-api/raw-native-query
- (driver.u/parsed-query driver)
- macaw/->ast
- (->> (sql.references/field-references driver)))]
- (mapcat #(->> (resolve-field driver native-query %)
- (keep :col))
- returned-fields)))
+ (sql-tools/returned-columns driver native-query))
(mu/defmethod driver/validate-native-query-fields :sql :- [:set [:ref driver-api/schema.validate.error]]
[driver :- :keyword
native-query :- :metabase.lib.schema/native-only-query]
- (let [{:keys [used-fields returned-fields errors]} (-> native-query
- driver-api/raw-native-query
- (driver.u/parsed-query driver)
- macaw/->ast
- (->> (sql.references/field-references driver)))
- check-fields #(mapcat (fn [col-spec]
- (->> (resolve-field driver (driver-api/->metadata-provider native-query) col-spec)
- (keep :error)))
- %)]
- (-> errors
- (into (check-fields used-fields))
- (into (check-fields returned-fields)))))
+ (sql-tools/validate-query driver native-query))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Convenience Imports |
;;; +----------------------------------------------------------------------------------------------------------------+
-(p/import-vars [sql.params.substitution ->prepared-substitution PreparedStatementSubstitution])
+(p/import-vars
+ [sql.params.substitution ->prepared-substitution PreparedStatementSubstitution]
+ [sql.normalize normalize-name reserved-literal])
diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.clj
index 58f29f7de2e..d621710d5a9 100644
--- a/src/metabase/driver/util.clj
+++ b/src/metabase/driver/util.clj
@@ -5,7 +5,6 @@
[clojure.core.memoize :as memoize]
[clojure.set :as set]
[clojure.string :as str]
- [macaw.core :as macaw]
[metabase.app-db.core :as mdb]
[metabase.auth-provider.core :as auth-provider]
[metabase.config.core :as config]
@@ -799,80 +798,11 @@
;;; | Macaw parsing helpers |
;;; +----------------------------------------------------------------------------------------------------------------+
-(def ^:private considered-drivers
- "Since we are unable to ask basic questions of the driver hierarchy outside of that module, we need to explicitly
- mention all sub-types. This is probably not a bad thing."
- #{:h2 :mysql :postgres :redshift :sqlite :sqlserver})
-
;; At some point, we may want a way for 3rd party drivers to opt in, but a public API deserves some hammock time.
-;; Using a separate list to the above, as we may want this to be more restrictive.
(def trusted-for-table-permissions?
"Do we trust that Macaw will not give us false negatives for tables referenced by a given query?"
#{:h2 :mysql :postgres})
-(defn macaw-options
- "Generate the options expected by Macaw based on the nature of the given driver."
- [driver]
- (merge
- ;; If this isn't a driver we've considered, fallback to Macaw's conservative defaults.
- (when (contains? considered-drivers driver)
- {;; According to the SQL-92 specification, non-quoted identifiers should be case-insensitive, and the majority of
- ;; engines are implemented this way.
- ;;
- ;; In practice there are exceptions, notably MySQL and SQL Server, where case sensitivity is a property of the
- ;; underlying resource referenced by the identifier, and the case-sensitivity does not depend on whether the
- ;; reference is quoted.
- ;;
- ;; For MySQL the case sensitivity of databases and tables depends on both the underlying file system, and a system
- ;; variable used to initialize the database. For SQL Server it depends on the collation settings of the collection
- ;; where the corresponding schema element is defined.
- ;;
- ;; For MySQL, columns and aliases can never be case-sensitive, and for SQL Server the default collation is case-
- ;; insensitive too, so it makes sense to just treat all databases as case-insensitive as a whole.
- ;;
- ;; In future, Macaw may support discriminating on the identifier type, in which case we could be more precise for
- ;; these databases. Being 100% correct would require querying system variables and schema configuration however,
- ;; which is likely a step too far in complexity.
- ;;
- ;; Currently, we go with :agnostic, as it is the most relaxed semantics (the case of both the identifiers and the
- ;; underlying schema is totally ignored, and correspondence is non-deterministic), but Macaw supports more nuanced
- ;; :lower and :upper configuration values which coerce the query identifiers to a given case then do an exact
- ;; comparison with the schema.
- :case-insensitive :agnostic
- ;; For both MySQL and SQL Server, whether identifiers are case-sensitive depends on database configuration only,
- ;; and quoting has no effect on this, so we disable this option for consistency with `:case-insensitive`.
- :quotes-preserve-case? (not (contains? #{:mysql :sqlserver} driver))
- :features {:postgres-syntax (isa? driver/hierarchy driver :postgres)
- :square-bracket-quotes (= :sqlserver driver)
- :unsupported-statements false
- :backslash-escape-char true
- ;; This will slow things down, but until we measure the difference, opt for correctness.
- :complex-parsing true}
- ;; 10 seconds
- :timeout 10000})
- {;; There is no plan to be exhaustive yet.
- ;; Note that while an allowed list would be more conservative, at the time of writing only 1 of the bundled
- ;; drivers use FINAL as a reserved word, and mentioning them all would be prohibitive.
- ;; In the future, we will use multimethods to define this explicitly per driver, or even discover it automatically
- ;; through the JDBC connection, where possible.
- :non-reserved-words (vec (remove nil? [(when-not (contains? #{:clickhouse} driver)
- :final)]))}))
-
-(defn parsed-query
- "Wrapped for `parsed-query` providing default options and throwing exceptions on parsing failures."
- [sql driver & {:as opts}]
- (let [result
- #_{:clj-kondo/ignore [:discouraged-var]}
- (macaw/parsed-query sql (merge (macaw-options driver)
- opts))]
- ;; TODO (lbrdnk 2026-01-23): In follow-up work we should ensure that failure to parse is not silently swallowed.
- ;; I'm leaving that off at the moment to avoid potential log flooding.
- #_(when (and (map? result) (some? (:error result)))
- (throw (ex-info "SQL parsing failed."
- {:macaw-error (:error result)}
- (-> result :context :cause))))
- result))
-
(def ^:const transform-temp-table-prefix
"Prefix used for temporary tables created during transform execution."
"mb_transform_temp_table")
diff --git a/src/metabase/llm/context.clj b/src/metabase/llm/context.clj
index f4744eb3433..9ae2d9dc0b6 100644
--- a/src/metabase/llm/context.clj
+++ b/src/metabase/llm/context.clj
@@ -3,12 +3,12 @@
and fetches table metadata formatted as DDL for SQL generation."
(:require
[clojure.string :as str]
- [macaw.core :as macaw]
[metabase.api.common :as api]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.interface :as mi]
+ [metabase.sql-tools.core :as sql-tools]
[metabase.sync.core :as sync]
[metabase.util :as u]
[metabase.util.log :as log]
@@ -52,7 +52,7 @@
[:= [:lower :name] table-lower])))
(defn extract-tables-from-sql
- "Extract table IDs from a raw SQL string using Macaw parser.
+ "Extract table IDs from a raw SQL string.
Parses the SQL to identify referenced table names, then queries the
database to resolve those names to table IDs. When the SQL includes
@@ -64,8 +64,8 @@
[database-id sql-string]
(if (and database-id (seq sql-string))
(try
- (let [result (macaw/query->tables sql-string {:mode :compound-select})
- tables (:tables result)]
+ (let [driver (t2/select-one-fn :engine :model/Database :id database-id)
+ tables (sql-tools/referenced-tables-raw driver sql-string)]
(if (seq tables)
(let [match-clauses (mapv table-match-clause tables)
matched-tables (t2/select :model/Table
diff --git a/src/metabase/sql_parsing/common.clj b/src/metabase/sql_parsing/common.clj
new file mode 100644
index 00000000000..428f92cc419
--- /dev/null
+++ b/src/metabase/sql_parsing/common.clj
@@ -0,0 +1,8 @@
+(ns metabase.sql-parsing.common
+ (:require
+ [potemkin :as p]))
+
+(p/defprotocol+ PythonEval
+ "Protocol for evaluating Python code. Abstracts over raw Context and pooled contexts."
+ (eval-python [this code]
+ "Evaluate Python code."))
diff --git a/src/metabase/sql_parsing/core.clj b/src/metabase/sql_parsing/core.clj
new file mode 100644
index 00000000000..7fc6385146a
--- /dev/null
+++ b/src/metabase/sql_parsing/core.clj
@@ -0,0 +1,395 @@
+(ns metabase.sql-parsing.core
+ "Stateless SQL parsing via sqlglot (Python) and GraalVM Polyglot.
+
+ This module provides dialect-aware SQL parsing with no Metabase dependencies.
+ All functions take strings and return strings/simple data structures.
+
+ API:
+ (referenced-tables sql dialect) → [[catalog schema table] ...]
+ (referenced-fields dialect sql) → [[catalog schema table field] ...]
+ (returned-columns-lineage dialect sql schema schema-map) → [[col pure? deps] ...]
+ (validate-query dialect sql schema schema-map) → {:status :ok} | {:status :error ...}"
+ (:require
+ [clojure.string :as str]
+ [metabase.analytics.core :as analytics]
+ [metabase.sql-parsing.common :as common]
+ [metabase.sql-parsing.pool :as python.pool]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log])
+ (:import
+ (java.io Closeable)
+ (java.util.concurrent ExecutionException TimeoutException)
+ (org.graalvm.polyglot Value)))
+
+(set! *warn-on-reflection* true)
+
+;;; -------------------------------------------------- Timeout handling --------------------------------------------------
+
+(def ^:private ^:const default-timeout-ms
+ "Default timeout for Python operations in milliseconds.
+ GraalVM Python can occasionally hang (DEV-1393), so we wrap calls with a timeout."
+ 30000) ; 30 seconds
+
+(defn- with-timeout*
+ "Execute f in a future with timeout. On timeout, throws TimeoutException.
+ The caller is responsible for cleaning up resources (e.g., disposing context)."
+ [timeout-ms f]
+ (let [fut (future (f))]
+ (try
+ (deref fut timeout-ms ::timeout)
+ (catch ExecutionException e
+ ;; Unwrap execution exception to get the real cause
+ (throw (or (.getCause e) e)))
+ (finally
+ ;; If we timed out or got an exception, try to cancel the future
+ ;; Note: This won't actually interrupt GraalVM, but prevents resource leaks
+ (future-cancel fut)))))
+
+(defmacro ^:private with-python-timeout
+ "Execute body with a timeout. If timeout is reached:
+ 1. Interrupts the GraalVM context (actually stops execution)
+ 2. Poisons context so it gets disposed rather than returned to pool
+ 3. Logs warning and throws TimeoutException"
+ [ctx timeout-ms & body]
+ `(let [result# (with-timeout* ~timeout-ms (^:once fn* [] ~@body))]
+ (if (= result# ::timeout)
+ (do
+ ;; Actually interrupt the GraalVM context (1s grace period for soft interrupt)
+ ;; This is necessary because future-cancel doesn't stop GraalVM execution
+ (python.pool/interrupt! ~ctx 1000)
+ (python.pool/poison! ~ctx)
+ (analytics/inc! :metabase-sql-parsing/timeouts)
+ (log/warn "Python execution timed out after" ~timeout-ms "ms - GraalVM interrupted")
+ (throw (TimeoutException. (str "Python execution timed out after " ~timeout-ms "ms"))))
+ result#)))
+
+;;; -------------------------------------------------- Public API --------------------------------------------------
+
+(defn referenced-tables
+ "Extract table references from SQL.
+
+ Returns a vector of [catalog schema table] 3-tuples:
+ [[nil nil \"users\"] [nil \"public\" \"orders\"] [\"myproject\" \"analytics\" \"events\"]]
+
+ This is the pure parsing layer - it returns what's literally in the SQL.
+ Default schema resolution happens in the matching layer (core.clj)."
+ [dialect sql]
+ (-> (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ (-> ^Value (common/eval-python ctx "sql_tools.referenced_tables")
+ (.execute ^Value (object-array [sql dialect]))
+ .asString)))
+ json/decode
+ vec))
+
+(defn referenced-fields
+ "Extract field references from SQL, returning only fields from actual database tables.
+
+ Returns a vector of [catalog schema table field] 4-tuples:
+ [[nil nil \"users\" \"id\"] [nil \"public\" \"orders\" \"total\"]]
+
+ Includes:
+ - Wildcards as [catalog schema table \"*\"]
+ - All specific column references
+
+ Excludes:
+ - Fields from CTEs or subqueries
+ - Table aliases (returns actual table names)
+
+ Examples:
+ (referenced-fields \"postgres\" \"SELECT id FROM users\")
+ => [[nil nil \"users\" \"id\"]]
+
+ (referenced-fields \"postgres\" \"SELECT * FROM public.users\")
+ => [[nil \"public\" \"users\" \"*\"]]
+
+ (referenced-fields \"bigquery\" \"SELECT * FROM myproject.analytics.events\")
+ => [[\"myproject\" \"analytics\" \"events\" \"*\"]]"
+ [dialect sql]
+ (-> (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ (-> ^Value (common/eval-python ctx "sql_tools.referenced_fields")
+ (.execute ^Value (object-array [sql dialect]))
+ .asString)))
+ json/decode
+ vec))
+
+(defn returned-columns-lineage
+ "Extract column lineage from SQL query, showing which output columns depend on which source columns.
+
+ Returns a vector of [alias pure? [[schema table col]...]] tuples:
+ - alias: The output column name/alias
+ - pure?: Boolean - true if the column is a direct pass-through from a source column
+ - deps: Vector of [schema table column] dependencies
+
+ Requires a schema map of the form:
+ {\"schema_name\" {\"table_name\" {\"column_name\" \"TYPE\"}}}
+
+ Examples:
+ (returned-columns-lineage \"postgres\" \"SELECT id FROM users\" nil {nil {\"users\" {\"id\" \"INT\"}}})
+ => [[\"id\" true [[[nil \"users\" \"id\"]]]]]
+
+ (returned-columns-lineage \"postgres\" \"SELECT id + 1 as computed FROM users\" nil schema)
+ => [[\"computed\" false [[[nil \"users\" \"id\"]]]]]"
+ [dialect sql default-table-schema sqlglot-schema]
+ (-> (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues
+ (-> ^Value (common/eval-python ctx "sql_tools.returned_columns_lineage")
+ (.execute ^Value (object-array [dialect
+ sql
+ default-table-schema
+ (json/encode sqlglot-schema)]))
+ .asString)))
+ json/decode
+ vec))
+
+(defn validate-query
+ "Validate a SQL query against a schema using sqlglot's qualify optimizer.
+
+ Operates in two modes based on whether a schema is provided:
+
+ **Strict mode** (sqlglot-schema provided):
+ Validates that column and table references exist in the provided schema.
+ Returns errors for unknown tables, unresolved columns, missing table aliases.
+
+ **Permissive mode** (sqlglot-schema is nil or empty):
+ Only checks SQL syntax. Infers schema from query structure.
+ Useful for UDTFs and queries where the schema is unknown.
+
+ Parameters:
+ - dialect: SQLGlot dialect string (e.g., \"postgres\", \"mysql\"), or nil
+ - sql: The SQL query string to validate
+ - default-table-schema: Default schema name for unqualified table references
+ - sqlglot-schema: Schema map of {schema-name {table-name {column-name type}}},
+ or nil/empty for permissive mode
+
+ Returns a map with:
+ - If valid: {:status \"ok\"}
+ - If error: {:status \"error\", :type \"...\", :message \"...\", ...}
+
+ Error types (strict mode):
+ - \"unknown_table\": Table not found in schema
+ - \"column_not_resolved\": Column not found (includes :column key)
+ - \"invalid_expression\": Syntax/parse error
+ - \"unhandled\": Other errors"
+ [dialect sql default-table-schema & [sqlglot-schema]]
+ (-> (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues
+ (-> ^Value (common/eval-python ctx "sql_tools.validate_query")
+ (.execute ^Value (object-array [dialect sql default-table-schema (json/encode (or sqlglot-schema "{}"))]))
+ .asString)))
+ json/decode+kw))
+
+(defn simple-query?
+ "Check if SQL is a simple SELECT without LIMIT, OFFSET, or CTEs.
+
+ Used by Workspaces to determine if automatic checkpoints can be inserted.
+
+ Parameters:
+ - dialect: SQLGlot dialect string (e.g., \"postgres\", \"mysql\"), or nil for default
+ - sql: The SQL query string to check
+
+ Returns a map with:
+ - :is_simple - boolean indicating if query is simple
+ - :reason - string explaining why query is not simple (when false)
+
+ Examples:
+ (simple-query? \"postgres\" \"SELECT * FROM users\")
+ => {:is_simple true}
+
+ (simple-query? nil \"SELECT * FROM users LIMIT 10\")
+ => {:is_simple false :reason \"Contains a LIMIT\"}"
+ [dialect sql]
+ (-> (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ (-> ^Value (common/eval-python ctx "sql_tools.simple_query")
+ (.execute ^Value (object-array [sql dialect]))
+ .asString)))
+ json/decode+kw))
+
+(defn add-into-clause
+ "Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.
+
+ Transforms: 'SELECT * FROM products'
+ Into: 'SELECT * INTO \"TABLE\" FROM products'
+
+ Used by SQL Server compile-transform which requires SELECT INTO syntax
+ instead of CREATE TABLE AS SELECT.
+
+ Parameters:
+ - dialect: SQLGlot dialect string (e.g., \"tsql\" for SQL Server)
+ - sql: The SELECT SQL query string
+ - table-name: The target table name (already formatted/quoted)
+
+ Returns: Modified SQL string with INTO clause"
+ [dialect sql table-name]
+ (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ (-> ^Value (common/eval-python ctx "sql_tools.add_into_clause")
+ (.execute ^Value (object-array [sql table-name dialect]))
+ .asString))))
+
+(defn- convert-field-type
+ "Convert field type string (snake_case) to keyword (kebab-case)."
+ [type-str]
+ (-> type-str
+ (str/replace "_" "-")
+ keyword))
+
+(defn- convert-error
+ "Convert Python error format to Metabase lib error format."
+ [error]
+ (let [;; Get type from various possible key formats
+ err-type (or (:type error) (get error "type"))
+ ;; Get name/table/column from various possible formats
+ table-name (or (:table error) (get error "table"))
+ column-name (or (:column error) (get error "column") (:name error) (get error "name"))]
+ (cond
+ ;; Python frozenset came through as vector of pairs: [["type" "syntax_error"]]
+ (and (sequential? error) (sequential? (first error)))
+ (let [m (into {} error)
+ err-type (get m "type")]
+ (case err-type
+ "syntax_error" {:type :syntax-error}
+ "missing_column" {:type :missing-column :name (get m "column")}
+ "missing_table_alias" {:type :missing-table-alias :name (get m "table")}
+ {:type (keyword err-type)}))
+
+ ;; Handle string or keyword types
+ (or (string? err-type) (keyword? err-type))
+ (case (if (keyword? err-type) (name err-type) err-type)
+ ("syntax-error" "syntax_error") {:type :syntax-error}
+ ("missing-column" "missing_column") {:type :missing-column :name column-name}
+ ("missing-table-alias" "missing_table_alias") {:type :missing-table-alias :name table-name}
+ {:type (keyword err-type)})
+
+ :else error)))
+
+(defn- convert-field
+ "Convert a field spec from Python format to Clojure format."
+ [field]
+ (when field
+ (let [field-type (some-> (or (:type field) (get field "type")) convert-field-type)]
+ (case field-type
+ :single-column
+ {:type :single-column
+ :column (or (:column field) (get field "column"))
+ :alias (or (:alias field) (get field "alias"))
+ :source-columns (mapv (fn [scope]
+ (mapv convert-field scope))
+ (or (:source-columns field)
+ (:source_columns field)
+ (get field "source_columns")
+ []))}
+
+ :all-columns
+ {:type :all-columns
+ :table (let [t (or (:table field) (get field "table"))]
+ (cond-> {}
+ (or (:table t) (get t "table"))
+ (assoc :table (or (:table t) (get t "table")))
+ (or (:schema t) (get t "schema"))
+ (assoc :schema (or (:schema t) (get t "schema")))
+ (or (:database t) (get t "database"))
+ (assoc :database (or (:database t) (get t "database")))
+ (or (:table-alias t) (:table_alias t) (get t "table_alias"))
+ (assoc :table-alias (or (:table-alias t) (:table_alias t) (get t "table_alias")))))}
+
+ :custom-field
+ {:type :custom-field
+ :alias (or (:alias field) (get field "alias"))
+ :used-fields (set (map convert-field
+ (or (:used-fields field)
+ (:used_fields field)
+ (get field "used_fields")
+ [])))}
+
+ :composite-field
+ {:type :composite-field
+ :alias (or (:alias field) (get field "alias"))
+ :member-fields (mapv convert-field
+ (or (:member-fields field)
+ (:member_fields field)
+ (get field "member_fields")
+ []))}
+
+ :unknown-columns
+ {:type :unknown-columns}
+
+ ;; Fallback - return as-is with type conversion
+ (assoc field :type field-type)))))
+
+(defn field-references
+ "Extract field references from SQL, returning used and returned fields.
+
+ This is the SQLGlot equivalent of Macaw's field-references function.
+ Returns a map with:
+ - :used-fields - set of field specs from WHERE, JOIN ON, GROUP BY, ORDER BY
+ - :returned-fields - vector of field specs from SELECT clause (ordered)
+ - :errors - set of validation errors
+
+ Each field spec has:
+ - :type - :single-column, :all-columns, :custom-field, :composite-field, or :unknown-columns
+ - :column - column name (for single-column)
+ - :alias - column alias (nil if none)
+ - :source-columns - nested list of possible source columns
+ - :table - table info (for all-columns)
+ - :used-fields - set of fields used (for custom-field)
+ - :member-fields - list of fields (for composite-field)
+
+ On timeout, returns an error map instead of throwing, consistent with the
+ 'fail soft' pattern used for parsing failures."
+ [dialect sql]
+ (try
+ (let [raw (-> (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ (-> ^Value (common/eval-python ctx "sql_tools.field_references")
+ (.execute ^Value (object-array [sql dialect]))
+ .asString)))
+ json/decode+kw)
+ used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])
+ returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])
+ errors (or (:errors raw) (get raw "errors") [])]
+ {:used-fields (set (map convert-field used-fields))
+ :returned-fields (vec (map convert-field returned-fields))
+ :errors (set (map convert-error errors))})
+ (catch TimeoutException e
+ {:used-fields #{}
+ :returned-fields []
+ :errors #{{:type :timeout :message (.getMessage e)}}})))
+
+(defn replace-names
+ "Replace schema, table, and column names in SQL.
+
+ Parameters:
+ - dialect: SQLGlot dialect string (e.g., \"postgres\", \"mysql\"), or nil
+ - sql: The SQL query string
+ - replacements: A map with optional keys:
+ - :schemas - map of old-schema-name -> new-schema-name
+ - :tables - seq of [[{:schema s :table t} new-name] ...]
+ - :columns - seq of [[{:schema s :table t :column c} new-name] ...]
+
+ Returns modified SQL string.
+
+ SECURITY: Replacement values are injected into the SQL AST as identifier names
+ without sanitization. Callers MUST ensure replacement values are system-generated.
+ See sql_tools.py replace_names for details.
+
+ Examples:
+ (replace-names \"postgres\" \"SELECT * FROM people\" {:tables [[{:table \"people\"} \"users\"]]})
+ => \"SELECT * FROM users\""
+ [dialect sql replacements]
+ (with-open [^Closeable ctx (python.pool/python-context)]
+ (with-python-timeout ctx default-timeout-ms
+ (-> ^Value (common/eval-python ctx "sql_tools.replace_names")
+ (.execute ^Value (object-array [sql (json/encode replacements) dialect]))
+ .asString))))
+
+(comment
+ (referenced-tables "postgres" "select * from transactions")
+
+ (validate-sql-query "postgres" "SELECT * FROM users")
+
+ (referenced-fields "postgres" "SELECT id, name FROM users WHERE active = true"))
diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.clj
new file mode 100644
index 00000000000..58723e15e2c
--- /dev/null
+++ b/src/metabase/sql_parsing/pool.clj
@@ -0,0 +1,316 @@
+(ns metabase.sql-parsing.pool
+ "Namespace to handle python environment. Has only a single public function, [[python-context]] which will return a
+ python context. This is an internal detail to the sql-parsing module. If you want something from this, add a
+ function to the core namespace that uses this. No one should know about this implmentation detail."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.java.shell :as shell]
+ [clojure.string :as str]
+ [metabase.analytics.core :as analytics]
+ [metabase.sql-parsing.common :as common]
+ [metabase.util.files :as u.files]
+ [metabase.util.log :as log]
+ [potemkin :as p])
+ (:import
+ (io.aleph.dirigiste
+ IPool$Controller
+ IPool$Generator
+ Pool
+ Pools)
+ (java.io Closeable File)
+ (java.nio.file Files Path)
+ (java.time Duration)
+ (java.util.concurrent TimeUnit TimeoutException)
+ (org.graalvm.polyglot Context HostAccess)))
+
+(set! *warn-on-reflection* true)
+
+;;; -------------------------------------------------- Python path resolution --------------------------------------------------
+
+(def ^:private python-sources-resource
+ "Resource path for Python sources (sql_tools.py and sqlglot)."
+ "python-sources")
+
+(defn- plugins-dir-path
+ "Get path to plugins directory. Lightweight alternative to metabase.plugins.core/plugins-dir
+ that avoids loading the entire plugin system (~8 seconds of namespace loading)."
+ ^Path []
+ (let [dir-name (or (System/getenv "MB_PLUGINS_DIR")
+ "plugins")
+ path (u.files/get-path dir-name)]
+ (u.files/create-dir-if-not-exists! path)
+ path))
+
+(defn- jar-resource?
+ "True if the given resource is inside a JAR (production), false if from filesystem (dev)."
+ [resource]
+ (boolean
+ (when-let [url (io/resource resource)]
+ (.contains (.getFile ^java.net.URL url) ".jar!/"))))
+
+(defn- copy-dir-recursive!
+ "Recursively copy all files from source Path to dest Path.
+ Works across filesystems (e.g., from JAR filesystem to default filesystem)."
+ [^Path source ^Path dest]
+ (doseq [^Path child (u.files/files-seq source)]
+ (let [relative-name (str (.getFileName child))
+ target (.resolve dest relative-name)]
+ (if (Files/isDirectory child (into-array java.nio.file.LinkOption []))
+ (do
+ (u.files/create-dir-if-not-exists! target)
+ (copy-dir-recursive! child target))
+ (log/with-no-logs ;; <-- Suppress per-file logs from copy-file! (hundreds of files in sqlglot)
+ (u.files/copy-file! child target))))))
+
+(defn- extract-python-sources!
+ "Extract python-sources from JAR to plugins directory. Returns the path as a string.
+ Uses the same plugins directory as driver modules for consistency.
+ Skips extraction if version file matches."
+ ^String []
+ (let [^Path plugins-path (plugins-dir-path)
+ dest-path (.resolve plugins-path "python-sources")
+ version-file (.resolve dest-path ".sqlglot-version")
+ jar-version (some-> (io/resource "python-sources/.sqlglot-version") slurp str/trim)
+ dest-version (when (u.files/exists? version-file)
+ (str/trim (slurp (.toFile version-file))))]
+ (if (and jar-version (= jar-version dest-version))
+ (log/info "Python sources already extracted (version" jar-version ")")
+ (do
+ (u.files/create-dir-if-not-exists! dest-path)
+ (log/info "Extracting Python sources to" (str dest-path))
+ (u.files/with-open-path-to-resource [source-path python-sources-resource]
+ (copy-dir-recursive! source-path dest-path))
+ (log/info "Python sources extracted")))
+ (str dest-path)))
+
+;;; -------------------------------------------------- Dev lazy installation --------------------------------------------------
+
+(def ^:private dev-python-sources-dir
+ "Directory where sqlglot is installed in dev mode."
+ "resources/python-sources")
+
+(defn- delete-recursive!
+ "Recursively delete a directory and all its contents."
+ [^File f]
+ (when (.isDirectory f)
+ (doseq [child (.listFiles f)]
+ (delete-recursive! child)))
+ (.delete f))
+
+(defn- expected-sqlglot-version
+ "Read the expected sqlglot version from .sqlglot-version resource file."
+ []
+ (some->> (io/resource "python-sources/.sqlglot-version")
+ slurp
+ ;; handle # comments in the version file
+ str/split-lines
+ (remove #(str/starts-with? % "#"))
+ (str/join "\n")
+ str/trim))
+
+(defn- package-installer-available?
+ "Check if uv or pip is available for installing Python packages."
+ []
+ (or (zero? (:exit (shell/sh "which" "uv")))
+ (zero? (:exit (shell/sh "which" "pip")))))
+
+(defn- version-installed?
+ "Check if sqlglot is installed with the expected version by looking for the dist-info directory."
+ [target-dir expected-version]
+ (let [dist-info (io/file target-dir (str "sqlglot-" expected-version ".dist-info"))]
+ (.exists dist-info)))
+
+(defn- install-sqlglot!
+ "Install sqlglot via uv (preferred) or pip (fallback).
+ Deletes any existing sqlglot directories first to handle version upgrades."
+ [target-dir version]
+ ;; Delete old versions first (sqlglot/ and any sqlglot-*.dist-info/)
+ (doseq [^File f (.listFiles (io/file target-dir))]
+ (when (or (= "sqlglot" (.getName f))
+ (.startsWith (.getName f) "sqlglot-"))
+ (log/info "Removing old sqlglot:" (.getName f))
+ (delete-recursive! f)))
+ ;; Try uv first (fast), fall back to pip
+ (let [pkg (str "sqlglot==" version)
+ uv-result (shell/sh "uv" "pip" "install" pkg "--target" target-dir "--no-compile")]
+ (if (zero? (:exit uv-result))
+ (log/info "sqlglot" version "installed via uv")
+ (do
+ (log/info "uv not available, trying pip...")
+ (let [pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile")]
+ (when-not (zero? (:exit pip-result))
+ (throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"
+ "Manual install: uv pip install " pkg " --target " target-dir "\n"
+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")
+ {:uv-error (:err uv-result)
+ :pip-error (:err pip-result)
+ :version version
+ :target-dir target-dir})))
+ (log/info "sqlglot" version "installed via pip"))))))
+
+(defn- ensure-sqlglot-installed!
+ "Ensure sqlglot is installed with correct version. Called lazily on first use in dev.
+ If version mismatch or missing, automatically installs the correct version."
+ []
+ (let [expected-ver (expected-sqlglot-version)]
+ (when-not expected-ver
+ (throw (ex-info "Missing .sqlglot-version file in resources/python-sources/"
+ {:resource "python-sources/.sqlglot-version"})))
+ (when-not (version-installed? dev-python-sources-dir expected-ver)
+ (if (package-installer-available?)
+ (do
+ (log/info "Installing sqlglot" expected-ver "(first use in dev)...")
+ (install-sqlglot! dev-python-sources-dir expected-ver))
+ (log/warn "sqlglot not installed and no package installer (uv or pip) found."
+ "SQL parsing features will fail until sqlglot is installed."
+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")))))
+
+;;; -------------------------------------------------- Python path delay --------------------------------------------------
+
+(defonce ^:private
+ ^{:doc "Path to Python sources directory. In dev, this is resources/python-sources.
+ When running from a JAR, we extract to the plugins directory.
+ In dev, lazily installs sqlglot if not present or version mismatched."}
+ python-path
+ (delay
+ (if (jar-resource? python-sources-resource)
+ (extract-python-sources!)
+ (do
+ (ensure-sqlglot-installed!)
+ dev-python-sources-dir))))
+
+;;; -------------------------------------------- Context Wrappers ----------------------------------------------------
+
+(p/defrecord+ PooledContext [^Context context ^Pool pool tuple poisoned?]
+ common/PythonEval
+ (eval-python [_this code]
+ (.eval context "python" ^String code))
+
+ Closeable
+ (close [_this]
+ (if @poisoned?
+ (do
+ (log/warn "Disposing poisoned Python context (likely hung GraalVM)")
+ (.dispose pool :python tuple))
+ (.release pool :python tuple))))
+
+(defn poison!
+ "Mark a PooledContext as poisoned. When closed, it will be disposed from the pool
+ rather than released back. Use this when GraalVM hangs to prevent returning a
+ broken context to the pool."
+ [ctx]
+ (when (instance? PooledContext ctx)
+ (reset! (:poisoned? ctx) true)))
+
+(defn interrupt!
+ "Interrupt Python execution in this context. Returns true if interrupted successfully.
+ If interrupt times out (guest in uninterruptible code), forces context closure with close(true).
+ This is necessary because future-cancel doesn't actually stop GraalVM execution."
+ [ctx ^long timeout-ms]
+ (when-let [^Context raw-ctx (when (instance? PooledContext ctx)
+ (:context ctx))]
+ (try
+ (.interrupt raw-ctx (Duration/ofMillis timeout-ms))
+ true
+ (catch TimeoutException _
+ ;; Interrupt timed out - guest app may be in uninterruptible native code
+ ;; Force close cannot be denied by guest application
+ (log/warn "GraalVM interrupt timed out, forcing context closure")
+ (.close raw-ctx true)
+ false))))
+
+(defn- create-graalvm-context
+ "Create a new GraalVM Python context configured for sqlglot.
+
+ The context is configured with:
+ - PythonPath pointing to python-sources (contains sql_tools.py and sqlglot)
+ - Full host access (needed for JSON serialization)
+ - IO access enabled (for Python imports)"
+ ^Context []
+ (let [ctx (.. (Context/newBuilder (into-array String ["python"]))
+ (option "engine.WarnInterpreterOnly" "false")
+ ;; python-sources contains both sql_tools.py shim and installed sqlglot
+ (option "python.PythonPath" @python-path)
+ (allowHostAccess HostAccess/ALL)
+ (allowIO true)
+ (build))]
+ (.eval ctx "python" "import sql_tools")
+ ctx))
+
+(defn- make-python-context-pool
+ "Create a pool of Python contexts. Accepts a generator function and optional config map.
+
+ Config options:
+ - :max-size - Maximum pool size (default: 3)
+ - :min-size - Minimum pool size (default: 1)
+ - :ttl-minutes - How long contexts live before expiry (default: 10)
+ - :utilization - Target utilization (default: 1.0 = 100%)
+
+ This allows for easy testing by injecting mock context generators and custom pool configs."
+ ([context-generator]
+ (make-python-context-pool context-generator {}))
+ ([context-generator {:keys [max-size min-size ttl-minutes utilization]
+ :or {max-size 3
+ min-size 1
+ ttl-minutes 10
+ utilization 1.0}}]
+ (let [base-controller (Pools/utilizationController utilization max-size max-size)]
+ (Pool. (reify IPool$Generator
+ (generate [_ _]
+ ;; Generate a tuple of the context and the expiry timestamp.
+ (analytics/inc! :metabase-sql-parsing/context-creations)
+ [(context-generator)
+ (+ (System/nanoTime) (.toNanos TimeUnit/MINUTES ttl-minutes))])
+ (destroy [_ _ [^Context ctx _expiry]]
+ ;; Close context when disposed from pool (expiry, poison, or shutdown)
+ (try
+ (.close ctx true) ;; Force close - can't wait for running code
+ (catch Exception _))))
+ ;; Wrap the utilization controller with a modification that doesn't allow the pool to go below min-size.
+ (reify IPool$Controller
+ (shouldIncrement [_ k a b] (.shouldIncrement base-controller k a b))
+ (adjustment [_ stats]
+ (let [adj (.adjustment base-controller stats)
+ ;; :python is arbitrary key, it just has to be consistent everywhere when working with the pool.
+ n (some-> ^io.aleph.dirigiste.Stats (:python stats) .getNumWorkers)
+ python-adj (:python adj)]
+ (if (and n python-adj (<= (+ n python-adj) min-size))
+ ;; If the adjustment is going to bring the pool below min-size, return empty adjustment instead.
+ {}
+ adj))))
+ 65000 ;; Queue size - doesn't matter much.
+ 25 ;; Sampling interval - doesn't matter much.
+ 10000 ;; Recheck every 10 seconds
+ TimeUnit/MILLISECONDS))))
+
+(def ^:private python-context-pool
+ "Pool of Python context objects. They are not thread-safe, so access to them has to be carefully managed between
+ threads. Each context with loaded Python interpreter takes significant memory, so we don't want too many of them.
+ However, one takes significant time to initialize, so we don't want to load them anew each time in prod. Under some
+ circumstances, the GraalVM Python context tends to leak memory, so we don't want to keep the reference to the
+ context forever. Considering all that, this pool targets 100% utilization (so, if the utilization is lower, the pool
+ will start dropping objects) and the maximum of 3 objects (to prevent OOMs), but at least 1 object will always be in
+ the pool to pick up. However, together with each context keep its creation timestamp so that we can throwaway
+ instances that are too old to avoid leaks.
+
+ Wrapped in delay to avoid initialization during AOT compilation."
+ (delay (make-python-context-pool create-graalvm-context {:max-size 3
+ :min-size 1
+ :ttl-minutes 10})))
+
+(defn- acquire-context
+ "Acquire a context from the pool, handling expiry. Returns a PooledContext."
+ [^Pool pool]
+ (loop []
+ (let [[context expiry-ts :as tuple] (.acquire pool :python)]
+ (if (>= (System/nanoTime) expiry-ts)
+ (do (analytics/inc! :metabase-sql-parsing/context-disposals-expired)
+ (.dispose pool :python tuple)
+ (recur))
+ (do (analytics/inc! :metabase-sql-parsing/context-acquisitions)
+ (->PooledContext context pool tuple (atom false)))))))
+
+(defn python-context
+ "Acquire a python context from the pool. Must be closed. Use in a `with-open` context."
+ []
+ (acquire-context @python-context-pool))
diff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.clj
new file mode 100644
index 00000000000..541231e50de
--- /dev/null
+++ b/src/metabase/sql_tools/common.clj
@@ -0,0 +1,189 @@
+(ns metabase.sql-tools.common
+ {:clj-kondo/config '{:linters {:metabase/modules {:level :off}}}}
+ (:require
+ [clojure.set :as set]
+ [clojure.string :as str]
+ [metabase.driver.sql :as driver.sql]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.sql-tools.interface :as sql-tools]
+ [metabase.util.humanization :as u.humanization]
+ [metabase.util.malli :as mu]))
+
+(defn normalize-table-spec
+ "Return tables spec with normalized values."
+ [driver {:keys [table schema]}]
+ {:table (driver.sql/normalize-name driver table)
+ :schema (some->> schema (driver.sql/normalize-name driver))})
+
+(defn find-table-or-transform
+ "Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.
+ It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."
+ [driver tables transforms {search-table :table raw-schema :schema}]
+ (let [search-schema (or raw-schema
+ (driver.sql/default-schema driver))
+ normalize (partial driver.sql/normalize-name driver)
+ matches? (fn [db-table db-schema]
+ (and (= (normalize search-table) (normalize db-table))
+ (= (some-> search-schema normalize) (some-> db-schema normalize))))]
+ (or (some (fn [{:keys [name schema id]}]
+ (when (matches? name schema)
+ {:table id}))
+ tables)
+ (some (fn [{:keys [id] {:keys [name schema]} :target}]
+ (when (matches? name schema)
+ {:transform id}))
+ transforms))))
+
+(mu/defn table-name :- [:maybe :string]
+ "Computes a table name from a table reference"
+ [raw-col :- [:map
+ [:database {:optional true} :string]
+ [:schema {:optional true} :string]
+ [:table {:optional true} :string]]]
+ (when (:table raw-col)
+ (->> [:database :schema :table]
+ (keep raw-col)
+ (str/join "."))))
+
+(defn wrap-col
+ "Wraps the argument in `{:col _}`"
+ [col]
+ {:col col})
+
+(defmulti resolve-field
+ "Resolves a field reference to one or more actual database fields.
+
+ This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field
+ references returned by sql-tools.macaw.references/field-references.
+
+ Note: this currently sets :lib/desired-column-alias but no other :lib/* fields, because the callers of this function
+ don't need the other fields. If we care about other :lib/* fields in the future, we can add them then."
+ {:added "0.57.0" :arglists '([driver metadata-provider col-spec])}
+ (fn [_driver _metadata-provider col-spec]
+ (:type col-spec)))
+
+(defmethod resolve-field :all-columns
+ [driver metadata-provider col-spec]
+ (or (some->> (:table col-spec)
+ (find-table-or-transform
+ driver (lib.metadata/tables metadata-provider) (lib.metadata/transforms metadata-provider))
+ :table
+ (lib.metadata/active-fields metadata-provider)
+ (map #(-> (assoc % :lib/desired-column-alias (:name %))
+ wrap-col)))
+ [{:error (lib/missing-table-alias-error
+ (table-name (:table col-spec)))}]))
+
+(defmethod resolve-field :single-column
+ [driver metadata-provider {:keys [alias] :as col-spec}]
+ [(if-let [{:keys [name] :as found}
+ (->> (:source-columns col-spec)
+ (some (fn [source-col-set]
+ ;; in cases like `select (select blah from ...) from ...`, if blah refers to a
+ ;; column in both the inner query and the outer query, the column from the inner
+ ;; query will be preferred. However, if blah doesn't refer to something in the
+ ;; inner query, it can also refer to something in the outer query.
+ ;; sql-tools.macaw.references/field-references organizes source-cols into a list of lists
+ ;; to account for this.
+ (->> (mapcat (fn [current-col]
+ ;; :unknown-columns is a placeholder for "we know there are columns being
+ ;; returned, but have no way of knowing what those are -- this is primarily
+ ;; used for table-functions like `select * from my_func()`. If we encounter
+ ;; something like that, assume that the query is valid and make up a matching
+ ;; column to avoid false positives.
+ (if (= (:type current-col) :unknown-columns)
+ (let [name (:column col-spec)]
+ [{:base-type :type/*
+ :name name
+ :display-name (->> name (u.humanization/name->human-readable-name :simple))
+ :effective-type :type/*
+ :semantic-type :Semantic/*}])
+ (keep :col (resolve-field driver metadata-provider current-col))))
+ source-col-set)
+ (some #(when (= (driver.sql/normalize-name driver (:name %))
+ (driver.sql/normalize-name driver (:column col-spec)))
+ %))))))]
+ {:col (assoc found :lib/desired-column-alias (or alias name))}
+ {:error (lib/missing-column-error (:column col-spec))})])
+
+(defn- get-name [m]
+ (or (:alias m) (str (gensym "new-col"))))
+
+(defn- get-display-name [m]
+ (->> (get-name m)
+ (u.humanization/name->human-readable-name :simple)))
+
+(defmethod resolve-field :custom-field
+ [_driver _metadata-provider col-spec]
+ [{:col {:base-type :type/*
+ :name (get-name col-spec)
+ :lib/desired-column-alias (get-name col-spec)
+ :display-name (get-display-name col-spec)
+ :effective-type :type/*
+ :semantic-type :Semantic/*}}])
+
+(defn- lca [default-type & types]
+ (let [ancestor-sets (for [t types
+ :when t]
+ (conj (set (ancestors t)) t))
+ common-ancestors (when (seq ancestor-sets)
+ (apply set/intersection ancestor-sets))]
+ (if (seq common-ancestors)
+ (apply (partial max-key (comp count ancestors)) common-ancestors)
+ default-type)))
+
+(defmethod resolve-field :composite-field
+ [driver metadata-provider col-spec]
+ (let [member-fields (mapcat #(->> (resolve-field driver metadata-provider %)
+ (keep :col))
+ (:member-fields col-spec))]
+ [{:col {:name (get-name col-spec)
+ :lib/desired-column-alias (get-name col-spec)
+ :display-name (get-display-name col-spec)
+ :base-type (apply lca :type/* (map :base-type member-fields))
+ :effective-type (apply lca :type/* (map :effective-type member-fields))
+ :semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}}]))
+
+(defmethod resolve-field :unknown-columns
+ [_driver _metadata-provider _col-spec]
+ [])
+
+(defn returned-columns
+ "Given a native query, return columns it produces using field-references pipeline."
+ [parser driver native-query]
+ (let [{:keys [returned-fields]} (sql-tools/field-references-impl parser driver (lib/raw-native-query native-query))]
+ (vec (mapcat #(->> (resolve-field driver native-query %)
+ (keep :col))
+ returned-fields))))
+
+(defn- normalize-error
+ "Normalize error names using driver-specific case normalization.
+ This ensures error names match database metadata conventions."
+ [driver error]
+ (if-let [error-name (:name error)]
+ (assoc error :name (driver.sql/normalize-name driver error-name))
+ error))
+
+(defn validate-query
+ "Validate native query."
+ [parser driver native-query]
+ (let [{:keys [used-fields returned-fields errors]} (sql-tools/field-references-impl parser driver (lib/raw-native-query native-query))
+ check-fields #(mapcat (fn [col-spec]
+ (->> (resolve-field driver (lib/->metadata-provider native-query) col-spec)
+ (keep :error)))
+ %)]
+ (->> (-> errors
+ (into (check-fields used-fields))
+ (into (check-fields returned-fields)))
+ (map (partial normalize-error driver))
+ set)))
+
+(defn referenced-fields
+ "Extract fields referenced (used) in a native query - fields in WHERE, JOIN ON, etc."
+ [parser driver native-query]
+ (let [{:keys [used-fields]} (sql-tools/field-references-impl parser driver (lib/raw-native-query native-query))]
+ (into #{}
+ (mapcat #(->> (resolve-field driver native-query %)
+ (keep :col)))
+ used-fields)))
diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.clj
new file mode 100644
index 00000000000..d37fdf60832
--- /dev/null
+++ b/src/metabase/sql_tools/core.clj
@@ -0,0 +1,184 @@
+(ns metabase.sql-tools.core
+ "Public API for sql-tools SQL parsing functionality.
+
+ This namespace provides parser-agnostic functions that delegate to the
+ configured backend (Macaw or SQLGlot) via multimethods.
+
+ Note: Implementation namespaces (macaw.core, sqlglot.core) are loaded via
+ metabase.sql-tools.init to avoid circular dependencies."
+ (:require
+ [metabase.sql-tools.interface :as interface]
+ [metabase.sql-tools.metrics :as metrics]
+ [metabase.sql-tools.settings :as sql-tools.settings]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [potemkin :as p]))
+
+;;; -------------------------------------------------- Schemas --------------------------------------------------
+
+(mr/def ::table-spec
+ "Schema for table identifier used in replacements."
+ [:map
+ [:table :string]
+ [:schema {:optional true} [:maybe :string]]])
+
+(mr/def ::column-spec
+ "Schema for column identifier used in replacements."
+ [:map
+ [:column :string]
+ [:table {:optional true} [:maybe :string]]
+ [:schema {:optional true} [:maybe :string]]])
+
+(mr/def ::table-replacement-value
+ "Schema for table replacement target - can be a string or a map with schema/table."
+ [:or
+ :string
+ [:map
+ [:schema {:optional true} [:maybe :string]]
+ [:table {:optional true} [:maybe :string]]]])
+
+(mr/def ::replacements
+ "Schema for replace-names replacements map."
+ [:map
+ [:schemas {:optional true} [:map-of :string :string]]
+ [:tables {:optional true} [:map-of ::table-spec ::table-replacement-value]]
+ [:columns {:optional true} [:map-of ::column-spec :string]]])
+
+(mr/def ::replace-names-opts
+ "Schema for replace-names options."
+ [:map
+ [:allow-unused? {:optional true} :boolean]])
+
+(mr/def ::simple-query-result
+ "Schema for simple-query? return value."
+ [:map
+ [:is_simple :boolean]
+ [:reason {:optional true} :string]])
+
+;; Re-export multimethod vars so implementations can use sql-tools/foo-impl
+(p/import-vars
+ [interface
+ returned-columns-impl
+ referenced-tables-impl
+ referenced-fields-impl
+ field-references-impl
+ validate-query-impl
+ replace-names-impl
+ referenced-tables-raw-impl
+ simple-query?-impl
+ add-into-clause-impl])
+
+(defn parser-backend
+ "Get current value of sql-tools-parser-backend."
+ []
+ (sql-tools.settings/current-parser-backend))
+
+(mu/defn returned-columns :- [:sequential :map]
+ "Return appdb columns for the `native-query`."
+ [driver :- :keyword
+ native-query]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "returned-columns"]
+ (interface/returned-columns-impl parser driver native-query))))
+
+(mu/defn referenced-tables :- [:set :map]
+ "Return tables referenced by the `native-query`"
+ [driver :- :keyword
+ native-query]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "referenced-tables"]
+ (interface/referenced-tables-impl parser driver native-query))))
+
+(mu/defn referenced-fields :- [:set :map]
+ "Return appdb fields referenced (used) by the `native-query`.
+
+ This includes fields in SELECT, WHERE, JOIN ON, GROUP BY, ORDER BY, etc.
+ Returns a set of :metadata/column maps."
+ [driver :- :keyword
+ native-query]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "referenced-fields"]
+ (interface/referenced-fields-impl parser driver native-query))))
+
+(mu/defn field-references :- :metabase.sql-tools.macaw.references/field-references
+ "Return field references for SQL string.
+
+ Returns a map with:
+ - :used-fields - set of field specs
+ - :returned-fields - vector of field specs
+ - :errors - set of validation errors"
+ [driver :- :keyword
+ sql-string :- :string]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "field-references"]
+ (interface/field-references-impl parser driver sql-string))))
+
+(mu/defn validate-query :- [:set :map]
+ "Validate native query. Returns a set of validation errors (empty set if valid)."
+ [driver :- :keyword
+ native-query]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "validate-query"]
+ (interface/validate-query-impl parser driver native-query))))
+
+(mu/defn replace-names :- :string
+ "Replace schema, table, and column names in a SQL query.
+
+ `replacements` is a map with optional keys:
+ - :schemas - map of old schema name -> new schema name
+ - :tables - map of old table name -> new table name
+ - :columns - map of {:table t :column c} -> new column name
+
+ `opts` is an optional map with:
+ - :allow-unused? - if true, don't error when a replacement isn't used
+
+ Returns the modified SQL string.
+
+ SECURITY: Replacement values are injected into the SQL AST as identifier names
+ without sanitization. Callers MUST ensure replacement values are system-generated
+ (e.g., workspace isolation schema names, database metadata). Never pass
+ user-controlled input as replacement values."
+ ([driver sql-string replacements]
+ (replace-names driver sql-string replacements {}))
+ ([driver :- [:maybe :keyword]
+ sql-string :- :string
+ replacements :- ::replacements
+ opts :- ::replace-names-opts]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "replace-names"]
+ (interface/replace-names-impl parser driver sql-string replacements opts)))))
+
+(mu/defn referenced-tables-raw :- [:sequential ::table-spec]
+ "Given a driver and sql string, returns a sequence of {:schema <name> :table <name>} maps."
+ [driver :- :keyword
+ sql-str :- :string]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "referenced-tables-raw"]
+ (interface/referenced-tables-raw-impl parser driver sql-str))))
+
+(mu/defn simple-query? :- ::simple-query-result
+ "Check if SQL string is a simple SELECT (no LIMIT, OFFSET, or CTEs).
+ Used by Workspaces to determine if automatic checkpoints can be inserted.
+
+ Returns a map with:
+ - `:is_simple` - boolean indicating if query is simple
+ - `:reason` - string explaining why query is not simple (when false)"
+ [sql-string :- :string]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "simple-query?"]
+ (interface/simple-query?-impl parser sql-string))))
+
+(mu/defn add-into-clause :- :string
+ "Add an INTO clause to a SELECT statement.
+
+ Transforms: 'SELECT * FROM products'
+ Into: 'SELECT * INTO \"TABLE\" FROM products'
+
+ Used by SQL Server compile-transform which requires SELECT INTO syntax
+ instead of CREATE TABLE AS SELECT."
+ [driver :- :keyword
+ sql :- :string
+ table-name :- :string]
+ (let [parser (sql-tools.settings/current-parser-backend)]
+ (metrics/with-operation-timing [parser "add-into-clause"]
+ (interface/add-into-clause-impl parser driver sql table-name))))
diff --git a/src/metabase/sql_tools/init.clj b/src/metabase/sql_tools/init.clj
new file mode 100644
index 00000000000..eb9fa473d5d
--- /dev/null
+++ b/src/metabase/sql_tools/init.clj
@@ -0,0 +1,7 @@
+(ns metabase.sql-tools.init
+ (:require
+ ;; metabase.sql-tools.<parser> is required to ensure method's registration
+ [metabase.sql-tools.macaw.core]
+ [metabase.sql-tools.metrics]
+ [metabase.sql-tools.settings]
+ [metabase.sql-tools.sqlglot.core]))
diff --git a/src/metabase/sql_tools/interface.clj b/src/metabase/sql_tools/interface.clj
new file mode 100644
index 00000000000..78ee23f266c
--- /dev/null
+++ b/src/metabase/sql_tools/interface.clj
@@ -0,0 +1,67 @@
+(ns metabase.sql-tools.interface
+ "Multimethod definitions for sql-tools parser implementations.
+
+ This namespace contains only the multimethod definitions. Implementations
+ are in metabase.sql-tools.macaw.core and metabase.sql-tools.sqlglot.core.")
+
+(defn parser-driver-dispatch
+ "Dispatch function for sql-tools multimethods. Returns the parser keyword (e.g. :macaw, :sqlglot)."
+ [parser _driver & _args]
+ parser)
+
+(defmulti returned-columns-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/returned-columns]]. Do not use directly."
+ {:arglists '([parser driver native-query])}
+ parser-driver-dispatch)
+
+(defmulti referenced-tables-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/referenced-tables]]. Do not use directly."
+ {:arglists '([parser driver native-query])}
+ parser-driver-dispatch)
+
+(defmulti referenced-fields-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/referenced-fields]]. Do not use directly."
+ {:arglists '([parser driver native-query])}
+ parser-driver-dispatch)
+
+(defmulti field-references-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/field-references]]. Do not use directly.
+
+ Returns a map with:
+ - :used-fields - set of field specs from WHERE, JOIN ON, GROUP BY, ORDER BY
+ - :returned-fields - vector of field specs from SELECT clause (ordered)
+ - :errors - set of validation errors"
+ {:arglists '([parser driver sql-string])}
+ parser-driver-dispatch)
+
+(defmulti validate-query-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/validate-query]]. Do not use directly."
+ {:arglists '([parser driver native-query])}
+ parser-driver-dispatch)
+
+(defmulti replace-names-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/replace-names]]. Do not use directly."
+ {:arglists '([parser driver sql-string replacements opts])}
+ parser-driver-dispatch)
+
+(defmulti referenced-tables-raw-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/referenced-tables-raw]]. Do not use directly."
+ {:arglists '([parser driver sql-str])}
+ parser-driver-dispatch)
+
+(defmulti simple-query?-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/simple-query?]]. Do not use directly.
+
+ Returns a map with:
+ - `:is_simple` - boolean indicating if query is simple
+ - `:reason` - string explaining why query is not simple (optional)"
+ {:arglists '([parser sql-string])}
+ parser-driver-dispatch)
+
+(defmulti add-into-clause-impl
+ "Parser specific implementation of [[metabase.sql-tools.core/add-into-clause]]. Do not use directly.
+
+ Transforms a SELECT statement to include an INTO clause for SQL Server style
+ SELECT INTO syntax."
+ {:arglists '([parser driver sql table-name])}
+ parser-driver-dispatch)
diff --git a/src/metabase/sql_tools/macaw/core.clj b/src/metabase/sql_tools/macaw/core.clj
new file mode 100644
index 00000000000..a4a010d8e77
--- /dev/null
+++ b/src/metabase/sql_tools/macaw/core.clj
@@ -0,0 +1,191 @@
+(ns metabase.sql-tools.macaw.core
+ (:require
+ [clojure.string :as str]
+ [macaw.core :as macaw]
+ [metabase.driver :as driver]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.sql-tools.common :as sql-tools.common]
+ [metabase.sql-tools.interface :as sql-tools]
+ [metabase.sql-tools.macaw.references :as sql-tools.macaw.references]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+;;;; Macaw options and parsing
+
+(def ^:private considered-drivers
+ "The set of drivers for which we configure non-trivial macaw options."
+ #{:h2 :mysql :postgres :redshift :sqlite :sqlserver})
+
+(defn- macaw-options
+ "Generate the options expected by Macaw based on the nature of the given driver."
+ [driver]
+ (merge
+ ;; If this isn't a driver we've considered, fallback to Macaw's conservative defaults.
+ (when (contains? considered-drivers driver)
+ {;; According to the SQL-92 specification, non-quoted identifiers should be case-insensitive, and the majority of
+ ;; engines are implemented this way.
+ ;;
+ ;; In practice there are exceptions, notably MySQL and SQL Server, where case sensitivity is a property of the
+ ;; underlying resource referenced by the identifier, and the case-sensitivity does not depend on whether the
+ ;; reference is quoted.
+ ;;
+ ;; For MySQL the case sensitivity of databases and tables depends on both the underlying file system, and a system
+ ;; variable used to initialize the database. For SQL Server it depends on the collation settings of the collection
+ ;; where the corresponding schema element is defined.
+ ;;
+ ;; For MySQL, columns and aliases can never be case-sensitive, and for SQL Server the default collation is case-
+ ;; insensitive too, so it makes sense to just treat all databases as case-insensitive as a whole.
+ ;;
+ ;; In future, Macaw may support discriminating on the identifier type, in which case we could be more precise for
+ ;; these databases. Being 100% correct would require querying system variables and schema configuration however,
+ ;; which is likely a step too far in complexity.
+ ;;
+ ;; Currently, we go with :agnostic, as it is the most relaxed semantics (the case of both the identifiers and the
+ ;; underlying schema is totally ignored, and correspondence is non-deterministic), but Macaw supports more nuanced
+ ;; :lower and :upper configuration values which coerce the query identifiers to a given case then do an exact
+ ;; comparison with the schema.
+ :case-insensitive :agnostic
+ ;; For both MySQL and SQL Server, whether identifiers are case-sensitive depends on database configuration only,
+ ;; and quoting has no effect on this, so we disable this option for consistency with `:case-insensitive`.
+ :quotes-preserve-case? (not (contains? #{:mysql :sqlserver} driver))
+ :features {:postgres-syntax (isa? driver/hierarchy driver :postgres)
+ :square-bracket-quotes (= :sqlserver driver)
+ :unsupported-statements false
+ :backslash-escape-char true
+ ;; This will slow things down, but until we measure the difference, opt for correctness.
+ :complex-parsing true}
+ ;; 10 seconds
+ :timeout 10000})
+ {;; There is no plan to be exhaustive yet.
+ ;; Note that while an allowed list would be more conservative, at the time of writing only 1 of the bundled
+ ;; drivers use FINAL as a reserved word, and mentioning them all would be prohibitive.
+ ;; In the future, we will use multimethods to define this explicitly per driver, or even discover it automatically
+ ;; through the JDBC connection, where possible.
+ :non-reserved-words (vec (remove nil? [(when-not (contains? #{:clickhouse} driver)
+ :final)]))}))
+
+(defn- parsed-query
+ "Parse SQL using Macaw with driver-specific options."
+ [sql driver & {:as opts}]
+ (macaw/parsed-query sql (merge (macaw-options driver) opts)))
+
+;;;; referenced-tables
+
+(defn split-compound-table-spec
+ "Macaw doesn't understand multi-part identifiers like BigQuery's `dataset.table` or
+ `project.dataset.table`, returning them as a single compound `:table` value.
+ This splits such compound names into separate `:schema` and `:table` parts.
+
+ - `table` → `{:table \"table\"}`
+ - `schema.table` → `{:schema \"schema\", :table \"table\"}`
+ - `catalog.schema.table` → `{:schema \"schema\", :table \"table\"}`"
+ [{:keys [table schema] :as table-spec}]
+ (if (and table (nil? schema) (str/includes? table "."))
+ (let [parts (str/split table #"\.")]
+ (case (count parts)
+ 1 table-spec
+ 2 {:schema (first parts) :table (second parts)}
+ ;; 3+ parts: take last two as schema.table (drop catalog/project prefix)
+ {:schema (nth parts (- (count parts) 2))
+ :table (last parts)}))
+ table-spec))
+
+(mu/defn- referenced-tables
+ [driver :- :keyword
+ query :- :metabase.lib.schema/native-only-query]
+ (let [db-tables (lib.metadata/tables query)
+ db-transforms (lib.metadata/transforms query)]
+ (-> query
+ lib/raw-native-query
+ (parsed-query driver)
+ (macaw/query->components {:strip-contexts? true})
+ :tables
+ (->> (map :component))
+ (->> (map split-compound-table-spec))
+ (->> (into #{}
+ (keep #(->> (sql-tools.common/normalize-table-spec driver %)
+ (sql-tools.common/find-table-or-transform driver db-tables db-transforms))))))))
+
+(defmethod sql-tools/referenced-tables-impl :macaw
+ [_parser driver query]
+ (referenced-tables driver query))
+
+;;;; field-references
+
+(defmethod sql-tools/field-references-impl :macaw
+ [_parser driver sql-string]
+ (-> sql-string
+ (parsed-query driver)
+ macaw/->ast
+ (->> (sql-tools.macaw.references/field-references driver))))
+
+;;;; returned-columns
+
+(defmethod sql-tools/returned-columns-impl :macaw
+ [parser driver query]
+ (sql-tools.common/returned-columns parser driver query))
+
+;;;; referenced-fields
+
+(defmethod sql-tools/referenced-fields-impl :macaw
+ [parser driver query]
+ (sql-tools.common/referenced-fields parser driver query))
+
+(defmethod sql-tools/validate-query-impl :macaw
+ [parser driver query]
+ (sql-tools.common/validate-query parser driver query))
+
+(defmethod sql-tools/referenced-tables-raw-impl :macaw
+ [_parser _driver sql-str]
+ (-> sql-str
+ (macaw/parsed-query)
+ (macaw/query->components {:strip-contexts? true})
+ :tables
+ (->> (map :component)
+ (mapv (fn [{:keys [schema table]}]
+ (cond-> {:table table}
+ schema (assoc :schema schema)))))))
+
+(defmethod sql-tools/simple-query?-impl :macaw
+ [_parser sql-string]
+ (try
+ ;; BEWARE: No driver available, so we pass nil. This means macaw-options will be minimal.
+ (let [^net.sf.jsqlparser.statement.select.PlainSelect parsed (parsed-query sql-string nil)]
+ (cond
+ (not (instance? net.sf.jsqlparser.statement.select.PlainSelect parsed))
+ {:is_simple false
+ :reason "Not a simple SELECT"}
+
+ (.getLimit parsed)
+ {:is_simple false
+ :reason "Contains a LIMIT"}
+
+ (.getOffset parsed)
+ {:is_simple false
+ :reason "Contains an OFFSET"}
+
+ (seq (.getWithItemsList parsed))
+ {:is_simple false
+ :reason "Contains a CTE"}
+
+ :else
+ {:is_simple true}))
+ (catch Exception e
+ (log/debugf e "Failed to parse query: %s" (ex-message e))
+ {:is_simple false})))
+
+(defmethod sql-tools/add-into-clause-impl :macaw
+ [_parser driver sql table-name]
+ (let [^net.sf.jsqlparser.statement.select.Select parsed-query (parsed-query sql driver)
+ ^net.sf.jsqlparser.statement.select.PlainSelect select-body (.getSelectBody parsed-query)]
+ (.setIntoTables select-body [(net.sf.jsqlparser.schema.Table. ^String table-name)])
+ (str parsed-query)))
+
+(defmethod sql-tools/replace-names-impl :macaw
+ [_parser driver sql-string replacements opts]
+ ;; Note: :case-insensitive :agnostic causes ClassCastException in Macaw's replace-names
+ ;; due to regex pattern handling. Omit it for now until Macaw is fixed.
+ (macaw/replace-names sql-string replacements (merge (dissoc (macaw-options driver) :case-insensitive) opts)))
diff --git a/src/metabase/driver/sql/references.clj b/src/metabase/sql_tools/macaw/references.clj
similarity index 88%
rename from src/metabase/driver/sql/references.clj
rename to src/metabase/sql_tools/macaw/references.clj
index 9d4d8c69c2f..46a1d73ddb8 100644
--- a/src/metabase/driver/sql/references.clj
+++ b/src/metabase/sql_tools/macaw/references.clj
@@ -1,11 +1,12 @@
-(ns metabase.driver.sql.references
+(ns metabase.sql-tools.macaw.references
(:refer-clojure :exclude [every? mapv select-keys some])
(:require
- [clojure.string :as str]
[macaw.ast-types :as macaw.ast-types]
[metabase.driver :as driver]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.sql.normalize :as sql.normalize]
+ [metabase.driver.sql :as driver.sql]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.validate :as lib.schema.validate]
+ [metabase.sql-tools.common :as sql-tools.common]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.performance :refer [every? mapv select-keys some]]))
@@ -62,12 +63,12 @@
[:map
[:used-fields [:set [:ref ::col-spec]]]
[:returned-fields [:sequential [:ref ::col-spec]]]
- [:errors [:set [:ref driver-api/schema.validate.error]]]])
+ [:errors [:set [:ref ::lib.schema.validate/error]]]])
(defn- normalize-fields [driver m]
(update-vals m
#(if (string? %)
- (sql.normalize/normalize-name driver %)
+ (driver.sql/normalize-name driver %)
%)))
(defn- col-fields [driver m]
@@ -89,22 +90,11 @@
sublist))
sources))
-(mu/defn table-name :- [:maybe :string]
- "Computes a table name from a table reference"
- [raw-col :- [:map
- [:database {:optional true} :string]
- [:schema {:optional true} :string]
- [:table {:optional true} :string]]]
- (when (:table raw-col)
- (->> [:database :schema :table]
- (keep raw-col)
- (str/join "."))))
-
(defn- get-column [driver sources raw-col {:keys [return-table-matches?]}]
(if (and (nil? (:table raw-col))
(nil? (:schema raw-col))
(nil? (:database raw-col))
- (sql.normalize/reserved-literal driver (:column raw-col)))
+ (driver.sql/reserved-literal driver (:column raw-col)))
[]
(let [{:keys [alias column table] :as expr} (col-fields driver raw-col)
valid-sources (if table
@@ -141,9 +131,9 @@
:source-columns source-columns}
;; if we didn't find any potential sources, signal an error here
:errors (when-not (some #(some identity %) valid-sources)
- #{(if-let [name (table-name raw-col)]
- (driver-api/missing-table-alias-error name)
- (driver-api/missing-column-error column))})}]))))
+ #{(if-let [name (sql-tools.common/table-name raw-col)]
+ (lib/missing-table-alias-error name)
+ (lib/missing-column-error column))})}]))))
(defmulti find-used-fields
"Finds the fields used in a given sql expression."
@@ -220,7 +210,7 @@
(defmethod find-returned-fields :default
[driver sources withs {:keys [alias] :as expr}]
[{:col {:alias (when alias
- (sql.normalize/normalize-name driver alias))
+ (driver.sql/normalize-name driver alias))
:type :custom-field
:used-fields (into #{} (keep :col) (find-used-fields driver sources withs expr))}}])
@@ -228,23 +218,18 @@
[driver sources _withs expr]
(get-column driver sources expr {:return-table-matches? true}))
-(defn wrap-col
- "Wraps the argument in `{:col _}`"
- [col]
- {:col col})
-
(defmethod find-returned-fields [:sql :macaw.ast/wildcard]
[_driver sources _withs _expr]
(into []
- (mapcat #(->> % :returned-fields (map wrap-col)))
+ (mapcat #(->> % :returned-fields (map sql-tools.common/wrap-col)))
(first sources)))
(defmethod find-returned-fields [:sql :macaw.ast/table-wildcard]
[driver sources _withs expr]
(or (some->> (find-source (col-fields driver expr) sources)
:returned-fields
- (map wrap-col))
- [{:errors #{(driver-api/missing-table-alias-error (table-name expr))}}]))
+ (map sql-tools.common/wrap-col))
+ [{:errors #{(lib/missing-table-alias-error (sql-tools.common/table-name expr))}}]))
(defmethod find-returned-fields [:sql :macaw.ast/set-operation]
[driver sources withs expr]
@@ -295,14 +280,14 @@
rec (partial find-used-fields driver with-select withs)]
(into #{}
cat
- [ ;; a select can't refer to its own aliases, so don't include them in sources here
+ [;; a select can't refer to its own aliases, so don't include them in sources here
(mapcat (partial find-used-fields driver with-outside withs)
(:select expr))
(rec (:where expr))
(mapcat rec (:join expr))
(mapcat rec (:group-by expr))
(mapcat rec (:order-by expr))
- (mapcat #(->> % :used-fields (map wrap-col))
+ (mapcat #(->> % :used-fields (map sql-tools.common/wrap-col))
local-sources)])))
(defmethod find-returned-fields [:sql :macaw.ast/select]
@@ -314,7 +299,7 @@
;; :alias is a column alias, so this is presumably something like `select (select * from ...)`
;; there should only be one field returned, and that field should have the appropriate alias
(do (assert (= 1 (count returned-fields)))
- (map #(assoc-in % [:col :alias] (sql.normalize/normalize-name driver (:alias expr)))
+ (map #(assoc-in % [:col :alias] (driver.sql/normalize-name driver (:alias expr)))
returned-fields))
returned-fields)))
@@ -336,7 +321,7 @@
(comp cat (mapcat :errors))
[returned-fields current-used-fields local-withs])
:names (when-let [alias (:table-alias expr)]
- {:table-alias (sql.normalize/normalize-name driver alias)})}))
+ {:table-alias (driver.sql/normalize-name driver alias)})}))
(defmethod field-references-impl [:sql :macaw.ast/join]
[driver outside-sources withs expr]
@@ -352,14 +337,14 @@
(comp cat (mapcat :errors))
[used-fields returned-fields])
:names (when-let [alias (:table-alias expr)]
- {:table-alias (sql.normalize/normalize-name driver alias)})}))
+ {:table-alias (driver.sql/normalize-name driver alias)})}))
(defmethod field-references-impl [:sql :macaw.ast/table-function]
[driver _outside-sources _withs expr]
{:used-fields #{}
:returned-fields [{:type :unknown-columns}]
:names (when-let [alias (:table-alias expr)]
- {:table-alias (sql.normalize/normalize-name driver alias)})
+ {:table-alias (driver.sql/normalize-name driver alias)})
:errors #{}})
(defmethod field-references-impl :default
@@ -367,7 +352,7 @@
{:used-fields #{}
:returned-fields []
:names nil
- :errors #{(driver-api/syntax-error)}})
+ :errors #{(lib/syntax-error)}})
(mu/defn field-references :- [:ref ::field-references]
"Takes a sql query in the macaw ast format and returns the fields referenced by a query.
diff --git a/src/metabase/sql_tools/metrics.clj b/src/metabase/sql_tools/metrics.clj
new file mode 100644
index 00000000000..659372b89f3
--- /dev/null
+++ b/src/metabase/sql_tools/metrics.clj
@@ -0,0 +1,79 @@
+(ns metabase.sql-tools.metrics
+ "Prometheus metrics instrumentation for sql-tools operations.
+
+ Provides recording functions and a `with-operation-timing` macro that wraps
+ sql-tools public API calls with counters and histograms, labeled by parser
+ backend and operation name."
+ (:require
+ [metabase.analytics.core :as analytics]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+;;; -------------------------------------------------- Known labels --------------------------------------------------
+
+(def ^:private parsers ["macaw" "sqlglot"])
+
+(def ^:private operations
+ ["returned-columns"
+ "referenced-tables"
+ "referenced-fields"
+ "field-references"
+ "validate-query"
+ "replace-names"
+ "referenced-tables-raw"
+ "simple-query?"
+ "add-into-clause"])
+
+(def ^:private operation-labels
+ (vec (for [parser parsers
+ operation operations]
+ {:parser parser :operation operation})))
+
+(defmethod analytics/known-labels :metabase-sql-tools/operations-total [_] operation-labels)
+(defmethod analytics/known-labels :metabase-sql-tools/operations-completed [_] operation-labels)
+(defmethod analytics/known-labels :metabase-sql-tools/operations-failed [_] operation-labels)
+
+;;; -------------------------------------------------- Recording fns --------------------------------------------------
+
+(defn record-operation-start!
+ "Record the start of an sql-tools operation."
+ [parser operation]
+ (analytics/inc! :metabase-sql-tools/operations-total
+ {:parser (name parser) :operation operation}))
+
+(defn record-operation-completion!
+ "Record the successful completion of an sql-tools operation."
+ [parser operation duration-ms]
+ (let [labels {:parser (name parser) :operation operation}]
+ (analytics/inc! :metabase-sql-tools/operations-completed labels)
+ (analytics/observe! :metabase-sql-tools/operation-duration-ms labels duration-ms)))
+
+(defn record-operation-failure!
+ "Record the failure of an sql-tools operation."
+ [parser operation duration-ms]
+ (let [labels {:parser (name parser) :operation operation}]
+ (analytics/inc! :metabase-sql-tools/operations-failed labels)
+ (analytics/observe! :metabase-sql-tools/operation-duration-ms labels duration-ms)))
+
+;;; -------------------------------------------------- Timing macro --------------------------------------------------
+
+(defmacro with-operation-timing
+ "Execute body while timing an sql-tools operation. Automatically records
+ start, completion or failure metrics labeled by parser and operation.
+
+ Usage:
+ (with-operation-timing [parser \"returned-columns\"]
+ (interface/returned-columns-impl parser driver native-query))"
+ [[parser operation] & body]
+ `(let [parser# ~parser
+ operation# ~operation
+ start# (u/start-timer)]
+ (record-operation-start! parser# operation#)
+ (try
+ (let [result# (do ~@body)]
+ (record-operation-completion! parser# operation# (long (u/since-ms start#)))
+ result#)
+ (catch Throwable t#
+ (record-operation-failure! parser# operation# (long (u/since-ms start#)))
+ (throw t#)))))
diff --git a/src/metabase/sql_tools/settings.clj b/src/metabase/sql_tools/settings.clj
new file mode 100644
index 00000000000..b9bad486bc1
--- /dev/null
+++ b/src/metabase/sql_tools/settings.clj
@@ -0,0 +1,54 @@
+(ns metabase.sql-tools.settings
+ "Settings for the sql-tools module.
+
+ ## Swapping Between Macaw and SQLGlot
+
+ The SQL parser backend can be switched using the `MB_SQL_TOOLS_PARSER_BACKEND` env var:
+
+ ```bash
+ # Use SQLGlot (default)
+ MB_SQL_TOOLS_PARSER_BACKEND=sqlglot
+
+ # Use Macaw (legacy)
+ MB_SQL_TOOLS_PARSER_BACKEND=macaw
+ ```
+
+ This is useful for:
+ - Running tests with a specific backend:
+ `MB_SQL_TOOLS_PARSER_BACKEND=macaw bin/mage run-tests :some/test-ns`
+ - Debugging parsing differences between backends
+ - Falling back to Macaw if SQLGlot has issues
+
+ **Note**: This is an internal flag only, not exposed in public docs.
+ Once SQLGlot is stable, this env var will be removed and Macaw deprecated."
+ (:require
+ [metabase.settings.core :as setting :refer [defsetting]]
+ [metabase.util.i18n :refer [deferred-tru tru]]))
+
+(def ^:private available-parser-backends
+ #{:macaw :sqlglot})
+
+(def ^:dynamic *parser-backend-override*
+ "Dynamic var for overriding the parser backend in tests.
+ When bound, [[current-parser-backend]] returns this instead of the setting."
+ nil)
+
+(defsetting sql-tools-parser-backend
+ (deferred-tru "Parser backend of `sql-tools` module.")
+ :visibility :internal
+ :export? false
+ :type :keyword
+ :default :sqlglot
+ :setter (fn [new-value]
+ (or (available-parser-backends (keyword new-value))
+ (throw (ex-info (tru "Invalid sql-tools parser backend.")
+ {:value (keyword new-value)
+ :available available-parser-backends})))
+ (setting/set-value-of-type! :keyword :sql-tools-parser-backend new-value)))
+
+(defn current-parser-backend
+ "Get the current parser backend. Checks [[*parser-backend-override*]] first,
+ then falls back to the [[sql-tools-parser-backend]] setting."
+ []
+ (or *parser-backend-override*
+ (sql-tools-parser-backend)))
diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.clj
new file mode 100644
index 00000000000..904d8942126
--- /dev/null
+++ b/src/metabase/sql_tools/sqlglot/core.clj
@@ -0,0 +1,178 @@
+(ns metabase.sql-tools.sqlglot.core
+ (:require
+ [clojure.string :as str]
+ [metabase.driver.sql :as driver.sql]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.sql-parsing.core :as sql-parsing]
+ [metabase.sql-tools.common :as sql-tools.common]
+ [metabase.sql-tools.interface :as sql-tools]
+ [metabase.util.log :as log])
+ (:import
+ (org.graalvm.polyglot PolyglotException)))
+
+(set! *warn-on-reflection* true)
+
+(defn- convert-error
+ "Convert sql-parsing generic error format to lib.validate format.
+ sql-parsing returns generic maps like {:type :syntax-error} or {:type :timeout :message \"...\"},
+ and this function translates them to the lib.validate error format."
+ [error]
+ (case (:type error)
+ :syntax-error (lib/syntax-error)
+ :missing-column (lib/missing-column-error (:name error))
+ :missing-table-alias (lib/missing-table-alias-error (:name error))
+ :timeout (lib/validation-exception-error (:message error))
+ ;; Fallback: pass through if already in correct format or unknown type
+ error))
+
+(defn driver->dialect
+ "Map a Metabase driver keyword to a SQLGlot dialect string.
+ Returns nil for drivers that should use SQLGlot's default dialect (e.g., H2)."
+ [driver]
+ (case driver
+ nil nil
+ :postgres "postgres"
+ :mysql "mysql"
+ :snowflake "snowflake"
+ :bigquery "bigquery"
+ :bigquery-cloud-sdk "bigquery"
+ :redshift "redshift"
+ :sqlserver "tsql"
+ :sparksql "spark"
+ :presto-jdbc "presto"
+ :vertica nil
+ :h2 nil
+ ;; Default: try using the driver name as dialect
+ (name driver)))
+
+;;;; Tables
+
+(defn- referenced-tables
+ [driver query]
+ (try
+ (let [db-tables (lib.metadata/tables query)
+ db-transforms (lib.metadata/transforms query)
+ sql (lib/raw-native-query query)
+ default-schema (driver.sql/default-schema driver)
+ query-tables (sql-parsing/referenced-tables (driver->dialect driver) sql)]
+ (into #{}
+ (keep (fn [[_catalog table-schema table]]
+ (sql-tools.common/find-table-or-transform
+ driver db-tables db-transforms
+ (sql-tools.common/normalize-table-spec
+ driver {:table table
+ :schema (or table-schema default-schema)}))))
+ query-tables))
+ (catch PolyglotException e
+ ;; Return empty sequence on parse error to follow the Macaw implementation behavior.
+ (if (str/starts-with? (str (.getMessage e)) "ParseError")
+ #{}
+ (throw e)))))
+
+(defmethod sql-tools/referenced-tables-impl :sqlglot
+ [_parser driver query]
+ (referenced-tables driver query))
+
+;;;; field-references
+
+(def ^:private ^:const normalizable-keys
+ "Keys that should have their string values normalized for case.
+ Matches Macaw's col-fields which normalizes [:type :column :table :schema :database :alias]."
+ #{:type :column :table :schema :database :alias})
+
+(defn- normalize-field
+ "Normalize identifier strings in a field spec using driver-specific case rules.
+ Only normalizes specific keys (column, table, schema, etc.) to match Macaw's col-fields."
+ [driver field]
+ (if (map? field)
+ (reduce-kv (fn [m k v]
+ (assoc m k
+ (cond
+ (and (normalizable-keys k) (string? v))
+ (driver.sql/normalize-name driver v)
+
+ (map? v)
+ (normalize-field driver v)
+
+ (set? v)
+ (set (map #(normalize-field driver %) v))
+
+ (sequential? v)
+ (mapv #(normalize-field driver %) v)
+
+ :else v)))
+ {}
+ field)
+ field))
+
+(defmethod sql-tools/field-references-impl :sqlglot
+ [_parser driver sql-string]
+ (let [result (sql-parsing/field-references (driver->dialect driver) sql-string)]
+ (-> result
+ (update :used-fields #(set (map (partial normalize-field driver) %)))
+ (update :returned-fields #(mapv (partial normalize-field driver) %))
+ (update :errors #(set (map convert-error %))))))
+
+;;;; referenced-fields
+
+(defmethod sql-tools/referenced-fields-impl :sqlglot
+ [parser driver query]
+ (sql-tools.common/referenced-fields parser driver query))
+
+;;;; Validation
+;; SQLGlot validation uses the same pipeline as Macaw:
+;; field-references (dispatched to :sqlglot) → resolve-field (Macaw's logic)
+(defmethod sql-tools/validate-query-impl :sqlglot
+ [parser driver query]
+ (sql-tools.common/validate-query parser driver query))
+
+;;;; returned-columns
+;; SQLGlot returned-columns uses the same pipeline as Macaw:
+;; field-references (dispatched to :sqlglot) → resolve-field (Macaw's logic)
+(defmethod sql-tools/returned-columns-impl :sqlglot
+ [parser driver query]
+ (sql-tools.common/returned-columns parser driver query))
+
+(defmethod sql-tools/referenced-tables-raw-impl :sqlglot
+ [_parser driver sql-str]
+ (try
+ (let [dialect (driver->dialect driver)
+ ;; sql-parsing/referenced-tables returns [[catalog schema table] ...]
+ ;; Convert to [{:schema ... :table ...} ...] format.
+ ;; Do NOT normalize case here — SQLGlot already applies dialect-appropriate case rules
+ ;; (e.g., uppercase for Snowflake, lowercase for Postgres). Additional normalization
+ ;; via normalize-table-spec would incorrectly lowercase Snowflake identifiers, breaking
+ ;; AppDB lookups where identifiers are stored in their native case.
+ ;; This matches the Macaw implementation which also returns raw identifiers.
+ table-tuples (sql-parsing/referenced-tables dialect sql-str)]
+ (mapv (fn [[_catalog schema table]]
+ {:schema schema :table table})
+ table-tuples))
+ (catch PolyglotException e
+ ;; Return empty sequence on parse error to follow the Macaw implementation behavior.
+ (if (str/starts-with? (str (.getMessage e)) "ParseError")
+ []
+ (throw e)))))
+
+(defmethod sql-tools/simple-query?-impl :sqlglot
+ [_parser sql-string]
+ (try
+ ;; No dialect available from caller, use nil for SQLGlot's default dialect
+ (sql-parsing/simple-query? nil sql-string)
+ (catch Exception e
+ (log/debugf e "Failed to parse query: %s" (ex-message e))
+ {:is_simple false})))
+
+(defmethod sql-tools/add-into-clause-impl :sqlglot
+ [_parser driver sql table-name]
+ (sql-parsing/add-into-clause (driver->dialect driver) sql table-name))
+
+(defmethod sql-tools/replace-names-impl :sqlglot
+ [_parser driver sql-string replacements _opts]
+ ;; Convert map keys to list-of-pairs for JSON serialization
+ ;; {:tables {{:table "a"} "b"}} -> {:tables [[{:table "a"} "b"]]}
+ (let [replacements' (-> replacements
+ (update :tables #(when % (vec %)))
+ (update :columns #(when % (vec %))))]
+ (sql-parsing/replace-names (driver->dialect driver) sql-string replacements')))
diff --git a/src/metabase/transforms/api/transform.clj b/src/metabase/transforms/api/transform.clj
index 322939f8614..aed523ae748 100644
--- a/src/metabase/transforms/api/transform.clj
+++ b/src/metabase/transforms/api/transform.clj
@@ -17,6 +17,7 @@
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.schema :as qp.schema]
[metabase.request.core :as request]
+ [metabase.sql-tools.core :as sql-tools]
[metabase.transforms.api.transform-job]
[metabase.transforms.api.transform-tag]
[metabase.transforms.canceling :as transforms.canceling]
@@ -33,11 +34,8 @@
[metabase.util.malli.schema :as ms]
[ring.util.response :as response]
[toucan2.core :as t2])
- ;; TODO (Chris 2026-01-22) -- Remove jsqlparser imports/typehints to be SQL parser-agnostic
(:import
- (java.sql PreparedStatement)
- ^{:clj-kondo/ignore [:metabase/no-jsqlparser-imports]}
- (net.sf.jsqlparser.statement.select PlainSelect)))
+ (java.sql PreparedStatement)))
(comment metabase.transforms.api.transform-job/keep-me
metabase.transforms.api.transform-tag/keep-me)
@@ -541,36 +539,10 @@
(run-transform! (api/write-check :model/Transform id)))
(defn- simple-native-query?
- "Checks if a native SQL query string is simple enough for automatic checkpoint insertion."
+ "Checks if a native SQL query string is simple enough for automatic checkpoint insertion.
+ Delegates to sql-tools which dispatches to the configured parser backend (macaw or sqlglot)."
[sql-string]
- (try
- ;; BEWARE: The API endpoint (caller) does not have info on database engine this query should run on. Hence
- ;; there's no way of providing appropriate [[metabase.driver.util/macaw-options]]. `nil` is best-effort
- ;; adding at least default :non-resserved-words.
- ;; TODO (Chris 2026-01-22) -- Remove jsqlparser typehints to be SQL parser-agnostic
- (let [^PlainSelect parsed (driver.u/parsed-query sql-string nil)]
- (cond
- (not (instance? PlainSelect parsed))
- {:is_simple false
- :reason "Not a simple SELECT"}
-
- (.getLimit parsed)
- {:is_simple false
- :reason "Contains a LIMIT"}
-
- (.getOffset ^PlainSelect parsed)
- {:is_simple false
- :reason "Contains an OFFSET"}
-
- (seq (.getWithItemsList ^PlainSelect parsed))
- {:is_simple false
- :reason "Contains a CTE"}
-
- :else
- {:is_simple true}))
- (catch Exception e
- (log/debugf e "Failed to parse query: %s" (ex-message e))
- {:is_simple false})))
+ (sql-tools/simple-query? sql-string))
(api.macros/defendpoint :post "/is-simple-query" :- [:map
[:is_simple :boolean]
diff --git a/test/metabase/sql_parsing/core_test.clj b/test/metabase/sql_parsing/core_test.clj
new file mode 100644
index 00000000000..8b0d0ac79d9
--- /dev/null
+++ b/test/metabase/sql_parsing/core_test.clj
@@ -0,0 +1,484 @@
+(ns metabase.sql-parsing.core-test
+ (:require
+ [clojure.java.io :as io]
+ [clojure.test :refer :all]
+ [metabase.sql-parsing.core :as sql-parsing]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------ referenced-tables API Tests -----------------------------------------
+
+(deftest ^:parallel referenced-tables-basic-test
+ (testing "Simple table extraction"
+ (is (= [[nil nil "users"]]
+ (sql-parsing/referenced-tables "postgres" "SELECT * FROM users"))))
+
+ (testing "Multiple tables from JOIN"
+ (is (= [[nil nil "orders"] [nil nil "users"]]
+ (sql-parsing/referenced-tables
+ "postgres"
+ "SELECT * FROM users u LEFT JOIN orders o ON u.id = o.user_id"))))
+
+ (testing "Schema-qualified tables are preserved"
+ (is (= [[nil "other" "users"] [nil "public" "users"]]
+ (sql-parsing/referenced-tables
+ "postgres"
+ "SELECT public.users.id, other.users.id
+ FROM public.users u1
+ LEFT JOIN other.users u2 ON u1.id = u2.id"))))
+
+ (testing "CTE names are excluded"
+ (is (= [[nil nil "users"]]
+ (sql-parsing/referenced-tables
+ "postgres"
+ "WITH active AS (SELECT * FROM users WHERE active) SELECT * FROM active")))))
+
+(deftest ^:parallel cte-test
+ (testing "CTE (WITH clause) parsing"
+ (is (= [[nil nil "users"]]
+ (sql-parsing/referenced-tables
+ "postgres"
+ "WITH active_users AS (SELECT * FROM users WHERE active)
+ SELECT * FROM active_users")))))
+
+(deftest ^:parallel join-test
+ (testing "JOIN parsing extracts all tables"
+ (is (=? [[nil nil "orders"] [nil nil "users"]]
+ (sql-parsing/referenced-tables
+ "postgres"
+ "SELECT u.name, o.total
+ FROM users u
+ JOIN orders o ON u.id = o.user_id")))))
+
+(deftest ^:parallel subquery-test
+ (testing "Subquery table extraction"
+ (is (=? [[nil nil "users"]]
+ (sql-parsing/referenced-tables
+ "postgres"
+ "SELECT * FROM (SELECT id FROM users) AS sub")))))
+
+;;; ---------------------------------------- Catalog/Schema Extraction Tests ----------------------------------------
+;;; These tests verify the 3-tuple (tables) and 4-tuple (fields) API per the design doc:
+;;; - BigQuery: project.dataset.table → [project, dataset, table]
+;;; - Snowflake: database.schema.table → [database, schema, table]
+;;; - PostgreSQL: schema.table → [nil, schema, table]
+;;; - MySQL: database.table → [nil, database, table] (MySQL's "database" = schema level)
+
+(deftest ^:parallel catalog-schema-tables-bigquery-test
+ (testing "BigQuery: project.dataset.table extracts all three levels"
+ (is (= [["myproject" "analytics" "events"]]
+ (sql-parsing/referenced-tables "bigquery" "SELECT * FROM myproject.analytics.events")))
+ (is (= [["myproject" "analytics" "events"] ["myproject" "analytics" "users"]]
+ (sql-parsing/referenced-tables "bigquery"
+ "SELECT * FROM myproject.analytics.events e
+ JOIN myproject.analytics.users u ON e.user_id = u.id")))))
+
+(deftest ^:parallel catalog-schema-tables-snowflake-test
+ (testing "Snowflake: database.schema.table extracts all three levels"
+ ;; Snowflake uppercases identifiers by default
+ (let [result (sql-parsing/referenced-tables "snowflake" "SELECT * FROM mydb.public.orders")
+ normalized (mapv (fn [[c s t]] [(some-> c u/lower-case-en)
+ (some-> s u/lower-case-en)
+ (u/lower-case-en t)])
+ result)]
+ (is (= [["mydb" "public" "orders"]] normalized)))))
+
+(deftest ^:parallel catalog-schema-tables-postgres-test
+ (testing "PostgreSQL: schema.table (no cross-database queries)"
+ (is (= [[nil "public" "users"]]
+ (sql-parsing/referenced-tables "postgres" "SELECT * FROM public.users")))
+ (is (= [[nil "other_schema" "accounts"] [nil "public" "users"]]
+ (sql-parsing/referenced-tables "postgres"
+ "SELECT * FROM public.users u
+ JOIN other_schema.accounts a ON u.id = a.user_id")))))
+
+(deftest ^:parallel catalog-schema-tables-mysql-test
+ (testing "MySQL: database.table (MySQL 'database' maps to schema slot)"
+ ;; In MySQL, CREATE SCHEMA = CREATE DATABASE, so there's no separate schema level
+ ;; SQLGlot maps MySQL's database to the schema slot (slot 2)
+ (is (= [[nil "mydb" "users"]]
+ (sql-parsing/referenced-tables "mysql" "SELECT * FROM mydb.users")))
+ (is (= [[nil nil "users"]]
+ (sql-parsing/referenced-tables "mysql" "SELECT * FROM users")))))
+
+(deftest ^:parallel catalog-schema-fields-bigquery-test
+ (testing "BigQuery: fully-qualified field references"
+ (is (= [["myproject" "analytics" "users" "email"]]
+ (sql-parsing/referenced-fields "bigquery"
+ "SELECT myproject.analytics.users.email
+ FROM myproject.analytics.users")))))
+
+(deftest ^:parallel catalog-schema-fields-postgres-test
+ (testing "PostgreSQL: schema-qualified field references"
+ (is (= [[nil "public" "orders" "total"]]
+ (sql-parsing/referenced-fields "postgres" "SELECT public.orders.total FROM public.orders")))))
+
+(deftest ^:parallel catalog-schema-fields-mysql-test
+ (testing "MySQL: database-qualified field references"
+ (is (= [[nil "mydb" "users" "id"]]
+ (sql-parsing/referenced-fields "mysql" "SELECT mydb.users.id FROM mydb.users")))
+ (is (= [[nil nil "users" "id"]]
+ (sql-parsing/referenced-fields "mysql" "SELECT id FROM users")))))
+
+;;; -------------------------------------------- UDTF (Table Function) Tests -----------------------------------------
+
+;; Dialect-specific UDTF queries - each dialect may have slightly different syntax
+;; for table-valued functions. The queries should be semantically equivalent.
+;; We use a generic function name for dialects without well-known UDTFs to test
+;; that unknown functions are handled gracefully.
+(def udtf-queries
+ {:postgres "SELECT * FROM generate_series(1, 10)"
+ :mysql "SELECT * FROM my_table_func(1, 10)" ; MySQL's JSON_TABLE has complex syntax
+ :snowflake "SELECT * FROM TABLE(FLATTEN(input => ARRAY_CONSTRUCT(1, 2, 3)))"
+ :bigquery "SELECT * FROM UNNEST([1, 2, 3]) AS val"
+ :redshift "SELECT * FROM generate_series(1, 10)"
+ :duckdb "SELECT * FROM generate_series(1, 10)"})
+
+;; UDTF queries mixed with real tables - tests that we correctly identify real tables
+;; while gracefully handling table functions
+(def udtf-with-table-queries
+ {:postgres "SELECT o.* FROM orders o, generate_series(1, 10) g"
+ :mysql "SELECT o.* FROM orders o, my_table_func(1, 10) t"
+ :snowflake "SELECT o.* FROM orders o, TABLE(FLATTEN(input => ARRAY_CONSTRUCT(1))) f"
+ :bigquery "SELECT o.* FROM orders o, UNNEST([1]) AS val"
+ :redshift "SELECT o.* FROM orders o, generate_series(1, 10) g"
+ :duckdb "SELECT o.* FROM orders o, generate_series(1, 10) g"})
+
+(deftest ^:parallel udtf-referenced-tables-test
+ (testing "UDTFs are handled by referenced-tables across dialects"
+ (doseq [[dialect sql] udtf-queries]
+ (testing (str "dialect: " dialect " - pure UDTF query")
+ ;; Table functions shouldn't appear as referenced tables - they're not real tables
+ (let [result (sql-parsing/referenced-tables (name dialect) sql)]
+ (is (vector? result)
+ (format "dialect %s: should return vector, got %s" dialect (type result)))
+ (is (every? vector? result)
+ (format "dialect %s: each element should be [catalog schema table] tuple" dialect)))))
+
+ (doseq [[dialect sql] udtf-with-table-queries]
+ (testing (str "dialect: " dialect " - UDTF mixed with real table")
+ (let [result (sql-parsing/referenced-tables (name dialect) sql)]
+ ;; Case-insensitive comparison since dialects like Snowflake uppercase identifiers
+ ;; 3-tuple: [catalog schema table] - table is third element
+ (is (some #(= "orders" (u/lower-case-en (nth % 2))) result)
+ (format "dialect %s: should find 'orders' table in %s" dialect (pr-str result))))))))
+
+;;; ------------------------------------------ Set Operation Tests (UNION, INTERSECT, EXCEPT) ------------------------------------------
+
+;; Set operation queries by dialect - tests UNION, UNION ALL, INTERSECT, EXCEPT
+;; All queries use tables "a" and "b" for consistent expected results
+(def dialect->set-operation->query
+ {:postgres
+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"
+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"
+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"
+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"
+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"
+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}
+
+ :mysql
+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"
+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"
+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"
+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"
+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"
+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}
+
+ :snowflake
+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"
+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"
+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"
+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"
+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"
+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}
+
+ :bigquery
+ {:union "SELECT id, name FROM a UNION DISTINCT SELECT id, name FROM b"
+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"
+ :intersect "SELECT id FROM a INTERSECT DISTINCT SELECT id FROM b"
+ :except "SELECT id FROM a EXCEPT DISTINCT SELECT id FROM b"
+ :nested-union "SELECT * FROM (SELECT id FROM a UNION DISTINCT SELECT id FROM b) AS combined"
+ :cte-with-union "WITH c AS (SELECT id FROM a UNION DISTINCT SELECT id FROM b) SELECT * FROM c"}
+
+ :redshift
+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"
+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"
+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"
+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"
+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"
+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}
+
+ :duckdb
+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"
+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"
+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"
+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"
+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"
+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}})
+
+(deftest ^:parallel set-operation-referenced-tables-test
+ (testing "Set operations correctly identify all referenced tables across dialects"
+ (doseq [[dialect ops] dialect->set-operation->query
+ [op-type sql] ops]
+ (testing (str "dialect: " (name dialect) " - " (name op-type))
+ (let [result (sql-parsing/referenced-tables (name dialect) sql)
+ ;; Normalize to lowercase for case-insensitive comparison (Snowflake uppercases)
+ ;; 3-tuple: [catalog schema table]
+ normalized (into #{} (map (fn [[catalog schema table]]
+ [(some-> catalog u/lower-case-en)
+ (some-> schema u/lower-case-en)
+ (u/lower-case-en table)])
+ result))]
+ (is (= #{[nil nil "a"] [nil nil "b"]} normalized)
+ (format "%s/%s should return [[nil nil a] [nil nil b]], got %s"
+ (name dialect) (name op-type) normalized)))))))
+
+(def ^:private set-operation->returned-columns
+ {:union ["id" "name"]
+ :union-all ["id" "name"]
+ :intersect ["id"]
+ :except ["id"]
+ :nested-union ["id"]
+ :cte-with-union ["id"]})
+
+(deftest ^:parallel set-operation-validate-query-test
+ (testing "Set operations validate correctly across dialects"
+ (doseq [[dialect ops] dialect->set-operation->query
+ [op-type sql] ops]
+ (testing (str "dialect: " (name dialect) " - " (name op-type))
+ (let [result (sql-parsing/validate-query (name dialect) sql "public" {})]
+ (is (= "ok" (:status result))
+ (format "%s/%s should validate OK, got %s"
+ (name dialect) (name op-type) result)))))))
+
+(deftest ^:parallel set-operation-returned-columns-lineage-test
+ (testing "Set operations return correct column lineage across dialects"
+ (doseq [[dialect ops] dialect->set-operation->query
+ [op-type sql] ops
+ :let [expected (get set-operation->returned-columns op-type)]]
+ (testing (str "dialect: " (name dialect) ", operation: " (name op-type))
+ (let [result (sql-parsing/returned-columns-lineage (name dialect) sql "public" {})
+ columns (sort (map (comp u/lower-case-en first) result))]
+ (is (= expected columns)
+ (format "%s %s should return %s, got %s"
+ (name dialect) (name op-type) (pr-str expected) columns)))))))
+
+;;; -------------------------------------------- Schema Validation Tests (validate-query) ------------------------------------------------
+
+;; validate-query validates SQL against a provided schema, detecting:
+;; - Missing columns (column doesn't exist in any table)
+;; - Missing table aliases (table wildcard references non-existent source)
+;; - Unknown tables (table not in schema)
+
+(def ^:private test-schema
+ "Test schema for validation tests.
+ Structure: {schema-name {table-name {column-name type}}}"
+ {"PUBLIC" {"PRODUCTS" {"ID" "INT"
+ "TITLE" "VARCHAR"
+ "CATEGORY" "VARCHAR"
+ "PRICE" "DECIMAL"}
+ "ORDERS" {"ID" "INT"
+ "USER_ID" "INT"
+ "PRODUCT_ID" "INT"
+ "TOTAL" "DECIMAL"}
+ "USERS" {"ID" "INT"
+ "NAME" "VARCHAR"
+ "EMAIL" "VARCHAR"}}})
+
+(deftest ^:parallel validate-query-valid-queries-test
+ (testing "Valid queries against schema"
+ (testing "simple select with existing columns"
+ (is (= "ok" (:status (sql-parsing/validate-query nil "SELECT id, title FROM products" "PUBLIC" test-schema)))))
+
+ (testing "wildcard select"
+ (is (= "ok" (:status (sql-parsing/validate-query nil "SELECT * FROM products" "PUBLIC" test-schema)))))
+
+ (testing "table-qualified columns"
+ (is (= "ok" (:status (sql-parsing/validate-query nil "SELECT products.id, products.title FROM products" "PUBLIC" test-schema)))))
+
+ (testing "join with valid columns"
+ (is (= "ok" (:status (sql-parsing/validate-query nil
+ "SELECT o.id, p.title FROM orders o JOIN products p ON o.product_id = p.id"
+ "PUBLIC" test-schema)))))
+
+ (testing "subquery"
+ (is (= "ok" (:status (sql-parsing/validate-query nil
+ "SELECT * FROM (SELECT id, title FROM products) AS sub"
+ "PUBLIC" test-schema)))))))
+
+(deftest ^:parallel validate-query-missing-column-test
+ (testing "Missing column errors"
+ (testing "non-existent column"
+ (let [result (sql-parsing/validate-query nil "SELECT bad_column FROM products" "PUBLIC" test-schema)]
+ (is (= "error" (:status result)))
+ (is (= "column_not_resolved" (:type result)))
+ (is (re-find #"(?i)bad_column" (:column result)))))
+
+ (testing "column from wrong table"
+ (let [result (sql-parsing/validate-query nil "SELECT email FROM products" "PUBLIC" test-schema)]
+ (is (= "error" (:status result)))
+ (is (= "column_not_resolved" (:type result)))))))
+
+(deftest ^:parallel validate-query-missing-table-alias-test
+ (testing "Missing table alias errors"
+ (testing "table wildcard with non-existent table"
+ (let [result (sql-parsing/validate-query nil "SELECT products.* FROM orders" "PUBLIC" test-schema)]
+ (is (= "error" (:status result)))
+ ;; SQLGlot reports this as unknown_table
+ (is (contains? #{"unknown_table" "column_not_resolved"} (:type result)))))
+
+ (testing "qualified column with non-existent alias"
+ (let [result (sql-parsing/validate-query nil "SELECT p.id FROM products" "PUBLIC" test-schema)]
+ (is (= "error" (:status result)))))))
+
+(deftest ^:parallel validate-query-unknown-table-test
+ ;; SQLGlot's qualify.qualify() does NOT detect unknown tables - it just doesn't qualify them.
+ ;; Unknown table detection happens at the sql-tools layer by matching referenced tables
+ ;; against Metabase's database metadata.
+ (testing "Unknown tables are not detected by SQLGlot (known limitation)"
+ (let [result (sql-parsing/validate-query nil "SELECT * FROM nonexistent" "PUBLIC" test-schema)]
+ (is (= "ok" (:status result))
+ "SQLGlot passes unknown tables through without error"))))
+
+(deftest ^:parallel validate-query-syntax-error-test
+ (testing "Syntax errors are caught"
+ (let [result (sql-parsing/validate-query nil "SELECT * FORM products" "PUBLIC" test-schema)]
+ (is (= "error" (:status result)))
+ (is (= "invalid_expression" (:type result))))))
+
+(deftest ^:parallel lenient-parsing-incomplete-limit-test
+ (testing "SQLGlot's lenient parsing treats 'SELECT 1 LIMIT' as 'SELECT 1 AS LIMIT'"
+ ;; This documents a known SQLGlot behavior where incomplete LIMIT clauses
+ ;; are parsed as column aliases.
+ (let [result (sql-parsing/referenced-tables "postgres" "SELECT 1 LIMIT")]
+ (is (= [] result)
+ "No tables are referenced in 'SELECT 1 LIMIT'")))
+
+ (testing "To reliably trigger SQL errors, use nonexistent tables instead"
+ ;; This is the recommended pattern for tests that need to trigger SQL failures
+ (let [result (sql-parsing/referenced-tables "postgres" "SELECT * FROM nonexistent_table_xyz")]
+ (is (= [[nil nil "nonexistent_table_xyz"]] result)
+ "Nonexistent table reference is parsed and will fail at execution time"))))
+
+(comment
+ (require '[clojure.string :as str])
+ (def query-corpus-path "/Users/bcm/dv/mb/query_corpus/")
+
+ (def sentinel (re-pattern "\n-----end-query-----\n"))
+
+ (def drivers ["mysql"])
+
+ (let [driver (first drivers)
+ corpus (slurp (str query-corpus-path driver ".log"))
+ queries (sort (distinct (str/split corpus sentinel)))]
+
+ (frequencies
+ (doall
+ (for [q queries]
+ (try
+ (sql-parsing/referenced-tables q driver)
+ true
+ (catch Exception _ false))))))
+
+ ;; Load and run validation tests interactively
+ (load-validation-test-cases)
+
+ ;; Test a single case
+ (validate-test-case {:name "test"
+ :sql "SELECT * FROM users"
+ :dialect "postgres"
+ :expected :valid})
+
+ ;; Run all valid query tests
+ (doseq [tc (:valid-queries (load-validation-test-cases))]
+ (let [result (validate-test-case tc)]
+ (when-not (:passed result)
+ result)))
+
+ ;; Run all invalid query tests
+ (doseq [tc (:invalid-queries (load-validation-test-cases))]
+ (let [result (validate-test-case tc)]
+ (when-not (:passed result)
+ result))))
+
+;;; -------------------------------------------- Referenced Fields Tests --------------------------------------------
+
+(defn- load-referenced-fields-test-cases
+ "Load referenced fields test cases from EDN file."
+ []
+ (let [resource (io/resource "metabase/sql_parsing/referenced_fields_test_cases.edn")]
+ (when-not resource
+ (throw (ex-info "Could not find referenced_fields_test_cases.edn" {})))
+ (read-string (slurp resource))))
+
+(defn- normalize-fields
+ "Normalize field references for comparison (sort and ensure consistent format).
+ Handles both 2-tuples (expected format in test cases) and 4-tuples (actual API format)."
+ [fields]
+ (vec (sort-by (fn [f]
+ (if (= 4 (count f))
+ ;; 4-tuple: [catalog schema table field]
+ [(u/lower-case-en (nth f 2)) (u/lower-case-en (nth f 3))]
+ ;; 2-tuple: [table field] (test case format)
+ [(u/lower-case-en (first f)) (u/lower-case-en (second f))]))
+ (map (fn [f]
+ (if (= 4 (count f))
+ ;; Convert 4-tuple to 2-tuple for comparison
+ [(u/lower-case-en (nth f 2)) (u/lower-case-en (nth f 3))]
+ ;; Already 2-tuple
+ [(u/lower-case-en (first f)) (u/lower-case-en (second f))]))
+ fields))))
+
+(defn- fields-match?
+ "Check if actual fields match expected fields (case-insensitive, order-independent)."
+ [expected actual]
+ (= (normalize-fields expected)
+ (normalize-fields actual)))
+
+(deftest ^:parallel referenced-fields-test
+ (testing "Referenced fields extraction from SQL queries"
+ (let [test-cases (:test-cases (load-referenced-fields-test-cases))]
+ (doseq [test-case test-cases]
+ (testing (:name test-case)
+ (let [{:keys [sql dialect expected]} test-case
+ actual (sql-parsing/referenced-fields dialect sql)]
+ (is (fields-match? expected actual)
+ (str "Test '" (:name test-case) "' failed"
+ "\n SQL: " sql
+ "\n Expected: " (pr-str (normalize-fields expected))
+ "\n Actual: " (pr-str (normalize-fields actual))))))))))
+
+(deftest ^:parallel referenced-fields-dialect-support-test
+ (testing "Referenced fields works across different SQL dialects"
+ (doseq [dialect ["postgres" "mysql" "snowflake" "bigquery" "redshift" "duckdb"]]
+ (testing (str "dialect: " dialect)
+ (let [result (sql-parsing/referenced-fields dialect "SELECT id, name FROM users WHERE active = true")]
+ (is (seq result)
+ (str "Should return fields for " dialect))
+ (is (every? #(and (vector? %) (= 4 (count %))) result)
+ (str "Each field should be [catalog schema table field] tuple for " dialect)))))))
+
+(deftest ^:parallel referenced-fields-wildcard-test
+ (testing "Wildcard handling in referenced fields"
+ (testing "Unqualified wildcard - single table"
+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT * FROM users")]
+ (is (fields-match? [["users" "*"]] result))))
+
+ (testing "Unqualified wildcard - multiple tables"
+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT * FROM users u LEFT JOIN orders o ON u.id = o.user_id")]
+ (is (some #(= ["orders" "*"] %) (normalize-fields result))
+ "Should include orders wildcard")
+ (is (some #(= ["users" "*"] %) (normalize-fields result))
+ "Should include users wildcard")))
+
+ (testing "Qualified wildcard"
+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT u.* FROM users u")]
+ (is (fields-match? [["users" "*"]] result))))
+
+ (testing "Mixed wildcards and specific columns"
+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT u.*, t.total FROM users u, transactions t WHERE u.id = t.user_id")]
+ (is (some #(= ["users" "*"] %) (normalize-fields result))
+ "Should include users wildcard")
+ (is (some #(= ["transactions" "total"] %) (normalize-fields result))
+ "Should include transactions.total")))))
diff --git a/test/metabase/sql_parsing/pool_test.clj b/test/metabase/sql_parsing/pool_test.clj
new file mode 100644
index 00000000000..8d09a45451c
--- /dev/null
+++ b/test/metabase/sql_parsing/pool_test.clj
@@ -0,0 +1,163 @@
+(ns metabase.sql-parsing.pool-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.sql-parsing.pool :as pool])
+ (:import
+ (io.aleph.dirigiste Pool)
+ (java.util.concurrent CountDownLatch)))
+
+(set! *warn-on-reflection* true)
+
+(defn- mock-context
+ "Create a mock 'context' object. Since Context is final, we just return a map with metadata.
+ The pool doesn't care what type the objects are - it just stores and returns them."
+ [id]
+ {:context-id id
+ :created-at (System/currentTimeMillis)})
+
+(defn- counting-generator
+ "Returns a generator function that counts how many contexts it has created."
+ [counter-atom]
+ (fn []
+ (let [id (swap! counter-atom inc)]
+ (mock-context id))))
+
+(defn- failing-generator
+ "Returns a generator that fails after creating n contexts."
+ [counter-atom fail-after]
+ (fn []
+ (let [id (swap! counter-atom inc)]
+ (when (> id fail-after)
+ (throw (ex-info "Generator limit reached" {:count id :limit fail-after})))
+ (mock-context id))))
+
+;;; ------------------------------------------------- Test Helpers ---------------------------------------------------
+
+(defn- test-pool
+ "Create a pool for testing with sensible defaults.
+ Returns [pool created-count-atom]."
+ ([]
+ (test-pool {}))
+ ([config]
+ (let [counter (atom 0)
+ generator (counting-generator counter)
+ pool (#'pool/make-python-context-pool generator config)]
+ [pool counter])))
+
+(defn- with-pool
+ "Execute f with a context from pool. Uses with-open for proper resource management."
+ [pool f]
+ (with-open [^java.io.Closeable ctx (#'pool/acquire-context pool)]
+ (f ctx)))
+
+(defn- run-concurrent
+ "Run n threads concurrently, each executing f with a context from pool.
+ Returns vector of results."
+ [pool n f]
+ (let [latch (CountDownLatch. n)
+ results (atom [])
+ futures (doall
+ (for [i (range n)]
+ (future
+ (.countDown latch)
+ (.await latch)
+ (let [result (with-pool pool (fn [ctx] (f i ctx)))]
+ (swap! results conj result)))))]
+ (doseq [fut futures] @fut)
+ @results))
+
+;;; ------------------------------------------------ Pool Size Tests -------------------------------------------------
+
+(deftest pool-respects-maximum-size-test
+ (testing "Pool never creates more than max contexts even under concurrent load"
+ (let [[pool created-count] (test-pool {:max-size 3})
+ num-threads 20
+ results (run-concurrent pool num-threads
+ (fn [_i ctx]
+ (Thread/sleep (long (+ 10 (rand-int 50))))
+ ctx))]
+
+ (is (<= @created-count 3)
+ (str "Pool created " @created-count " contexts but max is 3"))
+ (is (= num-threads (count results))
+ "All threads should have received a context"))))
+
+(deftest pool-maintains-minimum-size-test
+ (testing "Pool keeps at least 1 context available"
+ (let [[pool created-count] (test-pool)]
+ (with-pool pool (fn [_ctx] (is (= 1 @created-count) "First use creates one context")))
+ (Thread/sleep (long 100))
+ (with-pool pool (fn [_ctx] (is (= 1 @created-count) "Second use reuses the same context"))))))
+
+;;; ----------------------------------------------- Context Expiry Tests ---------------------------------------------
+
+(deftest pool-expires-old-contexts-test
+ (testing "Pool disposes of contexts that have exceeded their TTL"
+ (let [[pool _created-count] (test-pool {:ttl-minutes 10})
+ [context expiry-ts :as tuple] (.acquire ^Pool pool :python)]
+ (try
+ (is (some? context) "Context should be created")
+ (is (number? expiry-ts) "Expiry timestamp should be set")
+ (is (< (System/nanoTime) expiry-ts) "Expiry should be in the future")
+
+ (let [expiry-in-minutes (/ (- expiry-ts (System/nanoTime)) (* 1000000000 60))]
+ (is (< 9 expiry-in-minutes 11) "Expiry should be approximately 10 minutes"))
+ (finally
+ (.release ^Pool pool :python tuple))))))
+
+(deftest with-pooled-context-handles-expired-contexts-test
+ (testing "with-pooled-context properly handles and replaces expired contexts"
+ (let [[pool created-count] (test-pool)]
+ (with-pool pool (fn [_ctx] (is (= 1 @created-count))))
+
+ ;; Manually expire by disposing
+ (let [tuple (.acquire ^Pool pool :python)]
+ (.dispose ^Pool pool :python tuple))
+
+ (with-pool pool (fn [_ctx] (is (= 2 @created-count) "Expired context should be replaced"))))))
+
+;;; --------------------------------------------- Generator Failure Tests --------------------------------------------
+
+(deftest pool-handles-generator-failures-test
+ (testing "Pool gracefully handles when generator fails"
+ (let [counter (atom 0)
+ generator (failing-generator counter 1)
+ pool (#'pool/make-python-context-pool generator)]
+
+ (is (some? (with-pool pool identity)) "First context creation succeeds")
+
+ ;; Dispose to force new creation
+ (let [tuple (.acquire ^Pool pool :python)]
+ (.dispose ^Pool pool :python tuple))
+
+ (is (thrown? Exception (with-pool pool identity)) "Generator failure should propagate"))))
+
+;;; ------------------------------------------- Concurrent Access Tests ----------------------------------------------
+
+(deftest pool-handles-concurrent-access-safely-test
+ (testing "Pool handles concurrent access without race conditions"
+ (let [[pool created-count] (test-pool {:max-size 3})
+ num-threads 50
+ results (run-concurrent pool num-threads
+ (fn [i _ctx]
+ (Thread/sleep (long (rand-int 20)))
+ i))]
+
+ (is (= num-threads (count results)) "All threads should complete successfully")
+ (is (<= @created-count 3) (str "Created " @created-count " contexts, expected <= 3")))))
+
+;;; -------------------------------------------- Pool Behavior Tests -------------------------------------------------
+
+(deftest pool-stores-and-returns-objects-test
+ (testing "Pool correctly stores and returns arbitrary objects"
+ (let [[pool created-count] (test-pool)
+ result1 (with-pool pool :context)
+ result2 (with-pool pool :context)]
+
+ (is (map? result1))
+ (is (= 1 (:context-id result1)))
+ (is (= 1 @created-count))
+
+ (is (map? result2))
+ (is (= 1 (:context-id result2))) ;; Same context ID
+ (is (= 1 @created-count)))))
diff --git a/test/metabase/sql_parsing/referenced_fields_test_cases.edn b/test/metabase/sql_parsing/referenced_fields_test_cases.edn
new file mode 100644
index 00000000000..6e35418a9d9
--- /dev/null
+++ b/test/metabase/sql_parsing/referenced_fields_test_cases.edn
@@ -0,0 +1,244 @@
+;; Referenced Fields Test Cases
+;; Each test case has:
+;; - :name - descriptive name for the test
+;; - :sql - the SQL query to analyze
+;; - :dialect - SQL dialect (postgres, mysql, snowflake, bigquery, redshift, duckdb)
+;; - :expected - vector of [table field] pairs (sorted)
+
+{:test-cases
+ [;; Basic SELECT queries
+ {:name "simple-select-with-columns"
+ :sql "SELECT id, name FROM users"
+ :dialect "postgres"
+ :expected [["users" "id"] ["users" "name"]]}
+
+ {:name "select-with-where-clause"
+ :sql "SELECT id, name FROM users WHERE active = true"
+ :dialect "postgres"
+ :expected [["users" "active"] ["users" "id"] ["users" "name"]]}
+
+ {:name "select-with-multiple-where-conditions"
+ :sql "SELECT id, name FROM users WHERE active = true AND age > 18"
+ :dialect "postgres"
+ :expected [["users" "active"] ["users" "age"] ["users" "id"] ["users" "name"]]}
+
+ {:name "select-with-order-by"
+ :sql "SELECT id, name FROM users ORDER BY created_at DESC"
+ :dialect "postgres"
+ :expected [["users" "created_at"] ["users" "id"] ["users" "name"]]}
+
+ {:name "select-with-limit"
+ :sql "SELECT id, name FROM users LIMIT 10"
+ :dialect "postgres"
+ :expected [["users" "id"] ["users" "name"]]}
+
+ ;; Wildcards
+ {:name "unqualified-wildcard-single-table"
+ :sql "SELECT * FROM users"
+ :dialect "postgres"
+ :expected [["users" "*"]]}
+
+ {:name "unqualified-wildcard-with-where"
+ :sql "SELECT * FROM users WHERE active = true"
+ :dialect "postgres"
+ :expected [["users" "*"] ["users" "active"]]}
+
+ {:name "qualified-wildcard"
+ :sql "SELECT u.* FROM users u"
+ :dialect "postgres"
+ :expected [["users" "*"]]}
+
+ {:name "qualified-wildcard-with-specific-column"
+ :sql "SELECT u.*, t.total FROM users u, transactions t WHERE u.id = t.user_id"
+ :dialect "postgres"
+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "*"] ["users" "id"]]}
+
+ ;; JOINs
+ {:name "inner-join-simple"
+ :sql "SELECT u.id, o.total FROM users u INNER JOIN orders o ON u.id = o.user_id"
+ :dialect "postgres"
+ :expected [["orders" "total"] ["orders" "user_id"] ["users" "id"]]}
+
+ {:name "left-join-with-aliases"
+ :sql "SELECT t.id, u.email FROM transactions t LEFT JOIN users u ON t.user_id = u.id"
+ :dialect "postgres"
+ :expected [["transactions" "id"] ["transactions" "user_id"] ["users" "email"] ["users" "id"]]}
+
+ {:name "multiple-joins"
+ :sql "SELECT u.name, o.total, p.name FROM users u LEFT JOIN orders o ON u.id = o.user_id LEFT JOIN products p ON o.product_id = p.id"
+ :dialect "postgres"
+ :expected [["orders" "product_id"] ["orders" "total"] ["orders" "user_id"] ["products" "id"] ["products" "name"] ["users" "id"] ["users" "name"]]}
+
+ {:name "join-with-wildcard"
+ :sql "SELECT * FROM users u LEFT JOIN transactions t ON u.id = t.user_id"
+ :dialect "postgres"
+ :expected [["transactions" "*"] ["transactions" "user_id"] ["users" "*"] ["users" "id"]]}
+
+ {:name "cross-join"
+ :sql "SELECT u.id, t.total FROM users u, transactions t WHERE u.id = t.user_id"
+ :dialect "postgres"
+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "id"]]}
+
+ ;; CTEs
+ ;; Note: CTEs create scopes, and sqlglot attributes columns to the CTE scope in some cases
+ {:name "simple-cte"
+ :sql "WITH active_users AS (SELECT id FROM users WHERE active) SELECT * FROM active_users"
+ :dialect "postgres"
+ :expected [["users" "active"] ["users" "id"]]}
+
+ {:name "cte-with-join"
+ :sql "WITH active AS (SELECT id FROM users WHERE active = true) SELECT a.id, t.total FROM active a JOIN transactions t ON a.id = t.user_id"
+ :dialect "postgres"
+ ;; Note: sqlglot sees columns from CTE join and attributes some to transactions scope
+ :expected [["transactions" "active"] ["transactions" "id"] ["transactions" "total"] ["transactions" "user_id"] ["users" "active"] ["users" "id"]]}
+
+ {:name "multiple-ctes"
+ :sql "WITH active AS (SELECT id FROM users WHERE active), recent AS (SELECT user_id, total FROM transactions WHERE created_at > '2024-01-01') SELECT * FROM active a JOIN recent r ON a.id = r.user_id"
+ :dialect "postgres"
+ :expected [["transactions" "created_at"] ["transactions" "total"] ["transactions" "user_id"] ["users" "active"] ["users" "id"]]}
+
+ ;; Subqueries
+ {:name "subquery-in-from"
+ :sql "SELECT * FROM (SELECT id, name FROM users WHERE active = true) AS active_users"
+ :dialect "postgres"
+ :expected [["users" "active"] ["users" "id"] ["users" "name"]]}
+
+ {:name "subquery-in-where"
+ :sql "SELECT id, name FROM users WHERE id IN (SELECT user_id FROM transactions WHERE total > 100)"
+ :dialect "postgres"
+ ;; Note: sqlglot attributes some subquery columns to outer scope
+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "id"] ["users" "name"] ["users" "total"] ["users" "user_id"]]}
+
+ {:name "correlated-subquery"
+ :sql "SELECT u.id, u.name FROM users u WHERE EXISTS (SELECT 1 FROM transactions t WHERE t.user_id = u.id)"
+ :dialect "postgres"
+ :expected [["transactions" "user_id"] ["users" "id"] ["users" "name"]]}
+
+ ;; Aggregations
+ ;; Note: The * in COUNT(*) is not a column reference, it's part of the aggregate function syntax
+ {:name "count-aggregate"
+ :sql "SELECT COUNT(*) FROM users"
+ :dialect "postgres"
+ :expected []}
+
+ {:name "group-by-simple"
+ :sql "SELECT country, COUNT(*) FROM users GROUP BY country"
+ :dialect "postgres"
+ :expected [["users" "country"]]}
+
+ {:name "group-by-with-having"
+ :sql "SELECT country, COUNT(*) as cnt FROM users GROUP BY country HAVING COUNT(*) > 10"
+ :dialect "postgres"
+ :expected [["users" "country"]]}
+
+ {:name "multiple-aggregates"
+ :sql "SELECT department, COUNT(*), AVG(salary), MAX(salary) FROM employees GROUP BY department"
+ :dialect "postgres"
+ :expected [["employees" "department"] ["employees" "salary"]]}
+
+ ;; Window functions
+ {:name "window-function-simple"
+ :sql "SELECT name, salary, RANK() OVER (ORDER BY salary DESC) FROM employees"
+ :dialect "postgres"
+ :expected [["employees" "name"] ["employees" "salary"]]}
+
+ {:name "window-with-partition"
+ :sql "SELECT name, department, salary, AVG(salary) OVER (PARTITION BY department) FROM employees"
+ :dialect "postgres"
+ :expected [["employees" "department"] ["employees" "name"] ["employees" "salary"]]}
+
+ ;; CASE expressions
+ {:name "case-expression"
+ :sql "SELECT name, CASE WHEN age < 18 THEN 'minor' ELSE 'adult' END AS age_group FROM users"
+ :dialect "postgres"
+ :expected [["users" "age"] ["users" "name"]]}
+
+ {:name "case-with-multiple-conditions"
+ :sql "SELECT name, CASE WHEN active = true AND age > 18 THEN 'active_adult' WHEN active = true THEN 'active_minor' ELSE 'inactive' END FROM users"
+ :dialect "postgres"
+ :expected [["users" "active"] ["users" "age"] ["users" "name"]]}
+
+ ;; Set operations
+ {:name "union-simple"
+ :sql "SELECT id, name FROM users UNION SELECT id, name FROM archived_users"
+ :dialect "postgres"
+ :expected [["archived_users" "id"] ["archived_users" "name"] ["users" "id"] ["users" "name"]]}
+
+ {:name "union-all"
+ :sql "SELECT id FROM active_users UNION ALL SELECT id FROM inactive_users"
+ :dialect "postgres"
+ :expected [["active_users" "id"] ["inactive_users" "id"]]}
+
+ ;; Schema-qualified tables
+ {:name "schema-qualified-single"
+ :sql "SELECT id, name FROM public.users"
+ :dialect "postgres"
+ :expected [["users" "id"] ["users" "name"]]}
+
+ {:name "schema-qualified-multiple"
+ :sql "SELECT u.id, o.total FROM public.users u JOIN sales.orders o ON u.id = o.user_id"
+ :dialect "postgres"
+ :expected [["orders" "total"] ["orders" "user_id"] ["users" "id"]]}
+
+ ;; Complex queries
+ {:name "complex-with-cte-join-subquery"
+ :sql "WITH ranked AS (SELECT id, name, RANK() OVER (ORDER BY created_at DESC) as rank FROM users WHERE active = true) SELECT r.id, r.name, t.total FROM ranked r JOIN (SELECT user_id, SUM(amount) as total FROM transactions GROUP BY user_id) t ON r.id = t.user_id WHERE r.rank <= 10"
+ :dialect "postgres"
+ :expected [["transactions" "amount"] ["transactions" "user_id"] ["users" "active"] ["users" "created_at"] ["users" "id"] ["users" "name"]]}
+
+ {:name "nested-subqueries"
+ :sql "SELECT id, name FROM users WHERE id IN (SELECT user_id FROM transactions WHERE total > (SELECT AVG(total) FROM transactions))"
+ :dialect "postgres"
+ ;; Note: sqlglot attributes some subquery columns to outer scope
+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "id"] ["users" "name"] ["users" "total"] ["users" "user_id"]]}
+
+ ;; Edge cases
+ {:name "table-alias-same-as-table-name"
+ :sql "SELECT users.id, users.name FROM users users"
+ :dialect "postgres"
+ :expected [["users" "id"] ["users" "name"]]}
+
+ {:name "no-from-clause"
+ :sql "SELECT 1 as one, 2 as two"
+ :dialect "postgres"
+ :expected []}
+
+ {:name "select-from-values"
+ :sql "SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS t(id, name)"
+ :dialect "postgres"
+ :expected []}
+
+ ;; Different dialects
+ {:name "mysql-backticks"
+ :sql "SELECT `id`, `name` FROM `users` WHERE `active` = 1"
+ :dialect "mysql"
+ :expected [["users" "active"] ["users" "id"] ["users" "name"]]}
+
+ {:name "bigquery-struct"
+ :sql "SELECT id, name, address.city FROM users"
+ :dialect "bigquery"
+ ;; Note: Struct field access (address.city) is not parsed as a column reference
+ :expected [["users" "id"] ["users" "name"]]}
+
+ {:name "snowflake-uppercase"
+ :sql "SELECT ID, NAME FROM USERS"
+ :dialect "snowflake"
+ :expected [["USERS" "ID"] ["USERS" "NAME"]]}
+
+ ;; INSERT/UPDATE/DELETE (less common but valid)
+ {:name "insert-with-select"
+ :sql "INSERT INTO archive_users SELECT id, name, email FROM users WHERE active = false"
+ :dialect "postgres"
+ :expected [["users" "active"] ["users" "email"] ["users" "id"] ["users" "name"]]}
+
+ {:name "update-with-join"
+ :sql "UPDATE users SET last_purchase = t.max_date FROM (SELECT user_id, MAX(created_at) as max_date FROM transactions GROUP BY user_id) t WHERE users.id = t.user_id"
+ :dialect "postgres"
+ ;; Note: UPDATE statements may not be fully parsed by sqlglot for field extraction
+ :expected []}
+
+ {:name "delete-with-subquery"
+ :sql "DELETE FROM users WHERE id IN (SELECT user_id FROM transactions WHERE total < 10)"
+ :dialect "postgres"
+ ;; Note: DELETE statements don't include the target table's columns in references
+ :expected [["transactions" "total"] ["transactions" "user_id"]]}]}
diff --git a/test/metabase/sql_tools/core_test.clj b/test/metabase/sql_tools/core_test.clj
new file mode 100644
index 00000000000..11fa216c55d
--- /dev/null
+++ b/test/metabase/sql_tools/core_test.clj
@@ -0,0 +1,91 @@
+(ns metabase.sql-tools.core-test
+ "Tests for sql-tools that run against both :macaw and :sqlglot backends.
+
+ These tests verify that both parser implementations produce compatible results
+ for common operations, ensuring we can switch backends without breaking the app."
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.driver :as driver]
+ [metabase.lib.core :as lib]
+ [metabase.sql-tools.core :as sql-tools]
+ [metabase.sql-tools.test-util :as sql-tools.tu]
+ [metabase.test :as mt]))
+
+;;; ------------------------------------------------ validate-query ------------------------------------------------
+
+(deftest ^:parallel validate-query-syntax-error-test
+ (sql-tools.tu/test-parser-backends
+ (mt/test-driver :h2
+ (let [query (lib/native-query (mt/metadata-provider) "complete nonsense query")]
+ (testing "Gibberish SQL returns syntax error"
+ (is (= #{(lib/syntax-error)}
+ (sql-tools/validate-query driver/*driver* query))))))))
+
+(deftest ^:parallel validate-query-missing-column-test
+ (sql-tools.tu/test-parser-backends
+ (mt/test-driver :h2
+ (let [query (lib/native-query (mt/metadata-provider) "select nonexistent from orders")]
+ (testing "Reference to non-existent column returns missing-column error"
+ (is (= #{(lib/missing-column-error "NONEXISTENT")}
+ (sql-tools/validate-query driver/*driver* query))))))))
+
+(deftest ^:parallel validate-query-valid-test
+ (sql-tools.tu/test-parser-backends
+ (mt/test-driver :h2
+ (let [query (lib/native-query (mt/metadata-provider) "select id, total from orders")]
+ (testing "Valid query returns empty error set"
+ (is (= #{}
+ (sql-tools/validate-query driver/*driver* query))))))))
+
+;;; ---------------------------------------------- referenced-tables -----------------------------------------------
+
+(deftest ^:parallel referenced-tables-basic-test
+ (sql-tools.tu/test-parser-backends
+ (mt/test-driver :h2
+ (let [query (lib/native-query (mt/metadata-provider) "select id from orders")]
+ (testing "Single table reference"
+ (is (= #{{:table (mt/id :orders)}}
+ (sql-tools/referenced-tables driver/*driver* query))))))))
+
+(deftest ^:parallel referenced-tables-join-test
+ (sql-tools.tu/test-parser-backends
+ (mt/test-driver :h2
+ (let [query (lib/native-query (mt/metadata-provider)
+ "select o.id from orders o join products p on o.product_id = p.id")]
+ (testing "Join references both tables"
+ (is (= #{{:table (mt/id :orders)}
+ {:table (mt/id :products)}}
+ (sql-tools/referenced-tables driver/*driver* query))))))))
+
+;;; ------------------------------------------------ replace-names -------------------------------------------------
+
+(deftest ^:parallel replace-names-table-test
+ (sql-tools.tu/test-parser-backends
+ (testing "Basic table replacement"
+ (is (= "SELECT * FROM new_orders"
+ (sql-tools/replace-names :h2
+ "SELECT * FROM orders"
+ {:tables {{:table "orders"} "new_orders"}}))))))
+
+(deftest ^:parallel replace-names-schema-test
+ (sql-tools.tu/test-parser-backends
+ (testing "Schema replacement"
+ (is (= "SELECT * FROM new_schema.orders"
+ (sql-tools/replace-names :h2
+ "SELECT * FROM old_schema.orders"
+ {:schemas {"old_schema" "new_schema"}}))))))
+
+;;; -------------------------------------------- referenced-tables-raw ---------------------------------------------
+
+(deftest ^:parallel referenced-tables-raw-test
+ (sql-tools.tu/test-parser-backends
+ (testing "Returns table names without resolving to IDs"
+ ;; SQLGlot includes {:schema nil} while Macaw omits it - use =? for partial match
+ (is (=? [{:table "orders"}]
+ (sql-tools/referenced-tables-raw :h2 "SELECT * FROM orders"))))))
+
+(deftest ^:parallel referenced-tables-raw-with-schema-test
+ (sql-tools.tu/test-parser-backends
+ (testing "Includes schema when present"
+ (is (= [{:schema "public" :table "orders"}]
+ (sql-tools/referenced-tables-raw :postgres "SELECT * FROM public.orders"))))))
diff --git a/test/metabase/sql_tools/macaw/core_test.clj b/test/metabase/sql_tools/macaw/core_test.clj
new file mode 100644
index 00000000000..134247ce372
--- /dev/null
+++ b/test/metabase/sql_tools/macaw/core_test.clj
@@ -0,0 +1,39 @@
+(ns metabase.sql-tools.macaw.core-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ ;; Load implementations for multimethod registration
+ [metabase.sql-tools.init]
+ [metabase.sql-tools.interface :as sql-tools]
+ [metabase.sql-tools.macaw.core :as sql-tools.macaw]))
+
+(deftest ^:parallel split-compound-table-spec-test
+ (let [split sql-tools.macaw/split-compound-table-spec]
+ (testing "simple table name is unchanged"
+ (is (= {:table "orders" :schema nil}
+ (split {:table "orders" :schema nil}))))
+ (testing "two-part name splits into schema + table"
+ (is (= {:schema "analytics" :table "events"}
+ (split {:table "analytics.events" :schema nil}))))
+ (testing "three-part name takes last two as schema + table"
+ (is (= {:schema "analytics" :table "events"}
+ (split {:table "myproject.analytics.events" :schema nil}))))
+ (testing "already has schema — no splitting"
+ (is (= {:table "events" :schema "analytics"}
+ (split {:table "events" :schema "analytics"}))))
+ (testing "compound name with existing schema is not split"
+ (is (= {:table "analytics.events" :schema "other"}
+ (split {:table "analytics.events" :schema "other"}))))))
+
+(deftest ^:parallel referenced-tables-raw-bigquery-compound-names-test
+ (testing "Macaw referenced-tables-raw splits BigQuery-style dotted table names"
+ (is (= [{:schema "analytics" :table "events"}]
+ (sql-tools/referenced-tables-raw-impl :macaw :bigquery-cloud-sdk
+ "SELECT * FROM analytics.events")))
+ (is (= #{{:schema "analytics" :table "events"}
+ {:schema "analytics" :table "users"}}
+ (set (sql-tools/referenced-tables-raw-impl :macaw :bigquery-cloud-sdk
+ "SELECT * FROM analytics.events JOIN analytics.users ON events.user_id = users.id")))))
+ (testing "simple table names are unaffected"
+ (is (= [{:table "orders"}]
+ (sql-tools/referenced-tables-raw-impl :macaw :postgres
+ "SELECT * FROM orders")))))
diff --git a/test/metabase/driver/sql/references_test.clj b/test/metabase/sql_tools/macaw/references_test.clj
similarity index 99%
rename from test/metabase/driver/sql/references_test.clj
rename to test/metabase/sql_tools/macaw/references_test.clj
index 49e2e53fb27..44c382f2110 100644
--- a/test/metabase/driver/sql/references_test.clj
+++ b/test/metabase/sql_tools/macaw/references_test.clj
@@ -1,19 +1,19 @@
-(ns metabase.driver.sql.references-test
+(ns metabase.sql-tools.macaw.references-test
(:require
[clojure.test :refer :all]
[macaw.core :as macaw]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.sql.references :as sql.references]))
+ [metabase.lib.core :as lib]
+ [metabase.sql-tools.macaw.references :as sql-tools.macaw]))
;; TODO (Chris 2026-01-22) -- Refactor to use driver.u/parsed-query instead of macaw/parsed-query
#_{:clj-kondo/ignore [:discouraged-var]}
(defn- ->references [query]
- (->> query macaw/parsed-query macaw/->ast (sql.references/field-references :sql)))
+ (->> query macaw/parsed-query macaw/->ast (sql-tools.macaw/field-references :sql)))
(deftest ^:parallel garbage-test
(is (= {:used-fields #{}
:returned-fields []
- :errors #{(driver-api/syntax-error)}}
+ :errors #{(lib/syntax-error)}}
(->references "nothing"))))
(deftest ^:parallel basic-select-test
@@ -274,7 +274,7 @@
:source-columns [[{:type :all-columns,
:table {:table "orders"}}]]}},
:returned-fields []
- :errors #{(driver-api/missing-table-alias-error "foo")}}
+ :errors #{(lib/missing-table-alias-error "foo")}}
(->references "select foo.* from (select a from orders)"))))
(deftest ^:parallel bad-table-name-test
@@ -288,7 +288,7 @@
:alias nil,
:type :single-column,
:source-columns []}]
- :errors #{(driver-api/missing-table-alias-error "bad")}}
+ :errors #{(lib/missing-table-alias-error "bad")}}
(->references "select bad.a from products"))))
(deftest ^:parallel no-possible-sources-test
@@ -302,7 +302,7 @@
:alias nil
:type :single-column
:source-columns []}]
- :errors #{(driver-api/missing-column-error "a")}}
+ :errors #{(lib/missing-column-error "a")}}
(->references "select a"))))
(deftest ^:parallel select-constant-test
@@ -922,6 +922,7 @@ SELECT
c
FROM c;"))))
+;; composite field here
(deftest ^:parallel recursive-cte-test
(is (= {:used-fields
#{{:column "id",
diff --git a/test/metabase/sql_tools/macaw/replace_names_test.clj b/test/metabase/sql_tools/macaw/replace_names_test.clj
new file mode 100644
index 00000000000..9be389be6a7
--- /dev/null
+++ b/test/metabase/sql_tools/macaw/replace_names_test.clj
@@ -0,0 +1,112 @@
+(ns metabase.sql-tools.macaw.replace-names-test
+ "Tests for replace-names functionality via the sql-tools API.
+
+ Note: Macaw's replace-names uses a specific format for replacement maps:
+ - :tables - keys are {:schema s :table t} maps, values are new table names
+ - :columns - keys are {:schema s :table t :column c} maps, values are new column names
+ - :schemas - keys are schema name strings, values are new schema names"
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.sql-tools.core :as sql-tools]
+ ;; Load implementations for multimethod registration
+ [metabase.sql-tools.init]))
+
+;;; Test cases adapted from Macaw's test suite to verify the sql-tools wrapper.
+;;; Macaw keys use {:schema ... :table ... :column ...} maps, not plain strings.
+
+(deftest ^:parallel basic-table-rename-test
+ (testing "Basic table rename"
+ (is (= "SELECT * FROM users"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT * FROM people"
+ {:tables {{:table "people"} "users"}}
+ {})))))
+
+(deftest ^:parallel basic-column-rename-test
+ (testing "Basic column rename"
+ (is (= "SELECT user_id FROM orders"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT id FROM orders"
+ {:columns {{:table "orders" :column "id"} "user_id"}}
+ {})))))
+
+(deftest ^:parallel schema-rename-test
+ (testing "Schema rename"
+ (is (= "SELECT * FROM private.orders"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT * FROM public.orders"
+ {:schemas {"public" "private"}}
+ {})))))
+
+(deftest ^:parallel combined-renames-test
+ (testing "Combined schema, table, and column renames"
+ (is (= "SELECT amount FROM private.transactions"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT total FROM public.orders"
+ {:schemas {"public" "private"}
+ :tables {{:schema "public" :table "orders"} "transactions"}
+ :columns {{:schema "public" :table "orders" :column "total"} "amount"}}
+ {})))))
+
+(deftest ^:parallel aliased-table-test
+ (testing "Table with alias - alias is preserved, only table name changes"
+ ;; AS is optional in table aliases per SQL standard
+ (is (contains? #{"SELECT p.name FROM users p"
+ "SELECT p.name FROM users AS p"}
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT p.name FROM people p"
+ {:tables {{:table "people"} "users"}}
+ {})))))
+
+(deftest ^:parallel join-rename-test
+ (testing "Renames in JOIN queries - alias preserved"
+ ;; AS is optional in table aliases per SQL standard
+ (is (contains? #{"SELECT p.id, o.total FROM users p JOIN orders o ON p.id = o.user_id"
+ "SELECT p.id, o.total FROM users AS p JOIN orders AS o ON p.id = o.user_id"}
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT p.id, o.total FROM people p JOIN orders o ON p.id = o.user_id"
+ {:tables {{:table "people"} "users"}}
+ {})))))
+
+(deftest ^:parallel multiple-table-renames-test
+ (testing "Multiple table renames"
+ (is (= "SELECT * FROM users, transactions"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT * FROM people, orders"
+ {:tables {{:table "people"} "users"
+ {:table "orders"} "transactions"}}
+ {})))))
+
+(deftest ^:parallel qualified-column-rename-test
+ (testing "Qualified column in SELECT"
+ (is (= "SELECT orders.amount FROM orders"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT orders.total FROM orders"
+ {:columns {{:table "orders" :column "total"} "amount"}}
+ {})))))
+
+(deftest ^:parallel subquery-rename-test
+ (testing "Renames in subqueries"
+ ;; AS is optional in table aliases per SQL standard
+ (is (contains? #{"SELECT * FROM (SELECT id FROM users) sub"
+ "SELECT * FROM (SELECT id FROM users) AS sub"}
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT * FROM (SELECT id FROM people) sub"
+ {:tables {{:table "people"} "users"}}
+ {})))))
+
+(deftest ^:parallel case-sensitive-match-test
+ (testing "Case-sensitive matching - must match exact case"
+ ;; Without case-insensitive option, matches are exact
+ ;; PEOPLE won't match {:table "people"}
+ (is (= "SELECT * FROM users"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT * FROM people"
+ {:tables {{:table "people"} "users"}}
+ {})))
+ ;; Upper case needs upper case key
+ (is (= "SELECT * FROM users"
+ (sql-tools/replace-names-impl :macaw :postgres
+ "SELECT * FROM PEOPLE"
+ {:tables {{:table "PEOPLE"} "users"}}
+ {})))))
diff --git a/test/metabase/sql_tools/metrics_test.clj b/test/metabase/sql_tools/metrics_test.clj
new file mode 100644
index 00000000000..e305904d36e
--- /dev/null
+++ b/test/metabase/sql_tools/metrics_test.clj
@@ -0,0 +1,83 @@
+(ns metabase.sql-tools.metrics-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.analytics.prometheus-test :as prometheus-test]
+ [metabase.sql-tools.metrics :as metrics]
+ [metabase.test :as mt]))
+
+(set! *warn-on-reflection* true)
+
+(deftest with-operation-timing-success-test
+ (testing "with-operation-timing increments counters on success"
+ (mt/with-prometheus-system! [_ system]
+ (metrics/with-operation-timing [:macaw "returned-columns"]
+ :ok)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-total
+ {:parser "macaw" :operation "returned-columns"})))
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-completed
+ {:parser "macaw" :operation "returned-columns"})))
+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-sql-tools/operations-failed
+ {:parser "macaw" :operation "returned-columns"}))))))
+
+(deftest with-operation-timing-failure-test
+ (testing "with-operation-timing increments failure counter on exception"
+ (mt/with-prometheus-system! [_ system]
+ (is (thrown? Exception
+ (metrics/with-operation-timing [:sqlglot "validate-query"]
+ (throw (Exception. "boom")))))
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-total
+ {:parser "sqlglot" :operation "validate-query"})))
+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-sql-tools/operations-completed
+ {:parser "sqlglot" :operation "validate-query"})))
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-failed
+ {:parser "sqlglot" :operation "validate-query"}))))))
+
+(deftest with-operation-timing-returns-value-test
+ (testing "with-operation-timing returns the body's value"
+ (mt/with-prometheus-system! [_ _system]
+ (is (= :result
+ (metrics/with-operation-timing [:macaw "simple-query?"]
+ :result))))))
+
+(deftest with-operation-timing-duration-test
+ (testing "with-operation-timing records duration in histogram"
+ (mt/with-prometheus-system! [_ system]
+ (metrics/with-operation-timing [:macaw "replace-names"]
+ (Thread/sleep 10))
+ (let [histogram-val (mt/metric-value system :metabase-sql-tools/operation-duration-ms
+ {:parser "macaw" :operation "replace-names"})]
+ (is (pos? (:sum histogram-val)))))))
+
+(deftest record-functions-test
+ (testing "record-operation-start! increments operations-total"
+ (mt/with-prometheus-system! [_ system]
+ (metrics/record-operation-start! :macaw "field-references")
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-total
+ {:parser "macaw" :operation "field-references"})))))
+ (testing "record-operation-completion! increments completed and observes duration"
+ (mt/with-prometheus-system! [_ system]
+ (metrics/record-operation-completion! :sqlglot "referenced-tables" 42)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-completed
+ {:parser "sqlglot" :operation "referenced-tables"})))
+ (is (prometheus-test/approx= 42 (:sum (mt/metric-value system :metabase-sql-tools/operation-duration-ms
+ {:parser "sqlglot" :operation "referenced-tables"}))))))
+ (testing "record-operation-failure! increments failed and observes duration"
+ (mt/with-prometheus-system! [_ system]
+ (metrics/record-operation-failure! :macaw "add-into-clause" 100)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-failed
+ {:parser "macaw" :operation "add-into-clause"})))
+ (is (prometheus-test/approx= 100 (:sum (mt/metric-value system :metabase-sql-tools/operation-duration-ms
+ {:parser "macaw" :operation "add-into-clause"})))))))
+
+(deftest pool-metrics-registered-test
+ (testing "Pool metrics are registered in prometheus"
+ (mt/with-prometheus-system! [_ system]
+ (prometheus/inc! :metabase-sql-parsing/context-timeouts)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-timeouts)))
+ (prometheus/inc! :metabase-sql-parsing/context-acquisitions)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-acquisitions)))
+ (prometheus/inc! :metabase-sql-parsing/context-creations)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-creations)))
+ (prometheus/inc! :metabase-sql-parsing/context-disposals-expired)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-disposals-expired))))))
diff --git a/test/metabase/sql_tools/sqlglot/core_test.clj b/test/metabase/sql_tools/sqlglot/core_test.clj
new file mode 100644
index 00000000000..4a39acd3abc
--- /dev/null
+++ b/test/metabase/sql_tools/sqlglot/core_test.clj
@@ -0,0 +1,54 @@
+(ns metabase.sql-tools.sqlglot.core-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.driver :as driver]
+ [metabase.lib.core :as lib]
+ [metabase.sql-tools.core :as sql-tools]
+ [metabase.test :as mt]))
+
+;;;; validate-query
+
+(deftest validate-query-syntax-error-test
+ (mt/test-driver
+ :postgres
+ (let [mp (mt/metadata-provider)
+ query (lib/native-query mp "complete nonsense query")]
+ (testing "Gibberish SQL returns syntax error"
+ (is (= #{(lib/syntax-error)}
+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))
+
+(deftest validate-query-missing-table-alias-wildcard-test
+ (mt/test-driver
+ :postgres
+ (let [mp (mt/metadata-provider)
+ query (lib/native-query mp "select foo.* from (select id from orders)")]
+ (testing "Wildcard with unknown table alias returns missing-table-alias error"
+ (is (= #{(lib/missing-table-alias-error "foo")}
+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))
+
+(deftest validate-query-missing-table-alias-column-test
+ (mt/test-driver
+ :postgres
+ (let [mp (mt/metadata-provider)
+ query (lib/native-query mp "select bad.id from products")]
+ (sql-tools/validate-query-impl :sqlglot :postgres query)
+ (testing "Column with unknown table qualifier returns missing-table-alias error"
+ (sql-tools/validate-query-impl :sqlglot :postgres query)))))
+
+(deftest validate-query-missing-column-test
+ (mt/test-driver
+ :postgres
+ (let [mp (mt/metadata-provider)
+ query (lib/native-query mp "select nonexistent from orders")]
+ (testing "Reference to non-existent column returns missing-column error"
+ (is (= #{(lib/missing-column-error "nonexistent")}
+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))
+
+(deftest validate-query-valid-test
+ (mt/test-driver
+ :postgres
+ (let [mp (mt/metadata-provider)
+ query (lib/native-query mp "select id, total from orders")]
+ (testing "Valid query returns empty error set"
+ (is (= #{}
+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))
diff --git a/test/metabase/sql_tools/sqlglot/references_test.clj b/test/metabase/sql_tools/sqlglot/references_test.clj
new file mode 100644
index 00000000000..149f3820f21
--- /dev/null
+++ b/test/metabase/sql_tools/sqlglot/references_test.clj
@@ -0,0 +1,2295 @@
+;; SQLGlot version of references test
+(ns metabase.sql-tools.sqlglot.references-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib.core :as lib]
+ [metabase.sql-parsing.core :as sql-parsing]))
+
+(defn- ->references [query]
+ (sql-parsing/field-references "postgres" query))
+
+(deftest ^:parallel garbage-test
+ (is (= {:used-fields #{}
+ :returned-fields []
+ :errors #{(lib/syntax-error)}}
+ (->references "nothing"))))
+
+(deftest ^:parallel basic-select-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]
+ :errors #{}}
+ (->references "select * from (select a, b from products)"))))
+
+(deftest ^:parallel basic-join-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:type :all-columns, :table {:table "products"}}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]
+ :errors #{}}
+ (->references "select products.*, orders.id from products inner join orders on products.id = orders.product_id"))))
+
+(deftest ^:parallel indeterminite-join-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "orders"}}
+ {:type :all-columns, :table {:table "products"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "orders"}}
+ {:type :all-columns, :table {:table "products"}}]]}]
+ :errors #{}}
+ (->references "select id from products inner join orders on products.id = orders.product_id"))))
+
+(deftest ^:parallel table-wildcard-join-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:type :all-columns, :table {:table "orders"}}]
+ :errors #{}}
+ (->references "select orders.* from products inner join orders on products.id = orders.product_id"))))
+
+(deftest ^:parallel wildcard-join-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:type :all-columns, :table {:table "orders"}}
+ {:type :all-columns, :table {:table "products"}}]
+ :errors #{}}
+ (->references "select * from products inner join orders on products.id = orders.product_id"))))
+
+(deftest ^:parallel basic-alias-test
+ (is (= {:used-fields
+ #{{:column "c",
+ :alias "d",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}
+ {:column "a",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}
+ {:column "c",
+ :alias "d",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}]
+ :errors #{}}
+ (->references "select p.a as b, p.c as d from products p"))))
+
+(deftest ^:parallel basic-nested-query-test
+ (is (= {:used-fields
+ #{{:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}]
+ :errors #{}}
+ (->references "select a, b from (select a, b from products)"))))
+
+(deftest ^:parallel renamed-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "a",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}]
+ :errors #{}}
+ (->references "select b as c from (select a as b from products)"))))
+
+(deftest ^:parallel broken-nested-query-test
+ (is (= {:used-fields
+ #{{:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]]}]
+ :errors #{}}
+ (->references "select a from (select b from products)"))))
+
+(deftest ^:parallel different-case-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}]
+ :errors #{}}
+ (->references "select A from (select a from orders)"))))
+
+(deftest ^:parallel wildcard-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}]
+ :errors #{}}
+ (->references "select * from (select a from orders)"))))
+
+(deftest ^:parallel table-wildcard-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}]
+ :errors #{}}
+ (->references "select o.* from (select a from orders) o"))))
+
+(deftest ^:parallel bad-table-wildcard-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields []
+ :errors #{(lib/missing-table-alias-error "foo")}}
+ (->references "select foo.* from (select a from orders)"))))
+
+(deftest ^:parallel bad-table-name-test
+ (is (= #{(lib/missing-table-alias-error "bad")}
+ (:errors (->references "select bad.a from products")))))
+
+(deftest ^:parallel no-possible-sources-test
+ (is (= {:used-fields
+ #{{:column "a"
+ :alias nil
+ :type :single-column
+ :source-columns []}}
+ :returned-fields
+ [{:column "a"
+ :alias nil
+ :type :single-column
+ :source-columns []}]
+ :errors #{(lib/missing-column-error "a")}}
+ (->references "select a"))))
+
+(deftest ^:parallel select-constant-test
+ (is (= {:used-fields #{},
+ :returned-fields
+ [{:alias nil, :type :custom-field, :used-fields #{}}],
+ :errors #{}}
+ (->references "select 1"))))
+
+(deftest ^:parallel basic-where-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]
+ :errors #{}}
+ (->references "select * from products where category = 'hello'"))))
+
+(deftest ^:parallel basic-aggregation-test
+ (is (=? {:used-fields #{},
+ :returned-fields [{:alias nil, :type :custom-field, :used-fields #{}}]
+ :errors #{}}
+ (->references "select count(*) from products"))))
+
+(deftest ^:parallel named-aggregation-test
+ (is (=? {:used-fields #{},
+ :returned-fields [{:alias "count", :type :custom-field, :used-fields #{}}]
+ :errors #{}}
+ (->references "select count(*) as count from products"))))
+
+(deftest ^:parallel extra-names-test
+ (is (= {:used-fields
+ #{{:column "col",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:database "db",
+ :schema "schema",
+ :table "table"}}]]}},
+ :returned-fields
+ [{:column "col",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:database "db",
+ :schema "schema",
+ :table "table"}}]]}]
+ :errors #{}}
+ (->references "select db.schema.table.col from db.schema.table"))))
+
+(deftest ^:parallel basic-grouping-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:alias "sum",
+ :type :custom-field,
+ :used-fields
+ #{{:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:alias "sum",
+ :type :custom-field,
+ :used-fields
+ #{{:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]
+ :errors #{}}
+ (->references "select sum(total) as sum from orders group by category"))))
+
+(deftest ^:parallel grouping-order-by-test
+ (is (= {:used-fields
+ #{{:alias "cards_created",
+ :type :custom-field,
+ :used-fields
+ #{{:column "creator_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}
+ {:column "creator_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}
+ {:column "creator_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:alias "cards_created",
+ :type :custom-field,
+ :used-fields
+ #{{:column "creator_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}]
+ [{:type :all-columns, :table {:table "report_card"}}]]}},
+ :returned-fields
+ [{:column "creator_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}
+ {:alias "cards_created",
+ :type :custom-field,
+ :used-fields
+ #{{:column "creator_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}]
+ :errors #{}}
+ (->references "SELECT creator_id, COUNT(creator_id) AS cards_created FROM report_card GROUP BY creator_id ORDER BY cards_created DESC"))))
+
+(deftest ^:parallel basic-arg-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]
+ :errors #{}}
+ (->references "select * from products where category = ?"))))
+
+(deftest ^:parallel basic-case-test
+ (is (=? {:used-fields
+ #{{:column "tax",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "subtotal",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "tax",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "subtotal",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}}}]
+ :errors #{}}
+ (->references "select case when total < 0 then -subtotal else tax end from orders"))))
+
+(deftest ^:parallel switch-case-test
+ (is (=? {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields
+ [{:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}}}]
+ :errors #{}}
+ (->references "select case category when 'Gizmo' then 'is gizmo' else 'is not gizmo' end from products"))))
+
+(deftest ^:parallel basic-select-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]
+ :errors #{}}
+ (->references "select (select category from products where products.id = orders.product_id), * from orders"))))
+
+(deftest ^:parallel named-select-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias "category2",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]
+ :errors #{}}
+ (->references "select (select category from products where products.id = orders.product_id) as category2, * from orders"))))
+
+(deftest ^:parallel nested-select-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]
+ :errors #{}}
+ (->references "select (select (select category from products where products.id = orders.product_id)), * from orders"))))
+
+(deftest ^:parallel select-subquery-with-normal-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]
+ :errors #{}}
+ (->references "select (select * from (select category from products where products.id = orders.product_id) sub), * from orders"))))
+
+(deftest ^:parallel nested-select-subquery-with-reference-to-middle-select-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "products"}}]
+ [{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "products"}}]
+ [{:type :all-columns, :table {:table "orders"}}]]}]
+ :errors #{}}
+ (->references "select (select (select category) from products where products.id = orders.product_id) from orders"))))
+
+(deftest ^:parallel nested-select-subquery-with-direct-match-in-outer-query
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}
+ {:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "t1"}}]
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "t1"}}]
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}]
+ :errors #{}}
+ (->references "select (select a from t1) from (select a from t2)"))))
+
+(deftest ^:parallel basic-exists-test
+ (is (= {:used-fields
+ #{{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}
+ {:column "user_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "o", :table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}
+ {:column "email",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}},
+ :returned-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}
+ {:column "email",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}]
+ :errors #{}}
+ (->references "SELECT u.name, u.email
+FROM users u
+WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
+
+(deftest ^:parallel basic-not-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]
+ :errors #{}}
+ (->references "select * from products where not (category = 'Gizmo')"))))
+
+(deftest ^:parallel basic-is-null-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]
+ :errors #{}}
+ (->references "select * from products where category is null"))))
+
+(deftest ^:parallel negated-is-null-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]
+ :errors #{}}
+ (->references "select * from products where category is not null"))))
+
+(deftest ^:parallel basic-between-test
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "left",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "right",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields [{:type :all-columns, :table {:table "orders"}}]
+ :errors #{}}
+ (->references "select * from orders where created_at between left and right"))))
+
+(deftest ^:parallel basic-select-table-alias-test
+ (is (= {:used-fields #{},
+ :returned-fields
+ [{:type :custom-field,
+ :alias "o",
+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]
+ :errors #{}}
+ (->references "select o from (select * from orders) as o"))))
+
+(deftest ^:parallel select-table-alias-with-alias-test
+ (is (= {:used-fields #{},
+ :returned-fields
+ [{:type :custom-field,
+ :alias "o2",
+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]
+ :errors #{}}
+ (->references "select o as o2 from (select * from orders) as o"))))
+
+(deftest ^:parallel table-function-test
+ (is (= {:used-fields #{}, :returned-fields [{:type :unknown-columns}] :errors #{}}
+ (->references "select i.* from generate_series(1, 500) as i"))))
+
+(deftest ^:parallel basic-cte-test
+ (is (= {:used-fields
+ #{{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "active",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}},
+ :returned-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}]
+ :errors #{}}
+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)
+SELECT * FROM active_users"))))
+
+(deftest ^:parallel unused-cte-test
+ (is (= {:used-fields
+ #{{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "active",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]
+ :errors #{}}
+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)
+SELECT * FROM products"))))
+
+(deftest ^:parallel dependent-cte-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]
+ :errors #{}}
+ (->references "
+WITH orders_a AS (select * from orders),
+orders_b AS (select * from orders_a)
+SELECT id from orders_b"))))
+
+(deftest ^:parallel shadowed-cte-test
+ (is (= {:used-fields
+ #{{:column "y",
+ :alias "x",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "y",
+ :alias "a",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "z",
+ :alias "y",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "x",
+ :alias "z",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "z",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "x",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}},
+ :returned-fields
+ [{:column "y",
+ :alias "a",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "z",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "x",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}]
+ :errors #{}}
+ (->references "WITH c AS (
+ WITH b AS (
+ SELECT
+ x as z,
+ y as x,
+ z as y
+ FROM a
+ )
+ SELECT
+ x as a,
+ y as b,
+ z as c
+ FROM b
+)
+SELECT
+ a,
+ b,
+ c
+FROM c;"))))
+
+;; composite field here
+(deftest ^:parallel recursive-cte-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "h", :table "emp_hierarchy"}}]]}
+ {:alias "name",
+ :type :composite-field,
+ :member-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}]}
+ {:column "manager_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}
+ {:column "level",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "h",
+ :table "emp_hierarchy"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}
+ {:alias "level",
+ :type :composite-field,
+ :member-fields
+ [{:alias "level", :type :custom-field, :used-fields #{}}
+ {:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "level",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "h",
+ :table "emp_hierarchy"}}]]}}}]}
+ {:column "manager_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "manager_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:alias "level", :type :custom-field, :used-fields #{}}]
+ [{:type :all-columns, :table {:table "employees"}}]]}},
+ :returned-fields
+ [{:alias "name",
+ :type :composite-field,
+ :member-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}]}
+ {:alias "level",
+ :type :composite-field,
+ :member-fields
+ [{:alias "level", :type :custom-field, :used-fields #{}}
+ {:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "level",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "h",
+ :table "emp_hierarchy"}}]]}}}]}]
+ :errors #{}}
+ (->references "WITH RECURSIVE emp_hierarchy AS (
+ SELECT id, name, manager_id, 0 AS level
+ FROM employees
+ WHERE manager_id IS NULL
+ UNION ALL
+ SELECT e.id, e.name, e.manager_id, h.level + 1
+ FROM employees e
+ JOIN emp_hierarchy h ON e.manager_id = h.id
+)
+SELECT name, level FROM emp_hierarchy"))))
+
+(deftest ^:parallel basic-union-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}},
+ :returned-fields
+ [{:alias "id",
+ :type :composite-field,
+ :member-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}]}
+ {:alias "name",
+ :type :composite-field,
+ :member-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}]}]
+ :errors #{}}
+ (->references "SELECT id, name FROM users
+UNION
+SELECT id, name FROM archived_users"))))
+
+(deftest ^:parallel row-number-test
+ (is (= {:used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}},
+ :returned-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}
+ {:alias "rank",
+ :type :custom-field,
+ :used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}}}]
+ :errors #{}}
+ (->references "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) AS rank
+FROM employees"))))
+
+(deftest ^:parallel partition-by-test
+ (is (= {:used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "department",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}},
+ :returned-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:alias "dept_rank",
+ :type :custom-field,
+ :used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "department",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}}}]
+ :errors #{}}
+ (->references "SELECT name,
+ RANK() OVER (PARTITION BY department ORDER BY salary DESC) AS dept_rank
+FROM employees"))))
+
+(deftest ^:parallel week-test
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}},
+ :returned-fields
+ [{:alias "created_at",
+ :type :custom-field,
+ :used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "count", :type :custom-field, :used-fields #{}}]
+ :errors #{}}
+ (->references "SELECT
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ ) AS \"created_at\",
+ COUNT(*) AS \"count\"
+FROM
+ \"public\".\"orders\"
+GROUP BY
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ORDER BY
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ ) ASC"))))
+
+(deftest ^:parallel week-of-year-test
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}},
+ :returned-fields
+ [{:alias "created_at",
+ :type :custom-field,
+ :used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "count", :type :custom-field, :used-fields #{}}]
+ :errors #{}}
+ (->references "SELECT
+ CEIL(
+ (
+ CAST(
+ extract(
+ doy
+ from
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ ) AS integer
+ ) / 7.0
+ )
+ ) AS \"created_at\",
+ COUNT(*) AS \"count\"
+FROM
+ \"public\".\"orders\"
+GROUP BY
+ CEIL(
+ (
+ CAST(
+ extract(
+ doy
+ from
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ ) AS integer
+ ) / 7.0
+ )
+ )
+ORDER BY
+ CEIL(
+ (
+ CAST(
+ extract(
+ doy
+ from
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ ) AS integer
+ ) / 7.0
+ )
+ ) ASC"))))
+
+;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the
+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.
+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
+(deftest ^:parallel complicated-test-1
+ (is (= {:used-fields
+ #{{:column "title",
+ :alias "title",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "discount",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "rating",
+ :alias "rating",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:alias "upper_category",
+ :type :custom-field,
+ :used-fields
+ #{{:column "category",
+ :alias "category",
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}}}
+ {:column "ean",
+ :alias "ean",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "created_at",
+ :alias "created_at",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "product_id",
+ :alias "product_id",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "category",
+ :alias "Products__category",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "category",
+ :alias "category",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "created_at",
+ :alias "created_at",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "discount",
+ :alias "discount",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "vendor",
+ :alias "vendor",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "total",
+ :alias "total",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "id",
+ :alias "Products__id",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "id",
+ :alias "id",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "price",
+ :alias "price",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}},
+ :returned-fields
+ [{:alias "created_at",
+ :type :custom-field,
+ :used-fields
+ #{{:column "created_at",
+ :alias "created_at",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "upper_category",
+ :type :custom-field,
+ :used-fields
+ #{{:column "category",
+ :alias "category",
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}}}
+ {:alias "sum",
+ :type :custom-field,
+ :used-fields
+ #{{:column "total",
+ :alias "total",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "sum_2",
+ :type :custom-field,
+ :used-fields
+ #{{:column "discount",
+ :alias "discount",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "count", :type :custom-field, :used-fields #{}}]
+ :errors #{}}
+ (->references "SELECT
+ DATE_TRUNC('month', \"source\".\"created_at\") AS \"created_at\",
+ \"source\".\"upper_category\" AS \"upper_category\",
+ SUM(\"source\".\"total\") AS \"sum\",
+ SUM(\"source\".\"discount\") AS \"sum_2\",
+ COUNT(*) AS \"count\"
+FROM
+ (
+ SELECT
+ \"public\".\"orders\".\"product_id\" AS \"product_id\",
+ \"public\".\"orders\".\"total\" AS \"total\",
+ \"public\".\"orders\".\"discount\" AS \"discount\",
+ \"public\".\"orders\".\"created_at\" AS \"created_at\",
+ UPPER(\"Products\".\"category\") AS \"upper_category\",
+ \"Products\".\"id\" AS \"Products__id\",
+ \"Products\".\"category\" AS \"Products__category\"
+ FROM
+ \"public\".\"orders\"
+
+LEFT JOIN (
+ SELECT
+ \"public\".\"products\".\"id\" AS \"id\",
+ \"public\".\"products\".\"ean\" AS \"ean\",
+ \"public\".\"products\".\"title\" AS \"title\",
+ \"public\".\"products\".\"category\" AS \"category\",
+ \"public\".\"products\".\"vendor\" AS \"vendor\",
+ \"public\".\"products\".\"price\" AS \"price\",
+ \"public\".\"products\".\"rating\" AS \"rating\",
+ \"public\".\"products\".\"created_at\" AS \"created_at\"
+ FROM
+ \"public\".\"products\"
+ ) AS \"Products\" ON \"public\".\"orders\".\"product_id\" = \"Products\".\"id\"
+
+WHERE
+ \"public\".\"orders\".\"discount\" > 0
+ ) AS \"source\"
+GROUP BY
+ DATE_TRUNC('month', \"source\".\"created_at\"),
+ \"source\".\"upper_category\"
+ORDER BY
+ DATE_TRUNC('month', \"source\".\"created_at\") ASC,
+ \"source\".\"upper_category\" ASC"))))
+
+;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the
+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.
+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
+(deftest ^:parallel complicated-test-2
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "is_converted",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "annual_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "base_fee",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "account_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_alias",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_paid_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_active_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_active_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_paid_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_trialing_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "is_active_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "is_active_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_active_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_trialing_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:column "customer_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_trialing",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:column "is_active",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_paid_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_paid_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_paid_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}
+ {:column "valid_to",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table
+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "valid_from",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table
+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}},
+ :returned-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "customer_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "is_converted",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "is_active",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_alias",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "base_fee",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "annual_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}
+ {:alias "is_trialing",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_trialing_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_active_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_trialing_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_active_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_active_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_active_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}
+ {:alias "is_active_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "month",
+ :type :custom-field,
+ :used-fields #{}}}}]
+ :errors #{}}
+ (->references "with months as (
+ select
+ generate_series(date_trunc('month', '2024-01-01'::date),
+ date_trunc('month', current_date) - interval '1 month',
+ '1 month')::date as month
+
+)
+, monthly_account_subscription_status as (
+ select
+ a.id,
+ m.month,
+
+ max(case when ah.subscription_status is not null and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing,
+
+ -- cloud
+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_cloud,
+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' then 1 else 0 end) as is_active_accounts_cloud,
+ max(case when a.lifetime_value > 0 and a.deployment = 'cloud' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_cloud,
+
+ -- self-hosted
+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_self_hosted,
+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' then 1 else 0 end) as is_active_accounts_self_hosted,
+ max(case when a.lifetime_value > 0 and a.deployment = 'self-hosted' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_self_hosted,
+
+ -- enterprise
+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'enterprise' then 1 else 0 end) as is_active_accounts_enterprise,
+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'enterprise' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_enterprise,
+
+ -- starter
+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'starter' then 1 else 0 end) as is_active_accounts_starter,
+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'starter' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_starter,
+
+ -- pro
+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'pro' then 1 else 0 end) as is_active_accounts_pro,
+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'pro' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_pro
+ from dbt_models.account a
+ left join dbt_models.account_history ah
+ on a.id = ah.account_id
+ left join months m
+ on m.month between date_trunc('month', ah.valid_from) and date_trunc('month', ah.valid_to)
+ group by 1,2
+)
+select
+ a.id,
+ a.customer_name,
+ a.created_at,
+ a.is_converted,
+ a.lifetime_value,
+ a.is_active,
+ a.subscription_status,
+ a.deployment,
+ a.plan_name,
+ a.plan_alias,
+ a.base_fee,
+ a.annual_value,
+
+ -- monthly account status
+ mas.month,
+ mas.is_trialing,
+ mas.is_trialing_accounts_cloud,
+ mas.is_active_accounts_cloud,
+ mas.is_paid_accounts_cloud,
+ mas.is_trialing_accounts_self_hosted,
+ mas.is_active_accounts_self_hosted,
+ mas.is_paid_accounts_self_hosted,
+ mas.is_active_accounts_enterprise,
+ mas.is_paid_accounts_enterprise,
+ mas.is_active_accounts_starter,
+ mas.is_paid_accounts_starter,
+ mas.is_active_accounts_pro,
+ mas.is_paid_accounts_pro
+
+from dbt_models.account a
+left join monthly_account_subscription_status mas
+ on a.id = mas.id
+;"))))
diff --git a/test/metabase/sql_tools/sqlglot/replace_names_test.clj b/test/metabase/sql_tools/sqlglot/replace_names_test.clj
new file mode 100644
index 00000000000..4e350f991da
--- /dev/null
+++ b/test/metabase/sql_tools/sqlglot/replace_names_test.clj
@@ -0,0 +1,121 @@
+(ns metabase.sql-tools.sqlglot.replace-names-test
+ "Tests for replace-names functionality using SQLGlot backend.
+
+ These tests mirror the Macaw tests to ensure feature parity between backends.
+ Tests call sql-parsing/replace-names directly since it doesn't need Metabase metadata."
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.sql-parsing.core :as sql-parsing]))
+
+(defn- replace-names
+ "Helper that mirrors sql-tools/replace-names API but calls SQLGlot directly."
+ [driver sql-string replacements]
+ ;; Convert map keys to list-of-pairs for JSON serialization
+ (let [dialect (case driver
+ :postgres "postgres"
+ :mysql "mysql"
+ :sqlserver "tsql"
+ nil)
+ replacements' (-> replacements
+ (update :tables #(when % (vec %)))
+ (update :columns #(when % (vec %))))]
+ (sql-parsing/replace-names dialect sql-string replacements')))
+
+(deftest ^:parallel basic-table-rename-test
+ (testing "Basic table rename"
+ (is (= "SELECT * FROM users"
+ (replace-names :postgres
+ "SELECT * FROM people"
+ {:tables {{:table "people"} "users"}})))))
+
+(deftest ^:parallel basic-column-rename-test
+ (testing "Basic column rename"
+ (is (= "SELECT user_id FROM orders"
+ (replace-names :postgres
+ "SELECT id FROM orders"
+ {:columns {{:table "orders" :column "id"} "user_id"}})))))
+
+(deftest ^:parallel schema-rename-test
+ (testing "Schema rename"
+ (is (= "SELECT * FROM private.orders"
+ (replace-names :postgres
+ "SELECT * FROM public.orders"
+ {:schemas {"public" "private"}})))))
+
+(deftest ^:parallel combined-renames-test
+ (testing "Combined schema, table, and column renames"
+ (is (= "SELECT amount FROM private.transactions"
+ (replace-names :postgres
+ "SELECT total FROM public.orders"
+ {:schemas {"public" "private"}
+ :tables {{:schema "public" :table "orders"} "transactions"}
+ :columns {{:schema "public" :table "orders" :column "total"} "amount"}})))))
+
+(deftest ^:parallel aliased-table-test
+ (testing "Table with alias - alias is preserved, only table name changes"
+ (is (= "SELECT p.name FROM users AS p"
+ (replace-names :postgres
+ "SELECT p.name FROM people p"
+ {:tables {{:table "people"} "users"}})))))
+
+(deftest ^:parallel join-rename-test
+ (testing "Renames in JOIN queries - alias preserved"
+ (is (= "SELECT p.id, o.total FROM users AS p JOIN orders AS o ON p.id = o.user_id"
+ (replace-names :postgres
+ "SELECT p.id, o.total FROM people p JOIN orders o ON p.id = o.user_id"
+ {:tables {{:table "people"} "users"}})))))
+
+(deftest ^:parallel multiple-table-renames-test
+ (testing "Multiple table renames"
+ (is (= "SELECT * FROM users, transactions"
+ (replace-names :postgres
+ "SELECT * FROM people, orders"
+ {:tables {{:table "people"} "users"
+ {:table "orders"} "transactions"}})))))
+
+(deftest ^:parallel qualified-column-rename-test
+ (testing "Qualified column in SELECT"
+ (is (= "SELECT orders.amount FROM orders"
+ (replace-names :postgres
+ "SELECT orders.total FROM orders"
+ {:columns {{:table "orders" :column "total"} "amount"}})))))
+
+(deftest ^:parallel subquery-rename-test
+ (testing "Renames in subqueries"
+ (is (= "SELECT * FROM (SELECT id FROM users) AS sub"
+ (replace-names :postgres
+ "SELECT * FROM (SELECT id FROM people) sub"
+ {:tables {{:table "people"} "users"}})))))
+
+(deftest ^:parallel case-sensitive-match-test
+ (testing "Case-sensitive matching - must match exact case"
+ ;; Lower case matches lower case
+ (is (= "SELECT * FROM users"
+ (replace-names :postgres
+ "SELECT * FROM people"
+ {:tables {{:table "people"} "users"}})))
+ ;; Upper case needs upper case key
+ (is (= "SELECT * FROM users"
+ (replace-names :postgres
+ "SELECT * FROM PEOPLE"
+ {:tables {{:table "PEOPLE"} "users"}})))))
+
+(deftest ^:parallel table-rename-with-schema-map-value-test
+ (testing "Table rename using map value with schema and table (workspace isolation pattern)"
+ ;; This is the pattern used by workspaces to isolate tables into a new schema
+ (is (= "SELECT * FROM ws_isolated_123.public__orders"
+ (replace-names :postgres
+ "SELECT * FROM orders"
+ {:tables {{:table "orders"} {:schema "ws_isolated_123" :table "public__orders"}}})))
+ ;; Qualified source table
+ (is (= "SELECT * FROM ws_isolated_123.public__orders"
+ (replace-names :postgres
+ "SELECT * FROM public.orders"
+ {:tables {{:schema "public" :table "orders"} {:schema "ws_isolated_123" :table "public__orders"}}})))))
+
+(deftest ^:parallel table-rename-add-schema-only-test
+ (testing "Table rename that only adds schema (qualify unqualified reference)"
+ (is (= "SELECT * FROM public.orders"
+ (replace-names :postgres
+ "SELECT * FROM orders"
+ {:tables {{:table "orders"} {:schema "public" :table "orders"}}})))))
diff --git a/test/metabase/sql_tools/sqlglot/returned_columns_test.clj b/test/metabase/sql_tools/sqlglot/returned_columns_test.clj
new file mode 100644
index 00000000000..a546312f621
--- /dev/null
+++ b/test/metabase/sql_tools/sqlglot/returned_columns_test.clj
@@ -0,0 +1,480 @@
+(ns metabase.sql-tools.sqlglot.returned-columns-test
+ "Tests for SQLGlot returned-columns implementation.
+ Ported from metabase.sql-tools.macaw.references-test.
+
+ Note: These tests verify that returned-columns produces correct output,
+ not that it matches Macaw's exact format. The key assertions are:
+ - Correct column names
+ - Correct column count
+ - Type information when schema allows"
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.sql-tools.core :as sql-tools]
+ [metabase.test :as mt]
+ [metabase.util :as u]))
+
+;;; Test Helpers
+
+(defn- returned-column-names
+ "Extract just the column names from returned-columns result.
+ Normalizes to lowercase for cross-database comparison."
+ [columns]
+ (mapv (comp u/lower-case-en :name) columns))
+
+(defn- returned-column-aliases
+ "Extract just the column aliases from returned-columns result.
+ Normalizes to lowercase for cross-database comparison."
+ [columns]
+ (mapv (comp u/lower-case-en :lib/desired-column-alias) columns))
+
+(defn- make-query
+ "Create a native query with the test metadata provider."
+ [sql]
+ (lib/native-query (mt/metadata-provider) sql))
+
+(defn- returned-columns
+ "Get returned columns for a SQL query using the test metadata provider."
+ [sql]
+ (let [driver (:engine (lib.metadata/database (mt/metadata-provider)))]
+ (sql-tools/returned-columns-impl :sqlglot driver (make-query sql))))
+
+;;; Basic SELECT Tests
+
+(deftest ^:parallel basic-select-columns-test
+ (testing "Simple SELECT returns expected columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT ID, NAME FROM PEOPLE")]
+ (is (= 2 (count result)))
+ ;; SQLGlot normalizes to lowercase
+ (is (= ["id" "name"] (returned-column-names result)))
+ (is (every? #(= :metadata/column (:lib/type %)) result))))))
+
+(deftest ^:parallel select-star-test
+ (testing "SELECT * returns all columns from table"
+ (mt/test-driver :h2
+ ;; Note: SQLGlot with lineage expands * to actual column names
+ (let [result (returned-columns "SELECT * FROM PEOPLE")]
+ (is (pos? (count result)))
+ (is (every? #(= :metadata/column (:lib/type %)) result))))))
+
+(deftest ^:parallel select-with-alias-test
+ (testing "SELECT with alias preserves alias in output"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT ID AS user_id FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "id" (first (returned-column-names result))))
+ (is (= "user_id" (first (returned-column-aliases result))))))))
+
+;;; JOIN Tests
+
+(deftest ^:parallel basic-join-test
+ (testing "JOIN query returns columns from both tables"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT PEOPLE.NAME, ORDERS.TOTAL
+ FROM PEOPLE
+ INNER JOIN ORDERS ON PEOPLE.ID = ORDERS.USER_ID")]
+ (is (= 2 (count result)))
+ (is (= #{"name" "total"} (set (returned-column-names result))))))))
+
+(deftest ^:parallel join-with-table-wildcard-test
+ (testing "JOIN with table.* returns columns from that table"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ORDERS.*
+ FROM PEOPLE
+ INNER JOIN ORDERS ON PEOPLE.ID = ORDERS.USER_ID")]
+ (is (pos? (count result)))
+ ;; All returned columns should be from ORDERS
+ (is (every? some? result))))))
+
+;;; Subquery Tests
+
+(deftest ^:parallel subquery-test
+ (testing "Subquery columns are traced correctly"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT a, b FROM (SELECT ID AS a, NAME AS b FROM PEOPLE)")]
+ (is (= 2 (count result)))
+ (is (= ["a" "b"] (returned-column-aliases result)))))))
+
+(deftest ^:parallel nested-subquery-test
+ (testing "Nested subqueries trace through multiple levels"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT x FROM (SELECT a AS x FROM (SELECT ID AS a FROM PEOPLE))")]
+ (is (= 1 (count result)))
+ (is (= "x" (first (returned-column-aliases result))))))))
+
+;;; CTE Tests
+
+(deftest ^:parallel basic-cte-test
+ (testing "CTE columns are traced correctly"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "WITH active_users AS (SELECT ID, NAME FROM PEOPLE WHERE ID > 0)
+ SELECT * FROM active_users")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+;;; Aggregate Tests
+
+(deftest ^:parallel aggregate-test
+ (testing "Aggregate functions return custom columns (pure? = false)"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT COUNT(*) AS cnt FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "cnt" (first (returned-column-names result))))
+ ;; Aggregates should have :type/* since they're computed
+ (is (= :type/* (:base-type (first result))))))))
+
+(deftest ^:parallel group-by-aggregate-test
+ (testing "GROUP BY with aggregates"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT STATE, COUNT(*) AS total
+ FROM PEOPLE
+ GROUP BY STATE")]
+ (is (= 2 (count result)))
+ (is (= #{"state" "total"} (set (returned-column-names result))))))))
+
+;;; Error Handling Tests
+
+(deftest ^:parallel invalid-sql-test
+ (testing "Invalid SQL returns empty result (no crash)"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT * FORM PEOPLE")]
+ ;; Should return empty, not throw
+ (is (vector? result))))))
+
+(deftest ^:parallel missing-table-test
+ (testing "Reference to non-existent table returns empty or partial result"
+ (mt/test-driver :h2
+ ;; This might fail during qualification or return empty
+ (let [result (returned-columns "SELECT * FROM nonexistent_table")]
+ (is (vector? result))))))
+
+;;; Expression Tests
+
+(deftest ^:parallel expression-column-test
+ (testing "Expressions in SELECT create computed columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT ID + 1 AS incremented FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "incremented" (first (returned-column-names result))))
+ ;; Computed columns have :type/* base type
+ (is (= :type/* (:base-type (first result))))))))
+
+(deftest ^:parallel case-expression-test
+ (testing "CASE expressions create computed columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT CASE WHEN ID > 5 THEN 'high' ELSE 'low' END AS category
+ FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "category" (first (returned-column-names result))))))))
+
+;;; UNION Tests
+
+(deftest ^:parallel union-test
+ (testing "UNION combines columns from both branches"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, NAME FROM PEOPLE
+ UNION
+ SELECT ID, NAME FROM PEOPLE WHERE ID > 5")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+;;; Window Function Tests
+
+(deftest ^:parallel window-function-test
+ (testing "Window functions create computed columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT NAME, ROW_NUMBER() OVER (ORDER BY ID) AS rn
+ FROM PEOPLE")]
+ (is (= 2 (count result)))
+ (is (= #{"name" "rn"} (set (returned-column-names result))))))))
+
+;;; More Subquery Tests (ported from references_test.clj)
+
+(deftest ^:parallel renamed-nested-query-test
+ (testing "Alias chain through subquery: a -> b -> c"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT b AS c FROM (SELECT ID AS b FROM PEOPLE)")]
+ (is (= 1 (count result)))
+ (is (= "c" (first (returned-column-aliases result))))))))
+
+(deftest ^:parallel different-case-nested-query-test
+ (testing "Case-insensitive column matching through subquery"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT A FROM (SELECT id AS a FROM PEOPLE)")]
+ (is (= 1 (count result)))
+ ;; Should match despite case difference
+ (is (= "a" (first (returned-column-aliases result))))))))
+
+(deftest ^:parallel wildcard-nested-query-test
+ (testing "SELECT * from subquery expands to subquery columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT * FROM (SELECT ID, NAME FROM PEOPLE)")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+(deftest ^:parallel table-wildcard-nested-query-test
+ (testing "SELECT alias.* from subquery with alias"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT p.* FROM (SELECT ID, NAME FROM PEOPLE) p")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+;;; WHERE Clause Tests
+
+(deftest ^:parallel basic-where-test
+ (testing "WHERE clause doesn't affect returned columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT * FROM PEOPLE WHERE STATE = 'CA'")]
+ ;; WHERE doesn't change output columns
+ (is (pos? (count result)))
+ (is (every? #(= :metadata/column (:lib/type %)) result))))))
+
+(deftest ^:parallel where-is-null-test
+ (testing "WHERE with IS NULL"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, NAME FROM PEOPLE WHERE NAME IS NOT NULL")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+(deftest ^:parallel where-between-test
+ (testing "WHERE with BETWEEN"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, NAME FROM PEOPLE WHERE ID BETWEEN 1 AND 100")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+;;; Constant/Literal Tests
+
+(deftest ^:parallel select-constant-test
+ (testing "SELECT constant returns computed column"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT 1 AS one")]
+ (is (= 1 (count result)))
+ (is (= "one" (first (returned-column-aliases result))))
+ ;; Constants are computed (pure? = false)
+ (is (= :type/* (:base-type (first result))))))))
+
+(deftest ^:parallel select-string-constant-test
+ (testing "SELECT string constant"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT 'hello' AS greeting")]
+ (is (= 1 (count result)))
+ (is (= "greeting" (first (returned-column-names result))))))))
+
+;;; More CTE Tests
+
+(deftest ^:parallel unused-cte-test
+ (testing "Unused CTE doesn't affect output"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "WITH unused AS (SELECT ID FROM PEOPLE)
+ SELECT NAME FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "name" (u/lower-case-en (:name (first result)))))))))
+
+(deftest ^:parallel dependent-cte-test
+ (testing "CTE that references another CTE"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "WITH cte1 AS (SELECT ID, NAME FROM PEOPLE),
+ cte2 AS (SELECT ID FROM cte1)
+ SELECT * FROM cte2")]
+ (is (= 1 (count result)))
+ (is (= "id" (u/lower-case-en (:name (first result)))))))))
+
+(deftest ^:parallel multiple-cte-test
+ (testing "Multiple CTEs with different columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "WITH names AS (SELECT NAME FROM PEOPLE),
+ ids AS (SELECT ID FROM PEOPLE)
+ SELECT * FROM names")]
+ (is (= 1 (count result)))
+ (is (= "name" (u/lower-case-en (:name (first result)))))))))
+
+;;; Scalar Subquery Tests
+
+(deftest ^:parallel scalar-subquery-test
+ (testing "Scalar subquery in SELECT"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, (SELECT COUNT(*) FROM ORDERS) AS order_count
+ FROM PEOPLE")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "order_count"} (set (returned-column-names result))))))))
+
+;;; More Aggregate Tests
+
+(deftest ^:parallel multiple-aggregates-test
+ (testing "Multiple aggregates in one query"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT COUNT(*) AS cnt, SUM(ID) AS total, AVG(ID) AS avg_id
+ FROM PEOPLE")]
+ (is (= 3 (count result)))
+ (is (= #{"cnt" "total" "avg_id"} (set (returned-column-names result))))
+ ;; All aggregates are computed
+ (is (every? #(= :type/* (:base-type %)) result))))))
+
+(deftest ^:parallel aggregate-with-distinct-test
+ (testing "COUNT DISTINCT"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT COUNT(DISTINCT STATE) AS unique_states FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "unique_states" (first (returned-column-names result))))))))
+
+(deftest ^:parallel having-clause-test
+ (testing "GROUP BY with HAVING"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT STATE, COUNT(*) AS cnt
+ FROM PEOPLE
+ GROUP BY STATE
+ HAVING COUNT(*) > 1")]
+ (is (= 2 (count result)))
+ (is (= #{"state" "cnt"} (set (returned-column-names result))))))))
+
+;;; ORDER BY Tests
+
+(deftest ^:parallel order-by-test
+ (testing "ORDER BY doesn't affect returned columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, NAME FROM PEOPLE ORDER BY NAME DESC")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+(deftest ^:parallel order-by-expression-test
+ (testing "ORDER BY expression not in SELECT"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT NAME FROM PEOPLE ORDER BY ID")]
+ ;; ID is not in output, only NAME
+ (is (= 1 (count result)))
+ (is (= "name" (u/lower-case-en (:name (first result)))))))))
+
+;;; LIMIT/OFFSET Tests
+
+(deftest ^:parallel limit-test
+ (testing "LIMIT doesn't affect returned columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, NAME FROM PEOPLE LIMIT 10")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+;;; DISTINCT Tests
+
+(deftest ^:parallel select-distinct-test
+ (testing "SELECT DISTINCT returns same columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT DISTINCT STATE FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "state" (u/lower-case-en (:name (first result)))))))))
+
+;;; Qualified Column Names
+
+(deftest ^:parallel fully-qualified-column-test
+ (testing "Fully qualified column names"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT PEOPLE.ID, PEOPLE.NAME FROM PEOPLE")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+;;; More Window Function Tests
+
+(deftest ^:parallel window-partition-by-test
+ (testing "Window function with PARTITION BY"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT NAME, STATE,
+ RANK() OVER (PARTITION BY STATE ORDER BY NAME) AS state_rank
+ FROM PEOPLE")]
+ (is (= 3 (count result)))
+ (is (= #{"name" "state" "state_rank"} (set (returned-column-names result))))))))
+
+(deftest ^:parallel multiple-window-functions-test
+ (testing "Multiple window functions"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID,
+ ROW_NUMBER() OVER (ORDER BY ID) AS rn,
+ LAG(ID) OVER (ORDER BY ID) AS prev_id
+ FROM PEOPLE")]
+ (is (= 3 (count result)))
+ (is (= #{"id" "rn" "prev_id"} (set (returned-column-names result))))))))
+
+;;; UNION ALL / INTERSECT / EXCEPT
+
+(deftest ^:parallel union-all-test
+ (testing "UNION ALL preserves all rows but same columns"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, NAME FROM PEOPLE WHERE ID < 5
+ UNION ALL
+ SELECT ID, NAME FROM PEOPLE WHERE ID >= 5")]
+ (is (= 2 (count result)))
+ (is (= #{"id" "name"} (set (returned-column-names result))))))))
+
+;;; Coalesce / NVL / IFNULL
+
+(deftest ^:parallel coalesce-test
+ (testing "COALESCE creates computed column"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT COALESCE(NAME, 'Unknown') AS name_or_default FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "name_or_default" (first (returned-column-names result))))
+ (is (= :type/* (:base-type (first result))))))))
+
+;;; String Functions
+
+(deftest ^:parallel string-concat-test
+ (testing "String concatenation"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT NAME || ' from ' || STATE AS description FROM PEOPLE")]
+ (is (= 1 (count result)))
+ (is (= "description" (first (returned-column-names result))))))))
+
+;;; Math Expressions
+
+(deftest ^:parallel math-expression-test
+ (testing "Math expression in SELECT"
+ (mt/test-driver :h2
+ (let [result (returned-columns
+ "SELECT ID, ID * 2 AS doubled, ID + 10 AS plus_ten FROM PEOPLE")]
+ (is (= 3 (count result)))
+ (is (= #{"id" "doubled" "plus_ten"} (set (returned-column-names result))))
+ ;; ID is pure, expressions are computed
+ (is (= :type/* (:base-type (second result))))))))
+
+;;; NULL Handling
+
+(deftest ^:parallel null-literal-test
+ (testing "NULL literal in SELECT"
+ (mt/test-driver :h2
+ (let [result (returned-columns "SELECT NULL AS nothing")]
+ (is (= 1 (count result)))
+ (is (= "nothing" (first (returned-column-names result))))))))
diff --git a/test/metabase/sql_tools/test_util.clj b/test/metabase/sql_tools/test_util.clj
new file mode 100644
index 00000000000..2a4bdbebc41
--- /dev/null
+++ b/test/metabase/sql_tools/test_util.clj
@@ -0,0 +1,35 @@
+(ns metabase.sql-tools.test-util
+ "Utilities for testing sql-tools with multiple parser backends.
+
+ Use [[test-parser-backends]] to run tests against both :macaw and :sqlglot:
+
+ (deftest my-test
+ (test-parser-backends
+ (is (= expected (sql-tools/validate-query ...)))))
+
+ Set MB_SQL_TOOLS_PARSER_BACKEND to run just one backend:
+
+ MB_SQL_TOOLS_PARSER_BACKEND=macaw bin/mage run-tests ..."
+ (:require
+ [clojure.test :as t]
+ [colorize.core :as colorize]
+ [environ.core :as env]
+ [metabase.sql-tools.settings :as sql-tools.settings]))
+
+(def ^:private all-backends [:macaw :sqlglot])
+
+(defn parser-backends-to-test
+ "Returns [[all-backends]], or just the one specified by MB_SQL_TOOLS_PARSER_BACKEND."
+ []
+ (if-let [explicit (env/env :mb-sql-tools-parser-backend)]
+ [(keyword explicit)]
+ all-backends))
+
+(defmacro test-parser-backends
+ "Run body against both parser backends (or just one if env var is set)."
+ {:style/indent 0}
+ [& body]
+ `(doseq [backend# (parser-backends-to-test)]
+ (t/testing (colorize/magenta (name backend#))
+ (binding [sql-tools.settings/*parser-backend-override* backend#]
+ ~@body))))