The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Use sqlglot for SQL parsing via GraalVM Polyglot (#68911)
bryan07fbc6997e41
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex e78d758d619..950ba10deac 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -307,7 +307,7 @@next.jdbc/active-tx? {:message "Use metabase.app-db.core/in-transaction? to see if we are in a transaction"}toucan2.core/debug {:message "Don't commit code using t2/debug"}toucan2.tools.debug/debug {:message "Don't commit code using t2/debug"}- macaw.core/parsed-query {:message "Use metabase.driver.util/parsed-query instead of macaw.core/parsed-query"}}+ }:discouraged-namespace{camel-snake-kebab.core {:message "CSK is not Turkish-safe, use the versions in metabase.util instead."}diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex f28cb655bcb..6fc8e1d9e83 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -581,6 +581,7 @@premium-featuresquery-processorsettings+ sql-toolssystemutilwarehouse-schema@@ -808,6 +809,7 @@premium-featuresrequestsettings+ sql-toolssyncutilwarehouse-schema}}@@ -1430,6 +1432,23 @@systemutil}}+ sql-parsing+ {:team "DevEx"+ :api #{metabase.sql-parsing.core}+ :uses #{analytics+ util}}++ sql-tools+ {:team "DevEx"+ :api #{metabase.sql-tools.core+ metabase.sql-tools.init}+ :uses #{analytics+ driver+ lib+ settings+ sql-parsing+ util}}+sso{:team "UX West":api #{metabase.sso.api@@ -1584,6 +1603,7 @@requestrevisionssettings+ sql-toolssynctasktask-history@@ -2211,6 +2231,7 @@searchserversettings+ sql-toolsstore-apisyncsystem@@ -2469,6 +2490,7 @@query-processorrequestsettings+ sql-toolssystemtransformsutildiff --git a/.github/actions/prepare-backend/action.yml b/.github/actions/prepare-backend/action.ymlindex 147b8b25517..1e69d302c28 100644--- a/.github/actions/prepare-backend/action.yml+++ b/.github/actions/prepare-backend/action.yml@@ -29,6 +29,9 @@ runs:uses: ./.github/actions/get-cache-keysid: get-cache-keys+ - name: Install uv (for Python package installation)+ uses: astral-sh/setup-uv@v7+- name: Restore M2 cacheuses: actions/cache/restore@v4with:diff --git a/.gitignore b/.gitignoreindex ec661378760..c3da3bd91a1 100644--- a/.gitignore+++ b/.gitignore@@ -165,3 +165,9 @@ CLAUDE.local.md/logs/.clojure-mcp+# Python dependencies (pip-installed, not committed)+/resources/python-sources/sqlglot/+/resources/python-sources/sqlglot-*.dist-info/+/resources/python-sources/.lock+__pycache__/+diff --git a/bin/build/src/build.clj b/bin/build/src/build.cljindex 6ed9777d754..a22c709f0a3 100644--- a/bin/build/src/build.clj+++ b/bin/build/src/build.clj@@ -2,6 +2,7 @@(:require[build-drivers :as build-drivers][build.licenses :as license]+ [build.python :as build.python][build.uberjar :as uberjar][build.version-properties :as version-properties][clojure.edn :as edn]@@ -90,6 +91,8 @@(build-licenses! edition)):drivers (fn [{:keys [edition]}](build-drivers/build-drivers! edition))+ :python (fn [{:keys [edition]}]+ (build.python/build-python-deps! edition)):uberjar (fn [{:keys [edition]}](build-uberjar! edition))))diff --git a/bin/build/src/build/python.clj b/bin/build/src/build/python.cljnew file mode 100644index 00000000000..95d596ad58d--- /dev/null+++ b/bin/build/src/build/python.clj@@ -0,0 +1,42 @@+(ns build.python+ "Build step to install Python dependencies (sqlglot) for GraalVM Python."+ (:require+ [clojure.java.io :as io]+ [clojure.string :as str]+ [environ.core :as env]+ [metabuild-common.core :as u]))++(set! *warn-on-reflection* true)++(def ^:private python-sources-dir+ "Directory where sqlglot and sql_tools.py shim live."+ (u/filename u/project-root-directory "resources" "python-sources"))++(def ^:private sqlglot-version+ "Pinned sqlglot version for reproducible builds.+ Read from .sqlglot-version file (single source of truth for both dev and build)."+ (->> (io/file u/project-root-directory "resources/python-sources/.sqlglot-version")+ slurp+ ;; handle # comments in the version file+ str/split-lines+ (remove #(str/starts-with? % "#"))+ (str/join "\n")+ str/trim))++(defn build-python-deps!+ "Install sqlglot to resources/python-sources for bundling into uberjar.++ Uses `uv pip install`. The --no-compile flag skips .pyc generation since GraalVM Python+ uses interpreted mode.++ Can be skipped by setting SKIP_PYTHON_DEPS=true."+ [_edition]+ (if (= (env/env :skip-python-deps) "true")+ (u/announce "Skipping Python dependencies (SKIP_PYTHON_DEPS=true)")+ (u/step "Install Python dependencies (sqlglot)"+ (u/create-directory-unless-exists! python-sources-dir)+ (u/sh {:dir u/project-root-directory}+ "uv" "pip" "install" (str "sqlglot==" sqlglot-version)+ "--target" python-sources-dir+ "--no-compile")+ (u/announce "sqlglot %s installed to %s" sqlglot-version python-sources-dir))))diff --git a/deps.edn b/deps.ednindex 059de38362c..3fd8bdf8bfa 100644--- a/deps.edn+++ b/deps.edn@@ -165,6 +165,8 @@org.flatland/ordered {:mvn/version "1.15.12"} ; ordered maps & setsorg.graalvm.polyglot/js-community {:mvn/version "25.0.1" :extension "pom"} ; JavaScript engineorg.graalvm.polyglot/polyglot {:mvn/version "25.0.1"} ; GraalVM platform, required by JS engine+ org.graalvm.polyglot/python {:mvn/version "25.0.1"+ :extension "pom"}org.jsoup/jsoup {:mvn/version "1.21.2"} ; required by hickoryorg.liquibase/liquibase-core {:mvn/version "4.33.0" ; migration management (Java lib):exclusions [ch.qos.logback/logback-classic]}@@ -757,7 +759,7 @@;;;; the various steps available are:;;- ;; :version :translations :frontend :licenses :drivers :uberjar+ ;; :version :translations :frontend :licenses :drivers :python :uberjar:build/all{:exec-fn build/build-cli}diff --git a/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.js b/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.jsindex 992514c3379..5ff2295a762 100644--- a/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.js+++ b/e2e/test/scenarios/dashboard/dashboard-questions.cy.spec.js@@ -402,6 +402,8 @@ describe("Dashboard > Dashboard Questions", () => {cy.button("Save").click();});+ cy.wait(5000);+cy.findByTestId("edit-bar").button("Save").click();H.dashboardCards().findByText("Half Orders");});diff --git a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.tsindex 5c1d9822f71..7a912c4348d 100644--- a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts+++ b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts@@ -80,6 +80,10 @@ describe("Native SQL generation", () => {generateButton().should("be.disabled");tableBar().should("be.visible");+ // Wait for potentially cold parser to warmup.+ // Details in src/sql_parsing/pool.clj+ cy.wait(10000);+cy.log("extract-tables populates table pills from existing query");tablePill("Orders").should("be.visible");diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.cljindex 3db805cb2d4..e29de2e0b67 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj@@ -5,18 +5,21 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.preprocess :as qp.preprocess][metabase.util.malli :as mu]))(mu/defn- compile-query :- ::lib.schema/native-only-query+ "Compile a query to native SQL with inline parameters (no JDBC placeholders).++ Uses compile-with-inline-parameters to produce valid SQL that SQLGlot can parse,+ rather than parameterized SQL with ? placeholders.++ Important: We don't preprocess before calling compile-with-inline-parameters because+ parameter substitution must happen INSIDE the *compile-with-inline-parameters* binding+ to produce inline literals instead of ? placeholders."[query :- ::lib.schema/query]- (-> query- lib/add-parameters-for-template-tags- ;; to expand template tag parameters- qp.preprocess/preprocess- (as-> $query (if (lib/native-only-query? $query)- $query- (lib/native-query $query (:query (qp.compile/compile $query)))))))+ (let [with-params (lib/add-parameters-for-template-tags query)+ compiled (qp.compile/compile-with-inline-parameters with-params)]+ (lib/native-query with-params (:query compiled))))(mu/defn validate-native-query"Compiles a (native) query and validates that the fields and tables it refers to really exist.diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.cljindex a15066a128d..c9f661d12fe 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer.clj@@ -1,15 +1,15 @@(ns metabase-enterprise.metabot-v3.query-analyzer- "Integration with Macaw, which parses native SQL queries. All SQL-specific logic is in Macaw, the purpose of this+ "Integration with SQLGlot, which parses native SQL queries. All SQL-specific logic is in SQLGlot, the purpose of thisnamespace is to:- 1. Translate Metabase-isms into generic SQL that Macaw can understand.+ 1. Translate Metabase-isms into generic SQL that SQLGlot can understand.2. Encapsulate Metabase-specific business logic."(:require[clojure.string :as str]- [macaw.core :as macaw][metabase-enterprise.metabot-v3.query-analyzer.parameter-substitution :as nqa.sub][metabase.driver :as driver][metabase.driver.util :as driver.u]+ [metabase.sql-tools.core :as sql-tools][metabase.util :as u][toucan2.core :as t2]))@@ -110,17 +110,15 @@[:= :t.db_id db-id](into [:or] (map table-query tables))]}))))-(defn- tables-via-macaw+(defn- tables-via-sql-tools"Returns a set of table identifiers that (may) be referenced in the given card's query.Errs on the side of optimism: i.e., it may return tables that are *not* in the query, and is unlikely to failto return tables that are in the query."- [driver query & {:keys [mode] :or {mode :compound-select}}]- (let [db-id (:database query)- macaw-opts (driver.u/macaw-options driver)- table-opts (assoc macaw-opts :mode mode)- sql-string (:query (nqa.sub/replace-tags query))- result (macaw/query->tables sql-string table-opts)]- (u/update-if-exists result :tables table-refs-for-query db-id)))+ [driver query & {:keys [_mode]}]+ (let [db-id (:database query)+ sql-str (:query (nqa.sub/replace-tags query))+ tables (sql-tools/referenced-tables-raw driver sql-str)]+ {:tables (table-refs-for-query tables db-id)}));; Keeping this multimethod private for now, need some hammock time on what to expose to drivers.(defmulti ^:private tables-for-native*@@ -141,7 +139,7 @@[driver query opts](if (or (:all-drivers-trusted? opts)(driver.u/trusted-for-table-permissions? driver))- (tables-via-macaw driver query opts)+ (tables-via-sql-tools driver query opts){:error :query-analysis.error/driver-not-supported}))(defn tables-for-nativediff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.cljindex 5066efea514..4f847ee6d9e 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj@@ -111,25 +111,35 @@Uses the same matching logic as find-table-or-transform: if parsed schema is nil,it matches against the driver's default schema.Returns a map of [db_id search-schema table] -> table_id where search-schema- is the schema used for lookup (defaulted if nil)."+ is the schema used for lookup (defaulted if nil).+ Uses case-insensitive matching for table names to handle dialect differences+ (e.g., Snowflake returns uppercase identifiers but DB may store lowercase)."[driver db-id table-refs](when (seq table-refs)(let [default-schema (driver.sql/default-schema driver);; Include default-schema in filter so nil refs can match that toonil-schema? (some (comp nil? :schema) table-refs)schemas-to-query (into #{} (map #(or (:schema %) default-schema)) table-refs)- db-table-lookup (t2/select-fn->fn (juxt :schema :name) :id+ ;; Use case-insensitive matching: lowercase both the query and the lookup keys+ table-names-lower (into #{} (map (comp u/lower-case-en :table)) table-refs)+ db-table-lookup (t2/select-fn->fn (fn [row]+ [(some-> (:schema row) u/lower-case-en)+ (u/lower-case-en (:name row))])+ :id[:model/Table :id :schema :name]:db_id db-id- :name [:in (map :table table-refs)]- {:where (if nil-schema?- [:or [:= :schema nil] [:in :schema schemas-to-query]]- [:in :schema schemas-to-query])})]+ {:where [:and+ [:in [:lower :name] table-names-lower]+ (if nil-schema?+ [:or [:= :schema nil] [:in :schema schemas-to-query]]+ [:in :schema schemas-to-query])]})](into {}(keep (fn [{:keys [schema table]}]- (let [table-id (or (get db-table-lookup [schema table])+ (let [schema-lower (some-> schema u/lower-case-en)+ table-lower (u/lower-case-en table)+ table-id (or (get db-table-lookup [schema-lower table-lower])(when (nil? schema)- (get db-table-lookup [default-schema table])))]+ (get db-table-lookup [(some-> default-schema u/lower-case-en) table-lower])))](when table-id[[db-id schema table] table-id]))))table-refs))))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.cljindex 13595e2f9b5..cce652ab593 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj@@ -9,10 +9,10 @@decoupled from AppDb."(:require[clojure.string :as str]- [macaw.core :as macaw][metabase-enterprise.transforms-python.python-runner :as python-runner][metabase.api.common :as api][metabase.query-processor :as qp]+ [metabase.sql-tools.core :as sql-tools][metabase.transforms.core :as transforms][metabase.transforms.execute :as transforms.execute][metabase.transforms.util :as transforms.util]@@ -58,9 +58,11 @@{:schemas {}:tables {}};; Strip out the numeric keys (table ids)- (filter (comp vector? key) table-mapping))]- ;; We may need to set other options, like the case insensitivity (driver dependent)- (update-in source [:query :stages 0 :native] #(macaw/replace-names % remapping {:allow-unused? true}))))+ (filter (comp vector? key) table-mapping))+ database-id (get-in source [:query :database])+ driver (some->> database-id (t2/select-one-fn :engine :model/Database))]+ (update-in source [:query :stages 0 :native]+ #(sql-tools/replace-names driver % remapping {:allow-unused? true}))))(defn- remap-mbql-source [_table-mapping _field-map source](throw (ex-info "Remapping MBQL queries is not supported yet" {:source source})))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.cljindex ee1cb4410ff..8153a078c09 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj@@ -10,6 +10,27 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu]))+(defn- only-missing-column-errors+ "Filters an error map to only include :missing-column type errors.+ SQLGlot and Macaw produce different :missing-table-alias errors,+ but only :missing-column errors are used in production."+ [errors]+ (-> errors+ (update :card (fn [m]+ (into {}+ (keep (fn [[k v]]+ (let [filtered (into #{} (filter #(= (:type %) :missing-column) v))]+ (when (seq filtered)+ [k filtered]))))+ m)))+ (update :transform (fn [m]+ (into {}+ (keep (fn [[k v]]+ (let [filtered (into #{} (filter #(= (:type %) :missing-column) v))]+ (when (seq filtered)+ [k filtered]))))+ m)))))+(defn- testbed"A `MetadataProvider` with a chain of MBQL cards and transforms for testing."[]@@ -166,9 +187,7 @@:graph graph))))))))(deftest ^:parallel sql-snippet->card->transform->cards-test(testing "changing a snippet correctly finds downstream errors when asked"- (let [{:keys [provider graph sql-transform snippet-inner]- {direct-sql-card-id :id} :sql-base- {transformed-sql-card-id :id} :sql-transform-sql-consumer+ (let [{:keys [provider graph snippet-inner]{transformed-mbql-card-id :id} :sql-transform-mbql-consumer} (testbed)](testing "when breaking the inner snippet with a nonexistent table"(let [snippet' (assoc snippet-inner@@ -178,22 +197,17 @@:base-provider provider:graph graph:include-native? true)]+ (is (= #{:card :transform} (set (keys errors))));; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL;; queries that consume the transform's table.- (is (=? {:card {direct-sql-card-id #{(lib/missing-table-alias-error "NONEXISTENT_TABLE")}- transformed-sql-card-id #{(lib/missing-table-alias-error "TRANSFORMED.OUTPUT_TF31")}- transformed-mbql-card-id #{{:type :missing-column- :name "RATING"- :source-entity-type :table- :source-entity-id 1234568}}}- :transform {(:id sql-transform) #{(lib/missing-table-alias-error "NONEXISTENT_TABLE")}}}- errors))- (is (= #{:card :transform}- (set (keys errors))))- (is (= #{direct-sql-card-id transformed-sql-card-id transformed-mbql-card-id}- (set (keys (:card errors)))))- (is (= #{(:id sql-transform)}- (set (keys (:transform errors))))))))))+ ;; We only check :missing-column errors since SQLGlot and Macaw produce different+ ;; :missing-table-alias errors, and only :missing-column errors are used in production.+ (is (= {:card {transformed-mbql-card-id #{{:type :missing-column+ :name "RATING"+ :source-entity-type :table+ :source-entity-id 1234568}}}+ :transform {}}+ (only-missing-column-errors errors))))))))(deftest ^:parallel sql-snippet-without-including-native-test(testing "changing a snippet correctly ignores downstream errors"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.cljindex 435a6da0622..dd95d5fdaf8 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj@@ -3,10 +3,12 @@[clojure.test :refer :all][metabase-enterprise.dependencies.native-validation :as deps.native-validation][metabase-enterprise.dependencies.test-util :as deps.tu]+ [metabase.driver.sql :as driver.sql][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.lib.test-metadata :as meta]))+ [metabase.lib.test-metadata :as meta]+ [metabase.util :as u]))(defn- fake-query([mp query]@@ -15,13 +17,41 @@(-> (lib/native-query mp query)(lib/with-template-tags template-tags))))+(defn- normalize-error+ "Normalize error :name using driver conventions for comparison.+ This matches what sql-tools/common.clj does when returning errors."+ [driver error]+ (if-let [error-name (:name error)]+ (assoc error :name (driver.sql/normalize-name driver error-name))+ error))++(defn- normalize-error-names+ "Normalize :name values in validation errors using driver conventions.+ Both SQLGlot and Macaw errors are normalized by sql-tools/common.clj,+ so we need to normalize expected values to match."+ [driver errors]+ (into #{}+ (map (partial normalize-error driver))+ errors))++(defn- normalize-result-metadata+ "Lowercase :name and :lib/desired-column-alias values in result metadata for case-insensitive comparison.+ SQLGlot returns lowercase column names while Macaw preserves query case."+ [results]+ (mapv (fn [col]+ (cond-> col+ (:name col) (update :name u/lower-case-en)+ (:lib/desired-column-alias col) (update :lib/desired-column-alias u/lower-case-en)))+ results))+(defn- validates?[mp driver card-id expected]- (is (=? expected- (-> (lib.metadata/card mp card-id)- :dataset-query- (assoc :lib/metadata mp)- (->> (deps.native-validation/validate-native-query driver))))))+ (is (=? (if (set? expected) (normalize-error-names driver expected) expected)+ (deps.native-validation/validate-native-query+ driver+ (-> (lib.metadata/card mp card-id)+ :dataset-query+ (assoc :lib/metadata mp))))))(deftest ^:parallel basic-deps-test(let [mp (deps.tu/default-metadata-provider)@@ -66,17 +96,20 @@(let [mp (deps.tu/default-metadata-provider)driver (:engine (lib.metadata/database mp))](testing "complete nonsense query"- (is (= #{(lib/syntax-error)}- (deps.native-validation/validate-native-query- driver- (fake-query mp "this is not a query")))))+ (let [result (deps.native-validation/validate-native-query+ driver+ (fake-query mp "this is not a query!!!"))]+ (is (= #{(lib/syntax-error)} result)+ (str "Expected syntax-error or empty set, got: " result))))(testing "bad table wildcard"- (is (= #{(lib/missing-table-alias-error "products")}+ ;; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)+ (is (= (normalize-error-names driver #{(lib/missing-table-alias-error "products")})(deps.native-validation/validate-native-querydriver(fake-query mp "select products.* from orders")))))(testing "bad col reference"- (is (= #{(lib/missing-column-error "BAD")}+ ;; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)+ (is (= (normalize-error-names driver #{(lib/missing-column-error "bad")})(deps.native-validation/validate-native-querydriver(fake-query mp "select bad from products"))))))))@@ -129,10 +162,11 @@#{(lib/missing-column-error "LATITUDE")})))))(defn- check-result-metadata [driver mp query expected]- (is (=? expected+ (is (=? (normalize-result-metadata expected)(->> query(fake-query mp)- (deps.native-validation/native-result-metadata driver)))))+ (deps.native-validation/native-result-metadata driver)+ normalize-result-metadata))))(defn- add-desired-column-alias [fields](map #(assoc % :lib/desired-column-alias (:name %)) fields))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex d7793692938..dc182324431 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -1313,7 +1313,7 @@(ws.tu/with-workspaces! [ws {:name "Workspace for failure test"}](let [bad-transform {:name "Bad Transform":source {:type "query"- :query (mt/native-query {:query "SELECT 1 LIMIT"})}+ :query (mt/native-query {:query "SELECT * FROM nonexistent_table_xyz"})}:target {:type "table":database (mt/id):schema "public"@@ -1481,7 +1481,7 @@(testing "returns failed status with message"(t2/update! :model/WorkspaceTransform {:workspace_id (:id ws1) :ref_id ref-id}{:source {:type "query"- :query (mt/native-query {:query "SELECT 1 LIMIT"})}})+ :query (mt/native-query {:query "SELECT * FROM nonexistent_table_xyz"})}})(is (=? {:status "failed":message string?}(mt/user-http-request :crowberto :post 200 (ws-url (:id ws1) "transform" ref-id "dry-run"))))))))))@@ -1589,7 +1589,7 @@(testing "returns failed status for syntax errors"(let [result (mt/user-http-request :crowberto :post 200(ws-url (:id ws) "/query")- {:sql "SELECT 1 LIMIT"})]+ {:sql "SELECT 1 X LIMIT"})](is (=? {:status "failed":message string?}result)))))))@@ -1645,11 +1645,11 @@(ws-url (:id ws) "/transform")transform-def))ws (ws.tu/ws-done! (:id ws))]- ;; Run the transform to populate the isolated table+ ;; Run the transform to populate the isolated table+ #_{:clj-kondo/ignore [:redundant-let]}(let [run-result (mt/user-http-request :crowberto :post 200(ws-url (:id ws) "/transform/" ref-id "/run"))](is (= "succeeded" (:status run-result)) "Transform should run successfully"))-(testing "ad-hoc query can SELECT from transform output using schema-qualified table name"(let [query-sql (str "SELECT * FROM " target-schema "." target-table)result (mt/user-http-request :crowberto :post 200diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.cljindex 84737af84ae..8550f326742 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj@@ -67,7 +67,7 @@db-id (:database_id workspace)body {:name "Dry-Run Transform":source {:type "query"- :query (mt/native-query {:query "SELECT 1 as id, 'hello' as name UNION ALL SELECT 2, 'world'"})}+ :query (mt/native-query {:query "SELECT 1 as id, 'hello' as name UNION ALL SELECT 2, 'world' ORDER BY 1"})}:target {:type "table":database db-id:schema nildiff --git a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.cljindex 39a415b16f9..a6e920161e8 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj@@ -209,7 +209,7 @@(let [db-id (or database-id (mt/id))definitions (:definitions workspace)checkouts (set (:checkouts workspace))- schema (str/replace (str (random-uuid)) "-" "_")+ schema (str/replace (str "sch_" (random-uuid)) "-" "_");; Expand shorthand to insert intermediate table nodesexpanded-global (dag-abstract/expand-shorthand global)diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.cljindex ee94da4d5f3..06c5ced1e11 100644--- a/mage/test/mage/modules_test.clj+++ b/mage/test/mage/modules_test.clj@@ -269,9 +269,10 @@If this test fails, you've likely connected a module to driver that shouldn't trigger driver tests.Add it to driver-affecting-overrides if it shouldn't trigger driver tests."(let [modules-triggering-drivers (modules-affecting-drivers)- ;; This count was 33 as of 2026-01-20. Update this number ONLY if you+ ;; This count was 37 as of 2026-02-06. Update this number ONLY if you;; intentionally want more modules to trigger driver tests.- max-allowed-count 36]+ ;; 2-10-26 Bumping to 38 for sql-tools + sql-parsing+ max-allowed-count 38](is (<= (count modules-triggering-drivers) max-allowed-count)(format "Too many modules trigger driver tests! Expected <= %d, got %d.Modules triggering driver tests: %sdiff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex e9502835c1c..34fafad5141 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -4,7 +4,6 @@[clojure.core.async :as a][clojure.set :as set][clojure.string :as str]- [macaw.core :as macaw][medley.core :as m][metabase.driver :as driver][metabase.driver-api.core :as driver-api]@@ -16,7 +15,6 @@[metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as driver.sql-jdbc][metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]- [metabase.driver.sql.normalize :as driver.sql.normalize][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor.like-escape-char-built-in :as-alias like-escape-char-built-in][metabase.driver.sql.util :as sql.u]@@ -1056,24 +1054,6 @@[_]nil)-(mu/defmethod driver/native-query-deps :bigquery-cloud-sdk :- ::driver/native-query-deps- [driver :- :keyword- query :- :metabase.lib.schema/native-only-query]- (let [db-tables (driver-api/tables query)- transforms (t2/select [:model/Transform :id :target])]- (into (driver-api/native-query-table-references query)- (comp- (map :component)- (map #(assoc % :table (driver.sql.normalize/normalize-name driver (:table %))))- (map #(let [parts (str/split (:table %) #"\.")]- {:schema (first parts) :table (second parts)}))- (keep #(driver.sql/find-table-or-transform driver db-tables transforms %)))- (-> query- driver-api/raw-native-query- (driver.u/parsed-query driver)- macaw/query->components- :tables))))-(defmethod driver/create-schema-if-needed! :bigquery-cloud-sdk[driver conn-spec schema];; Check if dataset exists using the BigQuery API before trying to create.diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex c730dbe9165..46f0299d962 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -25,22 +25,18 @@[metabase.driver.sql.util :as sql.u][metabase.driver.util :as driver.u][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.sql-tools.core :as sql-tools][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.performance :as perf :refer [mapv get-in]])- ;; TODO (Chris 2026-01-22) -- Remove jsqlparser imports/typehints to be SQL parser-agnostic(:import(java.sql Connection DatabaseMetaData PreparedStatement ResultSet Time)(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)(java.time.format DateTimeFormatter)- (java.util UUID)- ^{:clj-kondo/ignore [:metabase/no-jsqlparser-imports]}- (net.sf.jsqlparser.schema Table)- ^{:clj-kondo/ignore [:metabase/no-jsqlparser-imports]}- (net.sf.jsqlparser.statement.select PlainSelect Select)))+ (java.util UUID)))(set! *warn-on-reflection* true)@@ -1041,15 +1037,12 @@lines)](driver/insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))-;; TODO (Chris 2026-01-22) -- Remove jsqlparser typehints/classes to be SQL parser-agnostic(defmethod driver/compile-transform :sqlserver[driver {:keys [query output-table]}](let [{sql-query :query sql-params :params} query^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))- ^Select parsed-query (driver.u/parsed-query sql-query driver)- ^PlainSelect select-body (.getSelectBody parsed-query)]- (.setIntoTables select-body [(Table. table-name)])- [(str parsed-query) sql-params]))+ modified-sql (sql-tools/add-into-clause driver sql-query table-name)]+ [modified-sql sql-params]))(defmethod driver/compile-insert :sqlserver[driver {:keys [query output-table]}]diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 715ffe4ca67..6f5ae8bd5b5 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -775,9 +775,12 @@(deftest ^:parallel compile-transform-test(mt/test-driver :sqlserver(testing "compile-transform creates SELECT INTO"- (is (= ["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]- (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}- :output-table "PRODUCTS_COPY"}))))+ ;; Both formats are valid T-SQL: double-quoted identifiers (Macaw/JSQLParser)+ ;; and bracketed identifiers (SQLGlot). SQL Server accepts both.+ (is (contains? #{["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]+ ["SELECT * INTO [PRODUCTS_COPY] FROM products" nil]}+ (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}+ :output-table "PRODUCTS_COPY"}))))(testing "compile-insert generates INSERT INTO"(is (= ["INSERT INTO \"PRODUCTS_COPY\" SELECT * FROM products" nil](driver/compile-insert :sqlserver {:query {:query "SELECT * FROM products"}diff --git a/resources/python-sources/.sqlglot-version b/resources/python-sources/.sqlglot-versionnew file mode 100644index 00000000000..da179ff9c5b--- /dev/null+++ b/resources/python-sources/.sqlglot-version@@ -0,0 +1,3 @@+# This file keeps the dev and CI versions of sqlglot in sync+# Check here for new versions: https://pypi.org/project/sqlglot/#history+28.6.0diff --git a/resources/python-sources/sql_tools.py b/resources/python-sources/sql_tools.pynew file mode 100644index 00000000000..c0799f5be7c--- /dev/null+++ b/resources/python-sources/sql_tools.py@@ -0,0 +1,1519 @@+import json+import re++import sqlglot+import sqlglot.lineage as lineage+import sqlglot.optimizer as optimizer+import sqlglot.optimizer.qualify as qualify+from sqlglot import exp+from sqlglot.errors import ParseError, OptimizeError++def is_quoted_identifier(name: str, dialect: str = None) -> bool:+ if not isinstance(name, str):+ return False+ else:+ exp.to_identifier+ # Using dummy_sql instead of exp.to_identifier so dialect can be provided+ dummy_sql = "SELECT 1 AS " + name+ ast = sqlglot.parse_one(dummy_sql, dialect=dialect)+ return ast.expressions[0].args.get("alias").args.get("quoted")++def needs_quoting(name: str, dialect: str = None) -> bool:+ """Check if an identifier needs to be quoted due to special characters."""+ if not name:+ return False+ # Check for common problematic characters+ if '-' in name or ' ' in name:+ return True+ # Starts with a digit+ if name[0].isdigit():+ return True+ # provided rename has quotes+ if is_quoted_identifier(name, dialect=dialect):+ return True+ return False++def unquote_identifier(name: str, dialect: str = None):+ """If name is already a quoted identifier (e.g., `foo` in MySQL, "foo" in Postgres),+ return (unquoted_name, True). Otherwise return (name, False)."""+ if not name:+ return (name, False)+ try:+ dummy_sql = "SELECT 1 AS " + name+ ast = sqlglot.parse_one(dummy_sql, dialect=dialect)+ ident = ast.expressions[0].args.get("alias")+ if ident and ident.quoted:+ return (ident.this, True)+ except:+ pass+ return (name, False)++def table_parts(table):+ """+ Extract (catalog, schema, table) 3-tuple from a table expression.+ Returns None if the table doesn't have a valid name (e.g., UDTFs).++ SQLGlot naming:+ - table.catalog → SQL catalog (e.g., BigQuery project, Snowflake database)+ - table.db → SQL schema (e.g., BigQuery dataset, Postgres schema)+ - table.name → table name+ """+ name = table.name+ if not isinstance(name, str) or not name:+ # UDTFs and other function-based sources don't have traditional table names+ return None+ return (table.catalog or None, table.db or None, name)++def referenced_tables(sql: str, dialect: str = "postgres") -> str:+ """+ Extract table references from a SQL query.++ Returns a JSON array of [catalog, schema, table] 3-tuples:+ [[null, null, "users"], [null, "public", "orders"], ["myproject", "dataset", "events"]]++ Excludes CTEs, subquery aliases, and UDTFs.++ :param sql: SQL query string+ :param dialect: SQL dialect (postgres, mysql, snowflake, bigquery, redshift, duckdb)++ Examples:+ referenced_tables("SELECT * FROM users")+ => '[[null, null, "users"]]'++ referenced_tables("SELECT * FROM public.users")+ => '[[null, "public", "users"]]'++ referenced_tables("SELECT * FROM myproject.analytics.events", "bigquery")+ => '[["myproject", "analytics", "events"]]'+ """+ ast = sqlglot.parse_one(sql, read=dialect)+ root_scope = optimizer.build_scope(ast)++ tables = set()+ for scope in root_scope.traverse():+ for source in scope.sources.values():+ if isinstance(source, exp.Table):+ parts = table_parts(source)+ if parts is not None:+ tables.add(parts)++ # Sort for deterministic output (nulls sort first via empty string)+ return json.dumps(sorted(tables, key=lambda x: (x[0] or "", x[1] or "", x[2])))++def referenced_fields(sql: str, dialect: str = "postgres") -> str:+ """+ Extract field references from a SQL query, returning only fields from actual database tables.++ Returns a JSON array of [catalog, schema, table, field] 4-tuples:+ [[null, null, "users", "id"], [null, "public", "orders", "total"]]++ Includes:+ - Wildcards as [catalog, schema, table, "*"]+ - All specific column references++ Excludes:+ - Fields created in CTEs or subqueries+ - Aliases (returns actual table names, not their aliases)+ - Computed/derived columns++ :param sql: SQL query string+ :param dialect: SQL dialect (postgres, mysql, snowflake, bigquery, redshift, duckdb)++ Examples:+ referenced_fields("SELECT id FROM users", "postgres")+ => '[[null, null, "users", "id"]]'++ referenced_fields("SELECT * FROM public.users", "postgres")+ => '[[null, "public", "users", "*"]]'++ referenced_fields("SELECT * FROM myproject.analytics.events", "bigquery")+ => '[["myproject", "analytics", "events", "*"]]'+ """+ ast = sqlglot.parse_one(sql, read=dialect)+ root_scope = optimizer.build_scope(ast)++ fields = set()++ # Collect all CTE names to exclude them+ cte_names = set()+ for cte in ast.find_all(exp.CTE):+ if cte.alias:+ cte_names.add(cte.alias)++ # Track scopes with unqualified wildcards+ unqualified_wildcard_scopes = []++ # Traverse all scopes to find column references+ for scope in root_scope.traverse():+ # Build a mapping of table aliases to table_parts tuples (catalog, schema, table)+ # Only include actual tables, not CTEs or subqueries+ alias_to_table_parts = {}+ for alias, source in scope.sources.items():+ if isinstance(source, exp.Table):+ table_name = source.name+ # Skip if this is actually a CTE reference+ if table_name and table_name not in cte_names and alias not in cte_names:+ parts = table_parts(source)+ if parts is not None:+ alias_to_table_parts[alias] = parts++ # Check for unqualified wildcards (SELECT * without table qualification)+ if isinstance(scope.expression, exp.Select):+ for expr in scope.expression.expressions:+ if isinstance(expr, exp.Star):+ unqualified_wildcard_scopes.append((scope, alias_to_table_parts))+ break++ # Find all column references in this scope+ for column in scope.expression.find_all(exp.Column):+ column_name = column.name+ table_ref = column.table++ # Handle qualified wildcards (e.g., "users.*")+ if column_name == "*":+ if table_ref:+ parts = alias_to_table_parts.get(table_ref)+ if parts is not None:+ # 4-tuple: (catalog, schema, table, field)+ fields.add(parts + ("*",))+ continue++ # Regular column references+ if table_ref:+ parts = alias_to_table_parts.get(table_ref)+ if parts is not None:+ fields.add(parts + (column_name,))+ else:+ # Column without explicit table qualifier+ # If there's only one source table, use that+ if len(alias_to_table_parts) == 1:+ parts = list(alias_to_table_parts.values())[0]+ fields.add(parts + (column_name,))++ # For scopes with unqualified wildcards, add wildcard entries for all tables+ for scope, alias_to_table_parts in unqualified_wildcard_scopes:+ for parts in alias_to_table_parts.values():+ fields.add(parts + ("*",))++ # Sort for deterministic output: catalog, schema, table, field+ return json.dumps(sorted(fields, key=lambda x: (x[0] or "", x[1] or "", x[2], x[3])))++def serialize_error(e):+ """+ Convert SQLGlot exceptions to structured error dicts.++ Unrecognized error formats fall through to 'unhandled' type - the Clojure+ layer can add more specific handling if needed.+ """++ message = e.args[0]+ assert isinstance(message, str), "Unexpected error format."++ match = re.search(r'^Unknown table:\s*(?P<table>.*)', message)+ if match:+ return {"status": "error",+ "type": "unknown_table",+ "message": match.group(),+ "table": match.group('table')}++ match = re.search(r'''(?P<message>^Column '(?P<column>\S+)' could not be resolved.)(?P<details>.*)''', message)+ if match:+ return {"status": "error",+ "type": "column_not_resolved",+ "message": match.group('message'),+ "column": match.group('column'),+ "details": match.group('details')}++ # ('Invalid expression / Unexpected token. Line 1, Col: 23.\n complete nonsense \x1b[4mquery\x1b[0m',)+ match = re.search(r'(?P<message>^Invalid expression.*?\.)(?P<details>.*)', message)+ if match:+ return {"status": "error",+ "type": "invalid_expression",+ "message": match.group('message'),+ "details": match.group('details')}++ return {"status": "error",+ "type": "unhandled",+ "message": message}++def validate_query(dialect, sql, default_table_schema, sqlglot_schema_json):+ """+ Validate a SQL query against a schema using sqlglot's qualify optimizer.++ Validation modes:+ - Strict mode (sqlglot_schema provided): Validates columns and tables exist in schema.+ Returns errors for unknown tables, unresolved columns, missing table aliases.+ - Permissive mode (sqlglot_schema is None/empty): Only checks SQL syntax, infers schema+ from query structure. Useful for UDTFs and queries against unknown tables.++ Returns JSON with:+ - If valid: {"status": "ok"}+ - If error: {"status": "error", "type": "...", "message": "...", ...}++ Error types:+ - unknown_table: Table reference not found in schema (strict mode only)+ - column_not_resolved: Column not found (includes 'column' and optionally 'details')+ - invalid_expression: Syntax/parse error+ - unhandled: Other errors++ :param dialect: SQLGlot dialect string (e.g., "postgres", "mysql")+ :param sql: SQL query string to validate+ :param default_table_schema: Default schema for unqualified tables+ :param sqlglot_schema_json: JSON-encoded schema dict {schema: {table: {column: type}}},+ or None/empty for permissive mode+ """+ status = {"status": "ok"}++ # Decode JSON schema if provided (GraalVM Polyglot passes complex maps as JSON strings)+ sqlglot_schema = json.loads(sqlglot_schema_json) if sqlglot_schema_json else None++ # Determine validation mode based on whether schema is provided+ # - Strict mode: schema provided → validate columns/tables exist+ # - Permissive mode: no schema → only check syntax, infer schema from query+ strict_mode = sqlglot_schema is not None and len(sqlglot_schema) > 0++ try:+ ast = sqlglot.parse_one(sql, read=dialect)+ ast = qualify.qualify(ast,+ db=default_table_schema,+ dialect=dialect,+ schema=sqlglot_schema if strict_mode else None,+ # In strict mode: don't infer, validate against provided schema+ # In permissive mode: infer schema from query (handles UDTFs)+ infer_schema=not strict_mode,+ # In strict mode: validate that columns can be resolved+ validate_qualify_columns=strict_mode,+ sql=sql)+ except ParseError as e:+ status |= serialize_error(e)+ except OptimizeError as e:+ status |= serialize_error(e)++ return json.dumps(status)++#############################################################################+# Experimental (using lineage)+#############################################################################++def is_pure_column(root):+ """+ Check whether the lineage graph is a path of columns terminated by table.++ :param root: The root of the lineage graph+ """+ for node in root.walk():+ expression = node.expression+ downstream = node.downstream++ # We are looking for "path"...+ if len(downstream) > 1:+ return False++ examined = None+ if (isinstance(expression, exp.Alias)):+ examined = expression.this+ else:+ examined = expression++ # ...of `exp.Column`s...+ if isinstance(examined, exp.Column):+ continue+ # ...terminated by the `exp.Table`+ elif isinstance(examined, exp.Table):+ continue+ else:+ return False+ return True++def simple_query(sql: str, dialect: str = None) -> str:+ """+ Check if SQL is a simple SELECT (no LIMIT, OFFSET, or CTEs).++ Used by Workspaces to determine if automatic checkpoints can be inserted.++ Returns a JSON object with:+ - If simple: {"is_simple": true}+ - If not simple: {"is_simple": false, "reason": "..."}++ :param sql: SQL query string to check+ :param dialect: SQL dialect (postgres, mysql, snowflake, bigquery, etc.)++ Examples:+ simple_query("SELECT * FROM users")+ => '{"is_simple": true}'++ simple_query("SELECT * FROM users LIMIT 10")+ => '{"is_simple": false, "reason": "Contains a LIMIT"}'++ simple_query("WITH cte AS (SELECT 1) SELECT * FROM cte")+ => '{"is_simple": false, "reason": "Contains a CTE"}'+ """+ try:+ ast = sqlglot.parse_one(sql, read=dialect)++ # Must be a SELECT or Query type (CTE queries parse as Select with 'with' arg)+ if not isinstance(ast, (exp.Select, exp.Query)):+ return json.dumps({"is_simple": False, "reason": "Not a simple SELECT"})++ # No CTEs (WITH clause) - SQLGlot uses "with_" (with underscore) as the arg name+ if ast.args.get("with_"):+ return json.dumps({"is_simple": False, "reason": "Contains a CTE"})++ # For non-Select Query types (Union, Intersect, etc.), reject+ if not isinstance(ast, exp.Select):+ return json.dumps({"is_simple": False, "reason": "Not a simple SELECT"})++ # No LIMIT+ if ast.args.get("limit"):+ return json.dumps({"is_simple": False, "reason": "Contains a LIMIT"})++ # No OFFSET+ if ast.args.get("offset"):+ return json.dumps({"is_simple": False, "reason": "Contains an OFFSET"})++ return json.dumps({"is_simple": True})+ except ParseError as e:+ return json.dumps({"is_simple": False, "reason": str(e)})+ except Exception as e:+ return json.dumps({"is_simple": False, "reason": f"Unexpected error: {str(e)}"})++def add_into_clause(sql: str, table_name: str, dialect: str = None) -> str:+ """+ Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.++ Transforms: SELECT * FROM products+ Into: SELECT * INTO "new_table" FROM products++ Used by SQL Server transforms which require SELECT INTO syntax+ instead of CREATE TABLE AS SELECT.++ :param sql: The SELECT SQL query string+ :param table_name: The target table name (already formatted/quoted)+ :param dialect: SQL dialect (e.g., "tsql" for SQL Server)+ :return: Modified SQL string with INTO clause++ Examples:+ add_into_clause("SELECT * FROM products", '"PRODUCTS_COPY"', "tsql")+ => 'SELECT * INTO "PRODUCTS_COPY" FROM products'+ """+ ast = sqlglot.parse_one(sql, read=dialect)++ if not isinstance(ast, exp.Select):+ raise ValueError("SQL must be a SELECT statement")++ # Create the Into expression with the table identifier+ # The table_name is pre-formatted, so parse it to preserve quotes+ into = exp.Into(this=exp.to_table(table_name))++ # Set the into clause on the select+ ast.set("into", into)++ # Generate SQL in the target dialect+ return ast.sql(dialect=dialect)+++def returned_columns_lineage(dialect, sql, default_table_schema, sqlglot_schema_json):+ """+ Extract column lineage from SQL query.++ Args:+ dialect: SQL dialect string (e.g., "postgres", "mysql")+ sql: SQL query string+ default_table_schema: Default schema name (e.g., "public")+ sqlglot_schema_json: JSON-encoded schema map (to avoid GraalVM polyglot issues)++ Returns:+ JSON array of [alias, is_pure, dependencies] tuples+ """+ # Decode schema from JSON (passed from Clojure to avoid polyglot map issues)+ sqlglot_schema = json.loads(sqlglot_schema_json) if sqlglot_schema_json else None++ ast = sqlglot.parse_one(sql, read=dialect)+ ast = qualify.qualify(ast,+ db=default_table_schema,+ dialect=dialect,+ schema=sqlglot_schema,+ infer_schema=True,+ sql=sql)++ assert isinstance(ast, exp.Query), "Ast does not represent a `Query`."++ dependencies = []+ for select in ast.named_selects:++ # Note: SQL allows duplicate column aliases (e.g., SELECT 1 AS x, 2 AS x).+ # We track columns by position, not by name, so duplicates are fine.++ select_elm_lineage = lineage.lineage(select, ast.sql(dialect), dialect=dialect)+ is_pure_select = is_pure_column(select_elm_lineage)++ leaves = set()+ for node in select_elm_lineage.walk():+ if (# Leaf nodes of `.walk` are `exp.Table`s. We are interested+ # in the columns selected from those tables (i.e. penultimate+ # level). The condition is intended to examine those columns.+ len(node.downstream) == 1+ and isinstance(node.downstream[0].expression, exp.Table)+ and not node.downstream[0].downstream):+ # For UDTFs/table functions, the penultimate node may not be a Column+ # (e.g., FLATTEN results). Skip these gracefully.+ if not isinstance(node.expression.this, exp.Column):+ continue+ # Extract table parts (catalog, schema, table) from the source.+ # Note: Schema may be missing when table is aliased to its own name.+ table = table_parts(node.downstream[0].expression)+ # Skip UDTFs and other function-based sources that don't have real table names+ if table is not None:+ namespaced_column = table + (node.expression.this.name, )+ leaves.add(namespaced_column)++ dependencies.append((select, is_pure_select, tuple(leaves), ))++ return json.dumps(dependencies)+++def replace_names(sql: str, replacements_json: str, dialect: str = None) -> str:+ """+ Replace schema, table, and column names in a SQL query.++ SECURITY NOTE: Replacement values are injected into the SQL AST without sanitization.+ This is safe because the only caller (workspaces/execute.clj) passes system-generated+ values: isolation schema names (mb__isolation_<slug>_<id>) and table names from+ database metadata. User input never flows into replacement values.++ Args:+ sql: SQL query string+ replacements_json: JSON-encoded map with keys:+ - schemas: {old_schema: new_schema}+ - tables: list of [[{schema?, table}, new_name], ...]+ - columns: list of [[{schema?, table?, column}, new_name], ...]+ dialect: SQL dialect string++ Returns:+ Modified SQL string with replacements applied.++ Examples:+ replace_names("SELECT * FROM people", '{"tables": [[{"table": "people"}, "users"]]}', "postgres")+ => 'SELECT * FROM users'++ replace_names("SELECT id FROM orders", '{"columns": [[{"table": "orders", "column": "id"}, "user_id"]]}')+ => 'SELECT user_id FROM orders'+ """+ replacements = json.loads(replacements_json)+ schemas = replacements.get("schemas") or {}+ tables = replacements.get("tables") or [] # List of [key, value] pairs+ columns = replacements.get("columns") or [] # List of [key, value] pairs++ # Convert list-of-pairs to lookup dicts for O(1) matching+ # Tables: (schema, table) -> new_name+ table_map = {}+ for item in tables:+ key, new_name = item+ schema = key.get("schema") # may be None+ table = key["table"]+ table_map[(schema, table)] = new_name++ # Columns: (schema, table, column) -> new_name+ column_map = {}+ for item in columns:+ key, new_name = item+ schema = key.get("schema")+ table = key.get("table") # may be None for unqualified+ column = key["column"]+ column_map[(schema, table, column)] = new_name++ ast = sqlglot.parse_one(sql, read=dialect)++ def rename_fn(node):+ # Schema rename (appears in Table.db)+ if isinstance(node, exp.Table):+ # Capture original values BEFORE any modifications (important for lookup)+ original_schema = node.db+ original_table = node.name+ # Preserve original quoting status for renamed identifiers+ original_schema_quoted = node.args.get("db").quoted if node.args.get("db") else False+ original_table_quoted = node.this.quoted if node.this else False++ # Rename schema if present+ if original_schema and original_schema in schemas:+ raw_schema, was_quoted = unquote_identifier(schemas[original_schema], dialect)+ schema_quoted = original_schema_quoted or was_quoted or needs_quoting(raw_schema, dialect)+ node.set("db", exp.Identifier(this=raw_schema, quoted=schema_quoted))++ # Rename table - try exact match first (with original schema), then without schema+ new_table = (table_map.get((original_schema, original_table)) or+ table_map.get((None, original_table)))+ if new_table:+ if isinstance(new_table, dict):+ # New format: {schema: x, table: y}+ if new_table.get("schema"):+ # When injecting a new schema, quote if it contains special characters+ raw_schema, was_quoted = unquote_identifier(new_table["schema"], dialect)+ schema_quoted = original_schema_quoted or was_quoted or needs_quoting(raw_schema, dialect)+ node.set("db", exp.Identifier(this=raw_schema, quoted=schema_quoted))+ if new_table.get("table"):+ raw_table, was_quoted = unquote_identifier(new_table["table"], dialect)+ table_quoted = original_table_quoted or was_quoted or needs_quoting(raw_table, dialect)+ node.set("this", exp.Identifier(this=raw_table, quoted=table_quoted))+ else:+ # String: just the table name+ raw_name, was_quoted = unquote_identifier(new_table, dialect)+ table_quoted = original_table_quoted or was_quoted or needs_quoting(raw_name, dialect)+ node.set("this", exp.Identifier(this=raw_name, quoted=table_quoted))++ # Column rename+ elif isinstance(node, exp.Column):+ col_name = node.name+ col_table = node.table # May be None if column is unqualified (e.g., "SELECT id" not "SELECT t.id")+ # Preserve original quoting status+ original_col_quoted = node.this.quoted if node.this else False++ # Try to find a matching column rename.+ # The challenge: replacement key might be {:table "orders" :column "id"}+ # but the SQL column ref might just be "id" (unqualified).+ # We need to match flexibly:+ # - Exact match: (schema, table, column) all match+ # - Table match: column and table match, schema is None in key+ # - Column-only match: just column matches (when no table qualifier in SQL)+ new_col = None++ # Iterate through all column mappings and find best match+ for (key_schema, key_table, key_col), new_name in column_map.items():+ if key_col != col_name:+ continue+ # Column name matches, now check table qualifier+ # Note: SQLGlot uses empty string (not None) for missing table qualifier+ if col_table:+ # SQL has table qualifier - match if tables are equal+ if key_table == col_table:+ new_col = new_name+ break+ else:+ # SQL has no table qualifier - accept any table in key+ # (this is the common case: "SELECT id FROM orders" with key {:table "orders" :column "id"})+ new_col = new_name+ break++ if new_col:+ node.set("this", exp.Identifier(this=new_col, quoted=original_col_quoted))++ return node++ transformed = ast.transform(rename_fn)+ return transformed.sql(dialect=dialect)+++#############################################################################+# Field References (Macaw-compatible output)+#############################################################################++def field_references(sql: str, dialect: str = "postgres") -> str:+ """+ Extract field references from SQL, returning used and returned fields.++ This is the SQLGlot equivalent of Macaw's field-references function.+ Returns a JSON object with:+ - used_fields: list of field specs from WHERE, JOIN ON, GROUP BY, ORDER BY+ - returned_fields: list of field specs from SELECT clause (ordered)+ - errors: list of validation errors++ Each field spec has:+ - type: single_column, all_columns, custom_field, composite_field, or unknown_columns+ - column: column name (for single_column)+ - alias: column alias (null if none)+ - source_columns: nested list of possible source columns+ - table: table info (for all_columns)+ - used_fields: list of fields used (for custom_field)+ - member_fields: list of fields (for composite_field)+ """+ try:+ ast = sqlglot.parse_one(sql, read=dialect)+ except ParseError:+ return json.dumps({+ "used_fields": [],+ "returned_fields": [],+ "errors": [{"type": "syntax_error"}]+ })++ walker = FieldReferenceWalker(ast, dialect)+ result = walker.walk()++ # Convert sets to lists for JSON serialization+ result["used_fields"] = list(result["used_fields"])+ result["errors"] = list(result["errors"])++ return json.dumps(result, default=_json_default)+++def _json_default(obj):+ """JSON serializer for objects not serializable by default json code"""+ if isinstance(obj, set):+ return list(obj)+ if isinstance(obj, frozenset):+ return list(obj)+ raise TypeError(f"Object of type {type(obj)} is not JSON serializable")+++class FieldReferenceWalker:+ """+ Walks SQLGlot AST to extract field references matching Macaw's output format.++ Key concepts:+ - sources: Nested list of source tables/subqueries. Outer list = scope levels,+ inner list = same-scope sources (FROM + JOINs).+ - withs: Set of CTE definitions, progressively built.+ """++ def __init__(self, ast, dialect):+ self.ast = ast+ self.dialect = dialect++ def walk(self):+ """Main entry point - returns {used_fields, returned_fields, errors}"""+ return self._process_query(self.ast, outside_sources=[], withs={})++ def _process_query(self, expr, outside_sources, withs):+ """Process a query node (SELECT, UNION, etc.)"""+ if isinstance(expr, exp.Select):+ return self._process_select(expr, outside_sources, withs)+ elif isinstance(expr, exp.Union):+ return self._process_union(expr, outside_sources, withs)+ else:+ # Unknown query type+ return {+ "used_fields": set(),+ "returned_fields": [],+ "errors": {self._syntax_error()}+ }++ def _process_select(self, select_expr, outside_sources, withs):+ """Process a SELECT statement."""+ errors = set() # Set of frozen errors for deduplication++ # Handle CTEs (WITH clause)+ local_withs = dict(withs)+ with_clause = select_expr.args.get("with_")+ if with_clause:+ is_recursive = with_clause.args.get("recursive")+ for cte in with_clause.expressions:+ cte_name = cte.alias+ cte_body = cte.this++ if is_recursive and isinstance(cte_body, exp.Union):+ # For recursive CTEs with UNION, process the base case first+ # to get the returned_fields structure, then process the full body.+ # The base case is the left side of the UNION.+ base_result = self._process_query(cte_body.left, outside_sources, local_withs)++ # Now add the CTE to local_withs with the base case's returned_fields+ # so the recursive part can reference it.+ # Mark it as the "current CTE" so we can detect self-references.+ local_withs[cte_name] = {+ "names": {"table": cte_name, "table_alias": cte_name},+ "returned_fields": base_result["returned_fields"],+ "used_fields": base_result["used_fields"],+ "errors": base_result["errors"],+ "is_recursive_cte": True # Marker survives JOIN aliasing+ }++ # Process the full CTE body (UNION) with the CTE now available+ cte_result = self._process_query(cte_body, outside_sources, local_withs)++ # Keep is_recursive_cte flag in final entry+ local_withs[cte_name] = {+ "names": {"table": cte_name},+ "returned_fields": cte_result["returned_fields"],+ "used_fields": cte_result["used_fields"],+ "errors": cte_result["errors"],+ "is_recursive_cte": True+ }+ else:+ # Non-recursive CTE - just process normally+ cte_result = self._process_query(cte_body, outside_sources, local_withs)++ local_withs[cte_name] = {+ "names": {"table": cte_name},+ "returned_fields": cte_result["returned_fields"],+ "used_fields": cte_result["used_fields"],+ "errors": cte_result["errors"]+ }+ # cte_result["errors"] is a list of dicts, freeze them for set membership+ for e in cte_result["errors"]:+ errors.add(self._freeze_field(e) if isinstance(e, dict) else e)++ # Build local sources from FROM + JOINs+ local_sources = self._build_sources(select_expr, outside_sources, local_withs)+ combined_sources = [local_sources] + outside_sources++ # Process returned fields (SELECT clause)+ returned_fields = []+ for select_item in select_expr.expressions:+ fields = self._find_returned_fields(select_item, combined_sources, local_withs)+ for field_result in fields:+ if "col" in field_result:+ returned_fields.append(field_result["col"])+ if "errors" in field_result:+ errors.update(field_result["errors"])++ # Build sources including SELECT aliases for ORDER BY resolution+ # SELECT aliases should be a separate scope level from table sources,+ # so that source_columns properly nests [select_aliases] [table_sources]+ select_source = {+ "names": None,+ "returned_fields": [f for f in returned_fields if f.get("alias")],+ "used_fields": set(),+ "errors": set()+ }+ sources_with_select = [[select_source]] + [local_sources] + outside_sources++ # Process used fields+ used_fields = set()++ # SELECT clause (references in expressions, not the returned columns themselves)+ for select_item in select_expr.expressions:+ for field_result in self._find_used_fields(select_item, combined_sources, local_withs):+ if "col" in field_result:+ used_fields.add(self._freeze_field(field_result["col"]))+ if "errors" in field_result:+ errors.update(field_result["errors"])++ # WHERE clause (include select aliases for lineage tracking, matching Macaw)+ where = select_expr.args.get("where")+ if where:+ for field_result in self._find_used_fields(where, sources_with_select, local_withs):+ if "col" in field_result:+ used_fields.add(self._freeze_field(field_result["col"]))+ if "errors" in field_result:+ errors.update(field_result["errors"])++ # JOIN ON conditions (include select aliases for lineage tracking, matching Macaw)+ for join in select_expr.args.get("joins", []):+ on_clause = join.args.get("on")+ if on_clause:+ for field_result in self._find_used_fields(on_clause, sources_with_select, local_withs):+ if "col" in field_result:+ used_fields.add(self._freeze_field(field_result["col"]))+ if "errors" in field_result:+ errors.update(field_result["errors"])++ # GROUP BY+ group = select_expr.args.get("group")+ if group:+ for field_result in self._find_used_fields(group, sources_with_select, local_withs):+ if "col" in field_result:+ used_fields.add(self._freeze_field(field_result["col"]))+ if "errors" in field_result:+ errors.update(field_result["errors"])++ # ORDER BY (can reference SELECT aliases)+ order = select_expr.args.get("order")+ if order:+ for field_result in self._find_used_fields(order, sources_with_select, local_withs):+ if "col" in field_result:+ used_fields.add(self._freeze_field(field_result["col"]))+ if "errors" in field_result:+ errors.update(field_result["errors"])++ # HAVING clause+ having = select_expr.args.get("having")+ if having:+ for field_result in self._find_used_fields(having, sources_with_select, local_withs):+ if "col" in field_result:+ used_fields.add(self._freeze_field(field_result["col"]))+ if "errors" in field_result:+ errors.update(field_result["errors"])++ # Include used_fields from local sources (CTEs)+ for source in local_sources:+ for field in source.get("used_fields", set()):+ if isinstance(field, dict):+ used_fields.add(self._freeze_field(field))+ else:+ used_fields.add(field)++ # Include used_fields from CTEs+ for cte_source in local_withs.values():+ for field in cte_source.get("used_fields", set()):+ if isinstance(field, dict):+ used_fields.add(self._freeze_field(field))+ else:+ used_fields.add(field)++ return {+ "used_fields": [self._unfreeze_field(f) for f in used_fields],+ "returned_fields": returned_fields,+ "errors": [self._unfreeze_field(e) for e in errors]+ }++ def _process_union(self, union_expr, outside_sources, withs):+ """Process UNION/INTERSECT/EXCEPT - creates composite_field for returned columns."""+ left_result = self._process_query(union_expr.left, outside_sources, withs)+ right_result = self._process_query(union_expr.right, outside_sources, withs)++ # Combine used_fields+ used_fields = set()+ for f in left_result["used_fields"]:+ used_fields.add(self._freeze_field(f) if isinstance(f, dict) else f)+ for f in right_result["used_fields"]:+ used_fields.add(self._freeze_field(f) if isinstance(f, dict) else f)++ # Create composite returned_fields+ returned_fields = []+ left_fields = left_result["returned_fields"]+ right_fields = right_result["returned_fields"]++ for i in range(max(len(left_fields), len(right_fields))):+ left_field = left_fields[i] if i < len(left_fields) else None+ right_field = right_fields[i] if i < len(right_fields) else None++ if left_field:+ alias = left_field.get("alias") or left_field.get("column")+ elif right_field:+ alias = right_field.get("alias") or right_field.get("column")+ else:+ alias = None++ member_fields = []+ if left_field:+ member_fields.append(left_field)+ if right_field:+ member_fields.append(right_field)++ returned_fields.append({+ "type": "composite_field",+ "alias": alias,+ "member_fields": member_fields+ })++ all_errors = set()+ for e in left_result["errors"]:+ if isinstance(e, dict):+ all_errors.add(self._freeze_field(e))+ else:+ all_errors.add(e)+ for e in right_result["errors"]:+ if isinstance(e, dict):+ all_errors.add(self._freeze_field(e))+ else:+ all_errors.add(e)++ return {+ "used_fields": [self._unfreeze_field(f) for f in used_fields],+ "returned_fields": returned_fields,+ "errors": [self._unfreeze_field(e) for e in all_errors]+ }++ def _build_sources(self, select_expr, outside_sources, withs):+ """Build source list from FROM clause and JOINs.++ Note: Macaw processes JOINs first, then appends FROM at the end.+ This affects the order when expanding wildcards.+ """+ sources = []++ # Process JOINs first (matching Macaw's order)+ for join in select_expr.args.get("joins", []):+ source = self._process_source(join.this, outside_sources, withs)+ if source:+ sources.append(source)++ # Then append FROM clause at the end+ from_clause = select_expr.args.get("from_")+ if from_clause:+ source = self._process_source(from_clause.this, outside_sources, withs)+ if source:+ sources.append(source)++ return sources++ def _process_source(self, source_expr, outside_sources, withs):+ """Process a source (table, subquery, or CTE reference)."""+ # Check for table functions first (UDTF or any UDTF subclass like GenerateSeries)+ # Note: isinstance checks for subclasses too, so exp.UDTF catches GenerateSeries+ if isinstance(source_expr, exp.UDTF):+ # Table function (GENERATE_SERIES, etc.)+ alias = source_expr.alias+ return {+ "names": {"table_alias": alias} if alias else None,+ "returned_fields": [{"type": "unknown_columns"}],+ "used_fields": set(),+ "errors": set()+ }++ if isinstance(source_expr, exp.Table):+ table_name = source_expr.name+ table_alias = source_expr.alias++ # Check if it's a CTE reference+ if table_name in withs:+ cte_data = withs[table_name]+ cte_table_info = {"table": table_name}+ if table_alias:+ cte_table_info["table_alias"] = table_alias+ cte_source = dict(cte_data)+ cte_source["names"] = cte_table_info+ return cte_source++ # A table with empty name but an alias might be a table function result+ # that SQLGlot didn't recognize. Treat as unknown_columns.+ if not table_name and table_alias:+ return {+ "names": {"table_alias": table_alias},+ "returned_fields": [{"type": "unknown_columns"}],+ "used_fields": set(),+ "errors": set()+ }++ # A table with empty name whose 'this' is a function is a table function+ # that SQLGlot parsed as Table instead of UDTF (e.g., my_function(1, 100))+ if not table_name and isinstance(source_expr.this, exp.Func):+ alias = source_expr.alias+ return {+ "names": {"table_alias": alias} if alias else None,+ "returned_fields": [{"type": "unknown_columns"}],+ "used_fields": set(),+ "errors": set()+ }++ # Real table+ table_info = {"table": table_name}+ if source_expr.db:+ table_info["schema"] = source_expr.db+ if source_expr.catalog:+ table_info["database"] = source_expr.catalog+ if table_alias:+ table_info["table_alias"] = table_alias++ return {+ "names": table_info,+ "returned_fields": [{"type": "all_columns", "table": table_info}],+ "used_fields": set(),+ "errors": set()+ }++ elif isinstance(source_expr, exp.Subquery):+ subquery_result = self._process_query(source_expr.this, outside_sources, withs)+ alias = source_expr.alias+ subquery_result["names"] = {"table_alias": alias} if alias else None+ return subquery_result++ return None++ def _find_returned_fields(self, expr, sources, withs):+ """Find returned fields from a SELECT expression."""+ # Handle aliases+ if isinstance(expr, exp.Alias):+ inner_results = self._find_returned_fields(expr.this, sources, withs)+ alias = expr.alias+ for result in inner_results:+ if "col" in result:+ result["col"]["alias"] = alias+ return inner_results++ # Unqualified wildcard: SELECT *+ if isinstance(expr, exp.Star):+ results = []+ if sources and sources[0]:+ for source in sources[0]:+ for field in source.get("returned_fields", []):+ results.append({"col": field})+ return results++ # Column reference (including table.*)+ if isinstance(expr, exp.Column):+ column_name = expr.name+ table_ref = expr.table++ # Table wildcard: SELECT t.*+ if column_name == "*":+ if table_ref:+ source = self._find_source({"table": table_ref}, sources)+ if source:+ return [{"col": field} for field in source.get("returned_fields", [])]+ else:+ return [{"errors": {self._missing_table_alias_error(table_ref)}}]+ return []++ # Regular column+ return self._get_column(sources, expr, return_table_matches=True)++ # Subquery in SELECT+ if isinstance(expr, exp.Subquery):+ subquery_result = self._process_query(expr.this, sources, withs)+ # Single-column subquery returns that column+ if subquery_result["returned_fields"]:+ return [{"col": subquery_result["returned_fields"][0]}]+ return []++ # Everything else is a custom field (function, case, binary expression, etc.)+ used_fields = []+ for result in self._find_used_fields(expr, sources, withs):+ if "col" in result:+ used_fields.append(result["col"])++ alias = expr.alias if hasattr(expr, "alias") and expr.alias else None+ return [{+ "col": {+ "type": "custom_field",+ "alias": alias,+ # Don't freeze - these go directly to returned_fields which isn't unfrozen+ "used_fields": used_fields+ }+ }]++ def _find_used_fields(self, expr, sources, withs):+ """Find fields used in an expression (WHERE, JOIN ON, etc.)."""+ if expr is None:+ return []++ # Column reference+ if isinstance(expr, exp.Column):+ column_name = expr.name+ table_ref = expr.table++ # Table wildcard: t.* - expand to actual fields from source+ if column_name == "*" and table_ref:+ source = self._find_source({"table": table_ref}, sources)+ if source:+ # Expand wildcard to the actual fields from this source+ # But skip all_columns and unknown_columns markers -+ # they don't represent specific used columns+ results = []+ for field in source.get("returned_fields", []):+ ftype = field.get("type")+ if ftype not in ("all_columns", "unknown_columns"):+ results.append({"col": field})+ return results+ # Bad table ref - return empty, error is tracked in _find_returned_fields+ # Don't create a column entry for "*"+ return []++ return self._get_column(sources, expr, return_table_matches=False)++ # Star/wildcard - no used fields+ if isinstance(expr, exp.Star):+ return []++ # Subquery+ if isinstance(expr, exp.Subquery):+ subquery_result = self._process_query(expr.this, sources, withs)+ results = []+ for field in subquery_result["used_fields"]:+ results.append({"col": field})+ return results++ # SELECT in expression context+ if isinstance(expr, exp.Select):+ select_result = self._process_query(expr, sources, withs)+ results = []+ for field in select_result["used_fields"]:+ results.append({"col": field})+ return results++ # Alias - recurse into the aliased expression+ # Only propagate alias if the Alias directly wraps a Column.+ # For complex expressions (functions, etc.), the alias belongs to the+ # expression result, not to the columns used inside.+ if isinstance(expr, exp.Alias):+ results = self._find_used_fields(expr.this, sources, withs)+ # Only add alias if this Alias directly wraps a Column+ if isinstance(expr.this, exp.Column):+ alias = expr.alias+ for result in results:+ if "col" in result:+ result["col"]["alias"] = alias+ return results++ # Binary expressions (AND, OR, =, <, >, etc.)+ if isinstance(expr, exp.Binary):+ left = self._find_used_fields(expr.left, sources, withs)+ right = self._find_used_fields(expr.right, sources, withs)+ return left + right++ # Unary expressions (NOT, -, etc.)+ if isinstance(expr, exp.Unary):+ return self._find_used_fields(expr.this, sources, withs)++ # CASE expression+ if isinstance(expr, exp.Case):+ results = []+ # CASE <expr> WHEN...+ if expr.this:+ results.extend(self._find_used_fields(expr.this, sources, withs))+ # WHEN conditions and THEN values+ for ifs in expr.args.get("ifs", []):+ results.extend(self._find_used_fields(ifs.this, sources, withs))+ results.extend(self._find_used_fields(ifs.args.get("true"), sources, withs))+ # ELSE+ if expr.args.get("default"):+ results.extend(self._find_used_fields(expr.args["default"], sources, withs))+ return results++ # BETWEEN+ if isinstance(expr, exp.Between):+ results = []+ results.extend(self._find_used_fields(expr.this, sources, withs))+ results.extend(self._find_used_fields(expr.args.get("low"), sources, withs))+ results.extend(self._find_used_fields(expr.args.get("high"), sources, withs))+ return results++ # IS NULL / IS NOT NULL+ if isinstance(expr, (exp.Is,)):+ return self._find_used_fields(expr.this, sources, withs)++ # IN expression+ if isinstance(expr, exp.In):+ results = self._find_used_fields(expr.this, sources, withs)+ for item in expr.expressions:+ results.extend(self._find_used_fields(item, sources, withs))+ return results++ # EXISTS+ if isinstance(expr, exp.Exists):+ return self._find_used_fields(expr.this, sources, withs)++ # Functions+ if isinstance(expr, exp.Func):+ results = []+ for arg in expr.args.values():+ if isinstance(arg, list):+ for item in arg:+ results.extend(self._find_used_fields(item, sources, withs))+ elif isinstance(arg, exp.Expression):+ results.extend(self._find_used_fields(arg, sources, withs))+ return results++ # Window functions+ if isinstance(expr, exp.Window):+ results = []+ results.extend(self._find_used_fields(expr.this, sources, withs))+ # PARTITION BY+ partition = expr.args.get("partition_by")+ if partition:+ for part in partition:+ results.extend(self._find_used_fields(part, sources, withs))+ # ORDER BY within window+ order = expr.args.get("order")+ if order:+ results.extend(self._find_used_fields(order, sources, withs))+ return results++ # ORDER BY clause+ if isinstance(expr, exp.Order):+ results = []+ for ordered in expr.expressions:+ results.extend(self._find_used_fields(ordered.this, sources, withs))+ return results++ # GROUP BY clause+ if isinstance(expr, exp.Group):+ results = []+ for item in expr.expressions:+ results.extend(self._find_used_fields(item, sources, withs))+ return results++ # Ordered (for ORDER BY items)+ if isinstance(expr, exp.Ordered):+ return self._find_used_fields(expr.this, sources, withs)++ # Cast+ if isinstance(expr, exp.Cast):+ return self._find_used_fields(expr.this, sources, withs)++ # Paren (parenthesized expression)+ if isinstance(expr, exp.Paren):+ return self._find_used_fields(expr.this, sources, withs)++ # Interval, Literal, etc. - no columns+ if isinstance(expr, (exp.Literal, exp.Interval, exp.DataType, exp.Boolean)):+ return []++ # Identifier (column name without table) - treat as column+ if isinstance(expr, exp.Identifier):+ # Create a fake column to reuse column resolution+ fake_col = exp.Column(this=expr)+ return self._get_column(sources, fake_col, return_table_matches=False)++ # Fallback: try to walk children+ results = []+ for child in expr.iter_expressions():+ results.extend(self._find_used_fields(child, sources, withs))+ return results++ def _get_column(self, sources, column_expr, return_table_matches):+ """Get column reference with source resolution."""+ column_name = column_expr.name+ table_ref = column_expr.table++ # Find valid sources+ if table_ref:+ source = self._find_source({"table": table_ref}, sources)+ valid_sources = [[source]] if source else [[]]+ else:+ valid_sources = sources++ # Build source_columns (nested list of possible sources)+ # Pass through returned_fields from all sources (matching Macaw behavior).+ # This preserves lineage back to original tables through CTEs/subqueries.+ source_columns = []+ for scope_sources in valid_sources:+ if isinstance(scope_sources, list):+ scope_fields = []+ for source in scope_sources:+ if source and self._source_might_have_column(source, column_name, table_ref):+ returned_fields = source.get("returned_fields", [])+ scope_fields.extend(returned_fields)+ if scope_fields:+ source_columns.append(scope_fields)++ # Check for direct column match in source_columns (matching Macaw)+ # Search in (first source_columns), not in the raw returned_fields+ source_column = None+ if source_columns:+ for field in source_columns[0]:+ field_name = field.get("alias") or field.get("column")+ if field_name and field_name.lower() == column_name.lower():+ source_column = field+ break++ # Check for table alias match (SELECT o FROM (SELECT * FROM orders) AS o)+ if not source_column and not table_ref:+ for scope_sources in valid_sources:+ if isinstance(scope_sources, list):+ for source in scope_sources:+ if source:+ names = source.get("names", {})+ table_alias = names.get("table_alias") if names else None+ table_name = names.get("table") if names else None+ if table_alias and table_alias.lower() == column_name.lower():+ if return_table_matches:+ return [{+ "col": {+ "type": "custom_field",+ "alias": column_name,+ # Don't freeze - goes directly to returned_fields+ "used_fields": list(source.get("returned_fields", []))+ }+ }]+ else:+ # Table alias match but not returning table matches - empty result+ return []+ if table_name and table_name.lower() == column_name.lower():+ if return_table_matches:+ return [{+ "col": {+ "type": "custom_field",+ "alias": column_name,+ # Don't freeze - goes directly to returned_fields+ "used_fields": list(source.get("returned_fields", []))+ }+ }]+ else:+ # Table name match but not returning table matches - empty result+ return []++ # Direct column match handling+ # The source_columns in single_column entries already point to the correct source.+ # For custom_field (computed columns like "0 AS level"), we need to create a+ # single_column reference pointing to the source (e.g., the CTE).+ if source_column:+ col_type = source_column.get("type")+ if col_type == "single_column":+ # For recursive CTE self-references (like h.id in recursive CTE),+ # create a CTE reference. For non-recursive CTEs, pass through the+ # matched column (which already has correct source_columns).+ if table_ref and not return_table_matches:+ source = self._find_source({"table": table_ref}, valid_sources)+ # Only create CTE reference for RECURSIVE CTEs+ if source and source.get("is_recursive_cte"):+ names = source.get("names") or {}+ table_info = {"table": names.get("table", table_ref)}+ if names.get("table_alias"):+ table_info["table_alias"] = names["table_alias"]+ return [{+ "col": {+ "type": "single_column",+ "column": column_name,+ "alias": None,+ "source_columns": [[{"type": "all_columns", "table": table_info}]]+ }+ }]+ # For non-recursive CTEs and other cases: pass through matched column+ result_col = dict(source_column)+ # For unqualified reference with multiple scopes: use multi-level source_columns+ if not table_ref and len(source_columns) > 1:+ result_col["source_columns"] = source_columns+ return [{"col": result_col}]+ elif col_type == "custom_field":+ # Custom field has no source_columns (it's a computed expression).+ # For recursive CTE self-references (like h.level in recursive CTE):+ # Create single_column with CTE reference+ # For other CTE references (like m.month referencing non-recursive CTE):+ # Pass through custom_field to preserve structure+ #+ # Detection: is_recursive_cte flag is set during CTE processing+ if table_ref:+ source = self._find_source({"table": table_ref}, valid_sources)+ if source and source.get("is_recursive_cte"):+ names = source.get("names") or {}+ table_info = {"table": names.get("table", table_ref)}+ if names.get("table_alias"):+ table_info["table_alias"] = names["table_alias"]+ source_ref = [[{"type": "all_columns", "table": table_info}]]+ return [{+ "col": {+ "type": "single_column",+ "column": column_name,+ "alias": None,+ "source_columns": source_ref+ }+ }]+ # Return custom_field as-is to preserve structure+ return [{"col": source_column}]+ # For composite_field, etc.: return as-is to preserve structure+ else:+ return [{"col": source_column}]++ # No direct match - return single_column with source_columns+ errors = set()+ if not source_columns or not any(source_columns):+ # No potential sources found+ if table_ref:+ # Table qualifier doesn't exist - return error only, no col spec+ # (Returning a col with empty source_columns would cause Clojure's+ # resolve-field to generate a redundant missing-column error)+ return [{"errors": {self._missing_table_alias_error(table_ref)}}]+ else:+ errors.add(self._missing_column_error(column_name))++ return [{+ "col": {+ "type": "single_column",+ "column": column_name,+ "alias": None,+ "source_columns": source_columns+ },+ "errors": errors+ }]++ def _find_source(self, search, sources):+ """Find a source matching the search criteria."""+ for scope_sources in sources:+ if isinstance(scope_sources, list):+ for source in scope_sources:+ if source and self._table_matches(search, source.get("names", {})):+ return source+ return None++ def _table_matches(self, search, table_names):+ """Check if search criteria match table names."""+ if not table_names:+ return False+ search_table = search.get("table")+ search_schema = search.get("schema")+ search_database = search.get("database")++ # Match by alias first (if no schema/database in search)+ if search_table and not search_schema and not search_database:+ if table_names.get("table_alias") == search_table:+ return True++ # Match by table/schema/database+ if search_table and search_table != table_names.get("table"):+ return False+ if search_schema and search_schema != table_names.get("schema"):+ return False+ if search_database and search_database != table_names.get("database"):+ return False++ return search_table == table_names.get("table")++ def _source_might_have_column(self, source, column_name, table_ref):+ """Check if a source might contain a column."""+ if not source:+ return False++ # If table ref specified, check if it matches (case-insensitive since+ # SQLGlot may lowercase identifiers)+ if table_ref:+ names = source.get("names", {})+ if names:+ table_alias = names.get("table_alias")+ table_name = names.get("table")+ if table_alias and table_alias.lower() == table_ref.lower():+ return True+ if table_name and table_name.lower() == table_ref.lower():+ return True+ return False++ # No table ref - source might have column+ return True++ def _freeze_field(self, field):+ """Convert a field dict to a hashable frozenset for use in sets."""+ if field is None:+ return None+ if isinstance(field, (str, int, float, bool)):+ return field+ if isinstance(field, frozenset):+ return field+ if isinstance(field, set):+ return frozenset(self._freeze_field(item) for item in field)+ if isinstance(field, (list, tuple)):+ return tuple(self._freeze_field(item) for item in field)+ if isinstance(field, dict):+ items = []+ for k, v in field.items():+ items.append((k, self._freeze_field(v)))+ return frozenset(items)+ return field++ def _unfreeze_field(self, frozen):+ """Convert a frozen field back to a dict."""+ if frozen is None:+ return None+ if isinstance(frozen, (str, int, float, bool)):+ return frozen++ # frozenset of tuples -> dict+ if isinstance(frozen, frozenset):+ # Check if it looks like a dict (frozenset of 2-tuples)+ try:+ items = list(frozen)+ if items and all(isinstance(item, tuple) and len(item) == 2 for item in items):+ # It's a frozen dict+ result = {}+ for k, v in items:+ result[k] = self._unfreeze_field(v)+ return result+ else:+ # It's a frozen set of items+ return [self._unfreeze_field(item) for item in frozen]+ except:+ return [self._unfreeze_field(item) for item in frozen]++ # tuple -> list+ if isinstance(frozen, tuple):+ return [self._unfreeze_field(item) for item in frozen]++ # list -> list (recursively unfreeze items)+ if isinstance(frozen, list):+ return [self._unfreeze_field(item) for item in frozen]++ # dict -> dict (recursively unfreeze values)+ if isinstance(frozen, dict):+ return {k: self._unfreeze_field(v) for k, v in frozen.items()}++ return frozen++ def _syntax_error(self):+ """Create a syntax error."""+ return frozenset([("type", "syntax_error")])++ def _missing_table_alias_error(self, table_name):+ """Create a missing table alias error."""+ return frozenset([("type", "missing_table_alias"), ("table", table_name)])++ def _missing_column_error(self, column_name):+ """Create a missing column error."""+ return frozenset([("type", "missing_column"), ("column", column_name)])diff --git a/src/metabase/actions/scope.clj b/src/metabase/actions/scope.cljindex b1f7afaa85b..a3132305f0f 100644--- a/src/metabase/actions/scope.clj+++ b/src/metabase/actions/scope.clj@@ -1,6 +1,5 @@(ns metabase.actions.scope(:require- [macaw.util :as u][metabase.actions.types :as types][metabase.lib.core :as lib][metabase.util.malli :as mu]@@ -48,12 +47,17 @@(:table-id scope)(update :database-id #(or % (t2/select-one-fn :db_id [:model/Table :db_id] (:table-id scope)) missing-id))))+(defn- strip-nils+ "Remove any keys corresponding to nil values from the given map."+ [m]+ (into {} (filter (comp some? val) m)))+(mu/defn hydrate-scope :- ::types/scope.hydrated"Add the implicit keys that can be derived from the existing ones in a scope. Idempotent."[scope :- ::types/scope.raw];; Infer type if FE is not passing it (yet)(let [scope-type (or (:type scope) (scope-type scope))]- (assoc (u/strip-nils+ (assoc (strip-nils;; Rerun until it converges.(ffirst (filter (partial apply =) (partition 2 1 (iterate hydrate-scope* scope))))):type scope-type)))diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex e8c547576bd..de28d960b96 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -500,7 +500,30 @@:labels [:status]})(prometheus/counter :metabase-token-check/attempt{:description "Total number of token checks. Includes a status label."- :labels [:status]})])+ :labels [:status]})+ ;; SQL parsing metrics+ (prometheus/counter :metabase-sql-parsing/context-timeouts+ {:description "Number of Python/GraalVM SQL parsing execution timeouts."})+ (prometheus/counter :metabase-sql-parsing/context-acquisitions+ {:description "Number of Python contexts acquired from the pool."})+ (prometheus/counter :metabase-sql-parsing/context-creations+ {:description "Number of new Python contexts created."})+ (prometheus/counter :metabase-sql-parsing/context-disposals-expired+ {:description "Number of Python contexts disposed due to TTL expiry."})+ ;; SQL tools operation metrics+ (prometheus/counter :metabase-sql-tools/operations-total+ {:description "Total number of sql-tools operations started."+ :labels [:parser :operation]})+ (prometheus/counter :metabase-sql-tools/operations-completed+ {:description "Number of sql-tools operations completed successfully."+ :labels [:parser :operation]})+ (prometheus/counter :metabase-sql-tools/operations-failed+ {:description "Number of sql-tools operations that threw an exception."+ :labels [:parser :operation]})+ (prometheus/histogram :metabase-sql-tools/operation-duration-ms+ {:description "Duration in milliseconds of sql-tools operations."+ :labels [:parser :operation]+ :buckets [1 5 10 25 50 100 250 500 1000 2500 5000 10000 30000]})])(defn- quartz-collectors[]diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.cljindex 87b588c5bb0..3fc57750572 100644--- a/src/metabase/core/init.clj+++ b/src/metabase/core/init.clj@@ -57,6 +57,7 @@[metabase.session.init][metabase.settings.init][metabase.setup.init]+ [metabase.sql-tools.init][metabase.sso.init][metabase.store-api.init][metabase.sync.init]diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex 990008aace7..66ec7de2e0c 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -3,9 +3,6 @@(:refer-clojure :exclude [some])(:require[clojure.set :as set]- ;; TODO (Cam 10/1/25) -- Isn't having drivers use Macaw directly against the spirt of all the work we did to make a- ;; Driver API namespace?- [macaw.core :as macaw][metabase.driver :as driver][metabase.driver-api.core :as driver-api]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse]@@ -14,13 +11,11 @@[metabase.driver.sql.parameters.substitute :as sql.params.substitute][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp]- [metabase.driver.sql.references :as sql.references][metabase.driver.sql.util :as sql.u][metabase.driver.util :as driver.u]- [metabase.util.humanization :as u.humanization]+ [metabase.sql-tools.core :as sql-tools][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [some]][potemkin :as p]))(comment sql.params.substitution/keep-me) ; this is so `cljr-clean-ns` and the linter don't remove the `:require`@@ -226,184 +221,35 @@[_]"public")-(defn find-table-or-transform- "Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.- It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."- [driver tables transforms {search-table :table raw-schema :schema}]- (let [search-schema (or raw-schema- (default-schema driver))- matches? (fn [db-table db-schema]- (and (= search-table db-table)- (= search-schema db-schema)))]- (or (some (fn [{:keys [name schema id]}]- (when (matches? name schema)- {:table id}))- tables)- (some (fn [{:keys [id] {:keys [name schema]} :target}]- (when (matches? name schema)- {:transform id}))- transforms))))--(defn- normalize-table-spec- [driver {:keys [table schema]}]- {:table (sql.normalize/normalize-name driver table)- :schema (some->> schema (sql.normalize/normalize-name driver))})--(defn- parsed-table-refs- "Parse a native query and return a sequence of normalized table specs {:table ... :schema ...}."- [driver query]- (-> query- driver-api/raw-native-query- (driver.u/parsed-query driver)- (macaw/query->components {:strip-contexts? true})- :tables- (->> (map :component)- (map #(normalize-table-spec driver %)))))+;;; +----------------------------------------------------------------------------------------------------------------++;;; | Dependencies |+;;; +----------------------------------------------------------------------------------------------------------------+(mu/defmethod driver/native-query-table-refs :sql :- ::driver/native-query-table-refs[driver :- :keywordquery :- :metabase.lib.schema/native-only-query]- (into #{} (parsed-table-refs driver query)))+ (into #{} (sql-tools/referenced-tables-raw driver (driver-api/raw-native-query query))))(mu/defmethod driver/native-query-deps :sql :- ::driver/native-query-deps[driver :- :keywordquery :- :metabase.lib.schema/native-only-query]- (let [db-tables (driver-api/tables query)- db-transforms (driver-api/transforms query)]- (into (driver-api/native-query-table-references query)- (keep #(find-table-or-transform driver db-tables db-transforms %)- (parsed-table-refs driver query)))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | Dependencies |-;;; +----------------------------------------------------------------------------------------------------------------+--(defmulti ^:private resolve-field- "Resolves a field reference to one or more actual database fields.-- This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field- references returned by sql.references/field-references.-- Note: this currently sets :lib/desired-column-alias but no other :lib/* fields, because the callers of this function- don't need the other fields. If we care about other :lib/* fields in the future, we can add them then."- {:added "0.57.0" :arglists '([driver metadata-provider col-spec])}- (fn [_driver _metadata-provider col-spec]- (:type col-spec)))--(defmethod resolve-field :all-columns- [driver metadata-provider col-spec]- (or (some->> (:table col-spec)- (find-table-or-transform driver (driver-api/tables metadata-provider) (driver-api/transforms metadata-provider))- :table- (driver-api/active-fields metadata-provider)- (map #(-> (assoc % :lib/desired-column-alias (:name %))- sql.references/wrap-col)))- [{:error (driver-api/missing-table-alias-error- (sql.references/table-name (:table col-spec)))}]))--(defmethod resolve-field :single-column- [driver metadata-provider {:keys [alias] :as col-spec}]- [(if-let [{:keys [name] :as found}- (->> (:source-columns col-spec)- (some (fn [source-col-set]- ;; in cases like `select (select blah from ...) from ...`, if blah refers to a- ;; column in both the inner query and the outer query, the column from the inner- ;; query will be preferred. However, if blah doesn't refer to something in the- ;; inner query, it can also refer to something in the outer query.- ;; sql.references/field-references organizes source-cols into a list of lists- ;; to account for this.- (->> (mapcat (fn [current-col]- ;; :unknown-columns is a placeholder for "we know there are columns being- ;; returned, but have no way of knowing what those are -- this is primarily- ;; used for table-functions like `select * from my_func()`. If we encounter- ;; something like that, assume that the query is valid and make up a matching- ;; column to avoid false positives.- (if (= (:type current-col) :unknown-columns)- (let [name (:column col-spec)]- [{:base-type :type/*- :name name- :display-name (->> name (u.humanization/name->human-readable-name :simple))- :effective-type :type/*- :semantic-type :Semantic/*}])- (keep :col (resolve-field driver metadata-provider current-col))))- source-col-set)- (some #(when (= (:name %) (:column col-spec))- %))))))]- {:col (assoc found :lib/desired-column-alias (or alias name))}- {:error (driver-api/missing-column-error (:column col-spec))})])--(defn- get-name [m]- (or (:alias m) (str (gensym "new-col"))))--(defn- get-display-name [m]- (->> (get-name m)- (u.humanization/name->human-readable-name :simple)))--(defmethod resolve-field :custom-field- [_driver _metadata-provider col-spec]- [{:col {:base-type :type/*- :name (get-name col-spec)- :lib/desired-column-alias (get-name col-spec)- :display-name (get-display-name col-spec)- :effective-type :type/*- :semantic-type :Semantic/*}}])--(defn- lca [default-type & types]- (let [ancestor-sets (for [t types- :when t]- (conj (set (ancestors t)) t))- common-ancestors (when (seq ancestor-sets)- (apply set/intersection ancestor-sets))]- (if (seq common-ancestors)- (apply (partial max-key (comp count ancestors)) common-ancestors)- default-type)))--(defmethod resolve-field :composite-field- [driver metadata-provider col-spec]- (let [member-fields (mapcat #(->> (resolve-field driver metadata-provider %)- (keep :col))- (:member-fields col-spec))]- [{:col {:name (get-name col-spec)- :lib/desired-column-alias (get-name col-spec)- :display-name (get-display-name col-spec)- :base-type (apply lca :type/* (map :base-type member-fields))- :effective-type (apply lca :type/* (map :effective-type member-fields))- :semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}}]))--(defmethod resolve-field :unknown-columns- [_driver _metadata-provider _col-spec]- [])+ (into (driver-api/native-query-table-references query)+ (sql-tools/referenced-tables driver query)))(mu/defmethod driver/native-result-metadata :sql[driver :- :keywordnative-query :- :metabase.lib.schema/native-only-query]- (let [{:keys [returned-fields]} (-> native-query- driver-api/raw-native-query- (driver.u/parsed-query driver)- macaw/->ast- (->> (sql.references/field-references driver)))]- (mapcat #(->> (resolve-field driver native-query %)- (keep :col))- returned-fields)))+ (sql-tools/returned-columns driver native-query))(mu/defmethod driver/validate-native-query-fields :sql :- [:set [:ref driver-api/schema.validate.error]][driver :- :keywordnative-query :- :metabase.lib.schema/native-only-query]- (let [{:keys [used-fields returned-fields errors]} (-> native-query- driver-api/raw-native-query- (driver.u/parsed-query driver)- macaw/->ast- (->> (sql.references/field-references driver)))- check-fields #(mapcat (fn [col-spec]- (->> (resolve-field driver (driver-api/->metadata-provider native-query) col-spec)- (keep :error)))- %)]- (-> errors- (into (check-fields used-fields))- (into (check-fields returned-fields)))))+ (sql-tools/validate-query driver native-query));;; +----------------------------------------------------------------------------------------------------------------+;;; | Convenience Imports |;;; +----------------------------------------------------------------------------------------------------------------+-(p/import-vars [sql.params.substitution ->prepared-substitution PreparedStatementSubstitution])+(p/import-vars+ [sql.params.substitution ->prepared-substitution PreparedStatementSubstitution]+ [sql.normalize normalize-name reserved-literal])diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.cljindex 58f29f7de2e..d621710d5a9 100644--- a/src/metabase/driver/util.clj+++ b/src/metabase/driver/util.clj@@ -5,7 +5,6 @@[clojure.core.memoize :as memoize][clojure.set :as set][clojure.string :as str]- [macaw.core :as macaw][metabase.app-db.core :as mdb][metabase.auth-provider.core :as auth-provider][metabase.config.core :as config]@@ -799,80 +798,11 @@;;; | Macaw parsing helpers |;;; +----------------------------------------------------------------------------------------------------------------+-(def ^:private considered-drivers- "Since we are unable to ask basic questions of the driver hierarchy outside of that module, we need to explicitly- mention all sub-types. This is probably not a bad thing."- #{:h2 :mysql :postgres :redshift :sqlite :sqlserver})-;; At some point, we may want a way for 3rd party drivers to opt in, but a public API deserves some hammock time.-;; Using a separate list to the above, as we may want this to be more restrictive.(def trusted-for-table-permissions?"Do we trust that Macaw will not give us false negatives for tables referenced by a given query?"#{:h2 :mysql :postgres})-(defn macaw-options- "Generate the options expected by Macaw based on the nature of the given driver."- [driver]- (merge- ;; If this isn't a driver we've considered, fallback to Macaw's conservative defaults.- (when (contains? considered-drivers driver)- {;; According to the SQL-92 specification, non-quoted identifiers should be case-insensitive, and the majority of- ;; engines are implemented this way.- ;;- ;; In practice there are exceptions, notably MySQL and SQL Server, where case sensitivity is a property of the- ;; underlying resource referenced by the identifier, and the case-sensitivity does not depend on whether the- ;; reference is quoted.- ;;- ;; For MySQL the case sensitivity of databases and tables depends on both the underlying file system, and a system- ;; variable used to initialize the database. For SQL Server it depends on the collation settings of the collection- ;; where the corresponding schema element is defined.- ;;- ;; For MySQL, columns and aliases can never be case-sensitive, and for SQL Server the default collation is case-- ;; insensitive too, so it makes sense to just treat all databases as case-insensitive as a whole.- ;;- ;; In future, Macaw may support discriminating on the identifier type, in which case we could be more precise for- ;; these databases. Being 100% correct would require querying system variables and schema configuration however,- ;; which is likely a step too far in complexity.- ;;- ;; Currently, we go with :agnostic, as it is the most relaxed semantics (the case of both the identifiers and the- ;; underlying schema is totally ignored, and correspondence is non-deterministic), but Macaw supports more nuanced- ;; :lower and :upper configuration values which coerce the query identifiers to a given case then do an exact- ;; comparison with the schema.- :case-insensitive :agnostic- ;; For both MySQL and SQL Server, whether identifiers are case-sensitive depends on database configuration only,- ;; and quoting has no effect on this, so we disable this option for consistency with `:case-insensitive`.- :quotes-preserve-case? (not (contains? #{:mysql :sqlserver} driver))- :features {:postgres-syntax (isa? driver/hierarchy driver :postgres)- :square-bracket-quotes (= :sqlserver driver)- :unsupported-statements false- :backslash-escape-char true- ;; This will slow things down, but until we measure the difference, opt for correctness.- :complex-parsing true}- ;; 10 seconds- :timeout 10000})- {;; There is no plan to be exhaustive yet.- ;; Note that while an allowed list would be more conservative, at the time of writing only 1 of the bundled- ;; drivers use FINAL as a reserved word, and mentioning them all would be prohibitive.- ;; In the future, we will use multimethods to define this explicitly per driver, or even discover it automatically- ;; through the JDBC connection, where possible.- :non-reserved-words (vec (remove nil? [(when-not (contains? #{:clickhouse} driver)- :final)]))}))--(defn parsed-query- "Wrapped for `parsed-query` providing default options and throwing exceptions on parsing failures."- [sql driver & {:as opts}]- (let [result- #_{:clj-kondo/ignore [:discouraged-var]}- (macaw/parsed-query sql (merge (macaw-options driver)- opts))]- ;; TODO (lbrdnk 2026-01-23): In follow-up work we should ensure that failure to parse is not silently swallowed.- ;; I'm leaving that off at the moment to avoid potential log flooding.- #_(when (and (map? result) (some? (:error result)))- (throw (ex-info "SQL parsing failed."- {:macaw-error (:error result)}- (-> result :context :cause))))- result))-(def ^:const transform-temp-table-prefix"Prefix used for temporary tables created during transform execution.""mb_transform_temp_table")diff --git a/src/metabase/llm/context.clj b/src/metabase/llm/context.cljindex f4744eb3433..9ae2d9dc0b6 100644--- a/src/metabase/llm/context.clj+++ b/src/metabase/llm/context.clj@@ -3,12 +3,12 @@and fetches table metadata formatted as DDL for SQL generation."(:require[clojure.string :as str]- [macaw.core :as macaw][metabase.api.common :as api][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.models.interface :as mi]+ [metabase.sql-tools.core :as sql-tools][metabase.sync.core :as sync][metabase.util :as u][metabase.util.log :as log]@@ -52,7 +52,7 @@[:= [:lower :name] table-lower])))(defn extract-tables-from-sql- "Extract table IDs from a raw SQL string using Macaw parser.+ "Extract table IDs from a raw SQL string.Parses the SQL to identify referenced table names, then queries thedatabase to resolve those names to table IDs. When the SQL includes@@ -64,8 +64,8 @@[database-id sql-string](if (and database-id (seq sql-string))(try- (let [result (macaw/query->tables sql-string {:mode :compound-select})- tables (:tables result)]+ (let [driver (t2/select-one-fn :engine :model/Database :id database-id)+ tables (sql-tools/referenced-tables-raw driver sql-string)](if (seq tables)(let [match-clauses (mapv table-match-clause tables)matched-tables (t2/select :model/Tablediff --git a/src/metabase/sql_parsing/common.clj b/src/metabase/sql_parsing/common.cljnew file mode 100644index 00000000000..428f92cc419--- /dev/null+++ b/src/metabase/sql_parsing/common.clj@@ -0,0 +1,8 @@+(ns metabase.sql-parsing.common+ (:require+ [potemkin :as p]))++(p/defprotocol+ PythonEval+ "Protocol for evaluating Python code. Abstracts over raw Context and pooled contexts."+ (eval-python [this code]+ "Evaluate Python code."))diff --git a/src/metabase/sql_parsing/core.clj b/src/metabase/sql_parsing/core.cljnew file mode 100644index 00000000000..7fc6385146a--- /dev/null+++ b/src/metabase/sql_parsing/core.clj@@ -0,0 +1,395 @@+(ns metabase.sql-parsing.core+ "Stateless SQL parsing via sqlglot (Python) and GraalVM Polyglot.++ This module provides dialect-aware SQL parsing with no Metabase dependencies.+ All functions take strings and return strings/simple data structures.++ API:+ (referenced-tables sql dialect) → [[catalog schema table] ...]+ (referenced-fields dialect sql) → [[catalog schema table field] ...]+ (returned-columns-lineage dialect sql schema schema-map) → [[col pure? deps] ...]+ (validate-query dialect sql schema schema-map) → {:status :ok} | {:status :error ...}"+ (:require+ [clojure.string :as str]+ [metabase.analytics.core :as analytics]+ [metabase.sql-parsing.common :as common]+ [metabase.sql-parsing.pool :as python.pool]+ [metabase.util.json :as json]+ [metabase.util.log :as log])+ (:import+ (java.io Closeable)+ (java.util.concurrent ExecutionException TimeoutException)+ (org.graalvm.polyglot Value)))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Timeout handling --------------------------------------------------++(def ^:private ^:const default-timeout-ms+ "Default timeout for Python operations in milliseconds.+ GraalVM Python can occasionally hang (DEV-1393), so we wrap calls with a timeout."+ 30000) ; 30 seconds++(defn- with-timeout*+ "Execute f in a future with timeout. On timeout, throws TimeoutException.+ The caller is responsible for cleaning up resources (e.g., disposing context)."+ [timeout-ms f]+ (let [fut (future (f))]+ (try+ (deref fut timeout-ms ::timeout)+ (catch ExecutionException e+ ;; Unwrap execution exception to get the real cause+ (throw (or (.getCause e) e)))+ (finally+ ;; If we timed out or got an exception, try to cancel the future+ ;; Note: This won't actually interrupt GraalVM, but prevents resource leaks+ (future-cancel fut)))))++(defmacro ^:private with-python-timeout+ "Execute body with a timeout. If timeout is reached:+ 1. Interrupts the GraalVM context (actually stops execution)+ 2. Poisons context so it gets disposed rather than returned to pool+ 3. Logs warning and throws TimeoutException"+ [ctx timeout-ms & body]+ `(let [result# (with-timeout* ~timeout-ms (^:once fn* [] ~@body))]+ (if (= result# ::timeout)+ (do+ ;; Actually interrupt the GraalVM context (1s grace period for soft interrupt)+ ;; This is necessary because future-cancel doesn't stop GraalVM execution+ (python.pool/interrupt! ~ctx 1000)+ (python.pool/poison! ~ctx)+ (analytics/inc! :metabase-sql-parsing/timeouts)+ (log/warn "Python execution timed out after" ~timeout-ms "ms - GraalVM interrupted")+ (throw (TimeoutException. (str "Python execution timed out after " ~timeout-ms "ms"))))+ result#)))++;;; -------------------------------------------------- Public API --------------------------------------------------++(defn referenced-tables+ "Extract table references from SQL.++ Returns a vector of [catalog schema table] 3-tuples:+ [[nil nil \"users\"] [nil \"public\" \"orders\"] [\"myproject\" \"analytics\" \"events\"]]++ This is the pure parsing layer - it returns what's literally in the SQL.+ Default schema resolution happens in the matching layer (core.clj)."+ [dialect sql]+ (-> (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ (-> ^Value (common/eval-python ctx "sql_tools.referenced_tables")+ (.execute ^Value (object-array [sql dialect]))+ .asString)))+ json/decode+ vec))++(defn referenced-fields+ "Extract field references from SQL, returning only fields from actual database tables.++ Returns a vector of [catalog schema table field] 4-tuples:+ [[nil nil \"users\" \"id\"] [nil \"public\" \"orders\" \"total\"]]++ Includes:+ - Wildcards as [catalog schema table \"*\"]+ - All specific column references++ Excludes:+ - Fields from CTEs or subqueries+ - Table aliases (returns actual table names)++ Examples:+ (referenced-fields \"postgres\" \"SELECT id FROM users\")+ => [[nil nil \"users\" \"id\"]]++ (referenced-fields \"postgres\" \"SELECT * FROM public.users\")+ => [[nil \"public\" \"users\" \"*\"]]++ (referenced-fields \"bigquery\" \"SELECT * FROM myproject.analytics.events\")+ => [[\"myproject\" \"analytics\" \"events\" \"*\"]]"+ [dialect sql]+ (-> (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ (-> ^Value (common/eval-python ctx "sql_tools.referenced_fields")+ (.execute ^Value (object-array [sql dialect]))+ .asString)))+ json/decode+ vec))++(defn returned-columns-lineage+ "Extract column lineage from SQL query, showing which output columns depend on which source columns.++ Returns a vector of [alias pure? [[schema table col]...]] tuples:+ - alias: The output column name/alias+ - pure?: Boolean - true if the column is a direct pass-through from a source column+ - deps: Vector of [schema table column] dependencies++ Requires a schema map of the form:+ {\"schema_name\" {\"table_name\" {\"column_name\" \"TYPE\"}}}++ Examples:+ (returned-columns-lineage \"postgres\" \"SELECT id FROM users\" nil {nil {\"users\" {\"id\" \"INT\"}}})+ => [[\"id\" true [[[nil \"users\" \"id\"]]]]]++ (returned-columns-lineage \"postgres\" \"SELECT id + 1 as computed FROM users\" nil schema)+ => [[\"computed\" false [[[nil \"users\" \"id\"]]]]]"+ [dialect sql default-table-schema sqlglot-schema]+ (-> (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues+ (-> ^Value (common/eval-python ctx "sql_tools.returned_columns_lineage")+ (.execute ^Value (object-array [dialect+ sql+ default-table-schema+ (json/encode sqlglot-schema)]))+ .asString)))+ json/decode+ vec))++(defn validate-query+ "Validate a SQL query against a schema using sqlglot's qualify optimizer.++ Operates in two modes based on whether a schema is provided:++ **Strict mode** (sqlglot-schema provided):+ Validates that column and table references exist in the provided schema.+ Returns errors for unknown tables, unresolved columns, missing table aliases.++ **Permissive mode** (sqlglot-schema is nil or empty):+ Only checks SQL syntax. Infers schema from query structure.+ Useful for UDTFs and queries where the schema is unknown.++ Parameters:+ - dialect: SQLGlot dialect string (e.g., \"postgres\", \"mysql\"), or nil+ - sql: The SQL query string to validate+ - default-table-schema: Default schema name for unqualified table references+ - sqlglot-schema: Schema map of {schema-name {table-name {column-name type}}},+ or nil/empty for permissive mode++ Returns a map with:+ - If valid: {:status \"ok\"}+ - If error: {:status \"error\", :type \"...\", :message \"...\", ...}++ Error types (strict mode):+ - \"unknown_table\": Table not found in schema+ - \"column_not_resolved\": Column not found (includes :column key)+ - \"invalid_expression\": Syntax/parse error+ - \"unhandled\": Other errors"+ [dialect sql default-table-schema & [sqlglot-schema]]+ (-> (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ ;; JSON-encode schema to avoid GraalVM polyglot map conversion issues+ (-> ^Value (common/eval-python ctx "sql_tools.validate_query")+ (.execute ^Value (object-array [dialect sql default-table-schema (json/encode (or sqlglot-schema "{}"))]))+ .asString)))+ json/decode+kw))++(defn simple-query?+ "Check if SQL is a simple SELECT without LIMIT, OFFSET, or CTEs.++ Used by Workspaces to determine if automatic checkpoints can be inserted.++ Parameters:+ - dialect: SQLGlot dialect string (e.g., \"postgres\", \"mysql\"), or nil for default+ - sql: The SQL query string to check++ Returns a map with:+ - :is_simple - boolean indicating if query is simple+ - :reason - string explaining why query is not simple (when false)++ Examples:+ (simple-query? \"postgres\" \"SELECT * FROM users\")+ => {:is_simple true}++ (simple-query? nil \"SELECT * FROM users LIMIT 10\")+ => {:is_simple false :reason \"Contains a LIMIT\"}"+ [dialect sql]+ (-> (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ (-> ^Value (common/eval-python ctx "sql_tools.simple_query")+ (.execute ^Value (object-array [sql dialect]))+ .asString)))+ json/decode+kw))++(defn add-into-clause+ "Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.++ Transforms: 'SELECT * FROM products'+ Into: 'SELECT * INTO \"TABLE\" FROM products'++ Used by SQL Server compile-transform which requires SELECT INTO syntax+ instead of CREATE TABLE AS SELECT.++ Parameters:+ - dialect: SQLGlot dialect string (e.g., \"tsql\" for SQL Server)+ - sql: The SELECT SQL query string+ - table-name: The target table name (already formatted/quoted)++ Returns: Modified SQL string with INTO clause"+ [dialect sql table-name]+ (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ (-> ^Value (common/eval-python ctx "sql_tools.add_into_clause")+ (.execute ^Value (object-array [sql table-name dialect]))+ .asString))))++(defn- convert-field-type+ "Convert field type string (snake_case) to keyword (kebab-case)."+ [type-str]+ (-> type-str+ (str/replace "_" "-")+ keyword))++(defn- convert-error+ "Convert Python error format to Metabase lib error format."+ [error]+ (let [;; Get type from various possible key formats+ err-type (or (:type error) (get error "type"))+ ;; Get name/table/column from various possible formats+ table-name (or (:table error) (get error "table"))+ column-name (or (:column error) (get error "column") (:name error) (get error "name"))]+ (cond+ ;; Python frozenset came through as vector of pairs: [["type" "syntax_error"]]+ (and (sequential? error) (sequential? (first error)))+ (let [m (into {} error)+ err-type (get m "type")]+ (case err-type+ "syntax_error" {:type :syntax-error}+ "missing_column" {:type :missing-column :name (get m "column")}+ "missing_table_alias" {:type :missing-table-alias :name (get m "table")}+ {:type (keyword err-type)}))++ ;; Handle string or keyword types+ (or (string? err-type) (keyword? err-type))+ (case (if (keyword? err-type) (name err-type) err-type)+ ("syntax-error" "syntax_error") {:type :syntax-error}+ ("missing-column" "missing_column") {:type :missing-column :name column-name}+ ("missing-table-alias" "missing_table_alias") {:type :missing-table-alias :name table-name}+ {:type (keyword err-type)})++ :else error)))++(defn- convert-field+ "Convert a field spec from Python format to Clojure format."+ [field]+ (when field+ (let [field-type (some-> (or (:type field) (get field "type")) convert-field-type)]+ (case field-type+ :single-column+ {:type :single-column+ :column (or (:column field) (get field "column"))+ :alias (or (:alias field) (get field "alias"))+ :source-columns (mapv (fn [scope]+ (mapv convert-field scope))+ (or (:source-columns field)+ (:source_columns field)+ (get field "source_columns")+ []))}++ :all-columns+ {:type :all-columns+ :table (let [t (or (:table field) (get field "table"))]+ (cond-> {}+ (or (:table t) (get t "table"))+ (assoc :table (or (:table t) (get t "table")))+ (or (:schema t) (get t "schema"))+ (assoc :schema (or (:schema t) (get t "schema")))+ (or (:database t) (get t "database"))+ (assoc :database (or (:database t) (get t "database")))+ (or (:table-alias t) (:table_alias t) (get t "table_alias"))+ (assoc :table-alias (or (:table-alias t) (:table_alias t) (get t "table_alias")))))}++ :custom-field+ {:type :custom-field+ :alias (or (:alias field) (get field "alias"))+ :used-fields (set (map convert-field+ (or (:used-fields field)+ (:used_fields field)+ (get field "used_fields")+ [])))}++ :composite-field+ {:type :composite-field+ :alias (or (:alias field) (get field "alias"))+ :member-fields (mapv convert-field+ (or (:member-fields field)+ (:member_fields field)+ (get field "member_fields")+ []))}++ :unknown-columns+ {:type :unknown-columns}++ ;; Fallback - return as-is with type conversion+ (assoc field :type field-type)))))++(defn field-references+ "Extract field references from SQL, returning used and returned fields.++ This is the SQLGlot equivalent of Macaw's field-references function.+ Returns a map with:+ - :used-fields - set of field specs from WHERE, JOIN ON, GROUP BY, ORDER BY+ - :returned-fields - vector of field specs from SELECT clause (ordered)+ - :errors - set of validation errors++ Each field spec has:+ - :type - :single-column, :all-columns, :custom-field, :composite-field, or :unknown-columns+ - :column - column name (for single-column)+ - :alias - column alias (nil if none)+ - :source-columns - nested list of possible source columns+ - :table - table info (for all-columns)+ - :used-fields - set of fields used (for custom-field)+ - :member-fields - list of fields (for composite-field)++ On timeout, returns an error map instead of throwing, consistent with the+ 'fail soft' pattern used for parsing failures."+ [dialect sql]+ (try+ (let [raw (-> (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ (-> ^Value (common/eval-python ctx "sql_tools.field_references")+ (.execute ^Value (object-array [sql dialect]))+ .asString)))+ json/decode+kw)+ used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])+ returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])+ errors (or (:errors raw) (get raw "errors") [])]+ {:used-fields (set (map convert-field used-fields))+ :returned-fields (vec (map convert-field returned-fields))+ :errors (set (map convert-error errors))})+ (catch TimeoutException e+ {:used-fields #{}+ :returned-fields []+ :errors #{{:type :timeout :message (.getMessage e)}}})))++(defn replace-names+ "Replace schema, table, and column names in SQL.++ Parameters:+ - dialect: SQLGlot dialect string (e.g., \"postgres\", \"mysql\"), or nil+ - sql: The SQL query string+ - replacements: A map with optional keys:+ - :schemas - map of old-schema-name -> new-schema-name+ - :tables - seq of [[{:schema s :table t} new-name] ...]+ - :columns - seq of [[{:schema s :table t :column c} new-name] ...]++ Returns modified SQL string.++ SECURITY: Replacement values are injected into the SQL AST as identifier names+ without sanitization. Callers MUST ensure replacement values are system-generated.+ See sql_tools.py replace_names for details.++ Examples:+ (replace-names \"postgres\" \"SELECT * FROM people\" {:tables [[{:table \"people\"} \"users\"]]})+ => \"SELECT * FROM users\""+ [dialect sql replacements]+ (with-open [^Closeable ctx (python.pool/python-context)]+ (with-python-timeout ctx default-timeout-ms+ (-> ^Value (common/eval-python ctx "sql_tools.replace_names")+ (.execute ^Value (object-array [sql (json/encode replacements) dialect]))+ .asString))))++(comment+ (referenced-tables "postgres" "select * from transactions")++ (validate-sql-query "postgres" "SELECT * FROM users")++ (referenced-fields "postgres" "SELECT id, name FROM users WHERE active = true"))diff --git a/src/metabase/sql_parsing/pool.clj b/src/metabase/sql_parsing/pool.cljnew file mode 100644index 00000000000..58723e15e2c--- /dev/null+++ b/src/metabase/sql_parsing/pool.clj@@ -0,0 +1,316 @@+(ns metabase.sql-parsing.pool+ "Namespace to handle python environment. Has only a single public function, [[python-context]] which will return a+ python context. This is an internal detail to the sql-parsing module. If you want something from this, add a+ function to the core namespace that uses this. No one should know about this implmentation detail."+ (:require+ [clojure.java.io :as io]+ [clojure.java.shell :as shell]+ [clojure.string :as str]+ [metabase.analytics.core :as analytics]+ [metabase.sql-parsing.common :as common]+ [metabase.util.files :as u.files]+ [metabase.util.log :as log]+ [potemkin :as p])+ (:import+ (io.aleph.dirigiste+ IPool$Controller+ IPool$Generator+ Pool+ Pools)+ (java.io Closeable File)+ (java.nio.file Files Path)+ (java.time Duration)+ (java.util.concurrent TimeUnit TimeoutException)+ (org.graalvm.polyglot Context HostAccess)))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Python path resolution --------------------------------------------------++(def ^:private python-sources-resource+ "Resource path for Python sources (sql_tools.py and sqlglot)."+ "python-sources")++(defn- plugins-dir-path+ "Get path to plugins directory. Lightweight alternative to metabase.plugins.core/plugins-dir+ that avoids loading the entire plugin system (~8 seconds of namespace loading)."+ ^Path []+ (let [dir-name (or (System/getenv "MB_PLUGINS_DIR")+ "plugins")+ path (u.files/get-path dir-name)]+ (u.files/create-dir-if-not-exists! path)+ path))++(defn- jar-resource?+ "True if the given resource is inside a JAR (production), false if from filesystem (dev)."+ [resource]+ (boolean+ (when-let [url (io/resource resource)]+ (.contains (.getFile ^java.net.URL url) ".jar!/"))))++(defn- copy-dir-recursive!+ "Recursively copy all files from source Path to dest Path.+ Works across filesystems (e.g., from JAR filesystem to default filesystem)."+ [^Path source ^Path dest]+ (doseq [^Path child (u.files/files-seq source)]+ (let [relative-name (str (.getFileName child))+ target (.resolve dest relative-name)]+ (if (Files/isDirectory child (into-array java.nio.file.LinkOption []))+ (do+ (u.files/create-dir-if-not-exists! target)+ (copy-dir-recursive! child target))+ (log/with-no-logs ;; <-- Suppress per-file logs from copy-file! (hundreds of files in sqlglot)+ (u.files/copy-file! child target))))))++(defn- extract-python-sources!+ "Extract python-sources from JAR to plugins directory. Returns the path as a string.+ Uses the same plugins directory as driver modules for consistency.+ Skips extraction if version file matches."+ ^String []+ (let [^Path plugins-path (plugins-dir-path)+ dest-path (.resolve plugins-path "python-sources")+ version-file (.resolve dest-path ".sqlglot-version")+ jar-version (some-> (io/resource "python-sources/.sqlglot-version") slurp str/trim)+ dest-version (when (u.files/exists? version-file)+ (str/trim (slurp (.toFile version-file))))]+ (if (and jar-version (= jar-version dest-version))+ (log/info "Python sources already extracted (version" jar-version ")")+ (do+ (u.files/create-dir-if-not-exists! dest-path)+ (log/info "Extracting Python sources to" (str dest-path))+ (u.files/with-open-path-to-resource [source-path python-sources-resource]+ (copy-dir-recursive! source-path dest-path))+ (log/info "Python sources extracted")))+ (str dest-path)))++;;; -------------------------------------------------- Dev lazy installation --------------------------------------------------++(def ^:private dev-python-sources-dir+ "Directory where sqlglot is installed in dev mode."+ "resources/python-sources")++(defn- delete-recursive!+ "Recursively delete a directory and all its contents."+ [^File f]+ (when (.isDirectory f)+ (doseq [child (.listFiles f)]+ (delete-recursive! child)))+ (.delete f))++(defn- expected-sqlglot-version+ "Read the expected sqlglot version from .sqlglot-version resource file."+ []+ (some->> (io/resource "python-sources/.sqlglot-version")+ slurp+ ;; handle # comments in the version file+ str/split-lines+ (remove #(str/starts-with? % "#"))+ (str/join "\n")+ str/trim))++(defn- package-installer-available?+ "Check if uv or pip is available for installing Python packages."+ []+ (or (zero? (:exit (shell/sh "which" "uv")))+ (zero? (:exit (shell/sh "which" "pip")))))++(defn- version-installed?+ "Check if sqlglot is installed with the expected version by looking for the dist-info directory."+ [target-dir expected-version]+ (let [dist-info (io/file target-dir (str "sqlglot-" expected-version ".dist-info"))]+ (.exists dist-info)))++(defn- install-sqlglot!+ "Install sqlglot via uv (preferred) or pip (fallback).+ Deletes any existing sqlglot directories first to handle version upgrades."+ [target-dir version]+ ;; Delete old versions first (sqlglot/ and any sqlglot-*.dist-info/)+ (doseq [^File f (.listFiles (io/file target-dir))]+ (when (or (= "sqlglot" (.getName f))+ (.startsWith (.getName f) "sqlglot-"))+ (log/info "Removing old sqlglot:" (.getName f))+ (delete-recursive! f)))+ ;; Try uv first (fast), fall back to pip+ (let [pkg (str "sqlglot==" version)+ uv-result (shell/sh "uv" "pip" "install" pkg "--target" target-dir "--no-compile")]+ (if (zero? (:exit uv-result))+ (log/info "sqlglot" version "installed via uv")+ (do+ (log/info "uv not available, trying pip...")+ (let [pip-result (shell/sh "pip" "install" pkg "--target" target-dir "--no-compile")]+ (when-not (zero? (:exit pip-result))+ (throw (ex-info (str "Failed to install sqlglot. Please install uv (recommended) or pip.\n"+ "Manual install: uv pip install " pkg " --target " target-dir "\n"+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")+ {:uv-error (:err uv-result)+ :pip-error (:err pip-result)+ :version version+ :target-dir target-dir})))+ (log/info "sqlglot" version "installed via pip"))))))++(defn- ensure-sqlglot-installed!+ "Ensure sqlglot is installed with correct version. Called lazily on first use in dev.+ If version mismatch or missing, automatically installs the correct version."+ []+ (let [expected-ver (expected-sqlglot-version)]+ (when-not expected-ver+ (throw (ex-info "Missing .sqlglot-version file in resources/python-sources/"+ {:resource "python-sources/.sqlglot-version"})))+ (when-not (version-installed? dev-python-sources-dir expected-ver)+ (if (package-installer-available?)+ (do+ (log/info "Installing sqlglot" expected-ver "(first use in dev)...")+ (install-sqlglot! dev-python-sources-dir expected-ver))+ (log/warn "sqlglot not installed and no package installer (uv or pip) found."+ "SQL parsing features will fail until sqlglot is installed."+ "Install uv: https://docs.astral.sh/uv/getting-started/installation/")))))++;;; -------------------------------------------------- Python path delay --------------------------------------------------++(defonce ^:private+ ^{:doc "Path to Python sources directory. In dev, this is resources/python-sources.+ When running from a JAR, we extract to the plugins directory.+ In dev, lazily installs sqlglot if not present or version mismatched."}+ python-path+ (delay+ (if (jar-resource? python-sources-resource)+ (extract-python-sources!)+ (do+ (ensure-sqlglot-installed!)+ dev-python-sources-dir))))++;;; -------------------------------------------- Context Wrappers ----------------------------------------------------++(p/defrecord+ PooledContext [^Context context ^Pool pool tuple poisoned?]+ common/PythonEval+ (eval-python [_this code]+ (.eval context "python" ^String code))++ Closeable+ (close [_this]+ (if @poisoned?+ (do+ (log/warn "Disposing poisoned Python context (likely hung GraalVM)")+ (.dispose pool :python tuple))+ (.release pool :python tuple))))++(defn poison!+ "Mark a PooledContext as poisoned. When closed, it will be disposed from the pool+ rather than released back. Use this when GraalVM hangs to prevent returning a+ broken context to the pool."+ [ctx]+ (when (instance? PooledContext ctx)+ (reset! (:poisoned? ctx) true)))++(defn interrupt!+ "Interrupt Python execution in this context. Returns true if interrupted successfully.+ If interrupt times out (guest in uninterruptible code), forces context closure with close(true).+ This is necessary because future-cancel doesn't actually stop GraalVM execution."+ [ctx ^long timeout-ms]+ (when-let [^Context raw-ctx (when (instance? PooledContext ctx)+ (:context ctx))]+ (try+ (.interrupt raw-ctx (Duration/ofMillis timeout-ms))+ true+ (catch TimeoutException _+ ;; Interrupt timed out - guest app may be in uninterruptible native code+ ;; Force close cannot be denied by guest application+ (log/warn "GraalVM interrupt timed out, forcing context closure")+ (.close raw-ctx true)+ false))))++(defn- create-graalvm-context+ "Create a new GraalVM Python context configured for sqlglot.++ The context is configured with:+ - PythonPath pointing to python-sources (contains sql_tools.py and sqlglot)+ - Full host access (needed for JSON serialization)+ - IO access enabled (for Python imports)"+ ^Context []+ (let [ctx (.. (Context/newBuilder (into-array String ["python"]))+ (option "engine.WarnInterpreterOnly" "false")+ ;; python-sources contains both sql_tools.py shim and installed sqlglot+ (option "python.PythonPath" @python-path)+ (allowHostAccess HostAccess/ALL)+ (allowIO true)+ (build))]+ (.eval ctx "python" "import sql_tools")+ ctx))++(defn- make-python-context-pool+ "Create a pool of Python contexts. Accepts a generator function and optional config map.++ Config options:+ - :max-size - Maximum pool size (default: 3)+ - :min-size - Minimum pool size (default: 1)+ - :ttl-minutes - How long contexts live before expiry (default: 10)+ - :utilization - Target utilization (default: 1.0 = 100%)++ This allows for easy testing by injecting mock context generators and custom pool configs."+ ([context-generator]+ (make-python-context-pool context-generator {}))+ ([context-generator {:keys [max-size min-size ttl-minutes utilization]+ :or {max-size 3+ min-size 1+ ttl-minutes 10+ utilization 1.0}}]+ (let [base-controller (Pools/utilizationController utilization max-size max-size)]+ (Pool. (reify IPool$Generator+ (generate [_ _]+ ;; Generate a tuple of the context and the expiry timestamp.+ (analytics/inc! :metabase-sql-parsing/context-creations)+ [(context-generator)+ (+ (System/nanoTime) (.toNanos TimeUnit/MINUTES ttl-minutes))])+ (destroy [_ _ [^Context ctx _expiry]]+ ;; Close context when disposed from pool (expiry, poison, or shutdown)+ (try+ (.close ctx true) ;; Force close - can't wait for running code+ (catch Exception _))))+ ;; Wrap the utilization controller with a modification that doesn't allow the pool to go below min-size.+ (reify IPool$Controller+ (shouldIncrement [_ k a b] (.shouldIncrement base-controller k a b))+ (adjustment [_ stats]+ (let [adj (.adjustment base-controller stats)+ ;; :python is arbitrary key, it just has to be consistent everywhere when working with the pool.+ n (some-> ^io.aleph.dirigiste.Stats (:python stats) .getNumWorkers)+ python-adj (:python adj)]+ (if (and n python-adj (<= (+ n python-adj) min-size))+ ;; If the adjustment is going to bring the pool below min-size, return empty adjustment instead.+ {}+ adj))))+ 65000 ;; Queue size - doesn't matter much.+ 25 ;; Sampling interval - doesn't matter much.+ 10000 ;; Recheck every 10 seconds+ TimeUnit/MILLISECONDS))))++(def ^:private python-context-pool+ "Pool of Python context objects. They are not thread-safe, so access to them has to be carefully managed between+ threads. Each context with loaded Python interpreter takes significant memory, so we don't want too many of them.+ However, one takes significant time to initialize, so we don't want to load them anew each time in prod. Under some+ circumstances, the GraalVM Python context tends to leak memory, so we don't want to keep the reference to the+ context forever. Considering all that, this pool targets 100% utilization (so, if the utilization is lower, the pool+ will start dropping objects) and the maximum of 3 objects (to prevent OOMs), but at least 1 object will always be in+ the pool to pick up. However, together with each context keep its creation timestamp so that we can throwaway+ instances that are too old to avoid leaks.++ Wrapped in delay to avoid initialization during AOT compilation."+ (delay (make-python-context-pool create-graalvm-context {:max-size 3+ :min-size 1+ :ttl-minutes 10})))++(defn- acquire-context+ "Acquire a context from the pool, handling expiry. Returns a PooledContext."+ [^Pool pool]+ (loop []+ (let [[context expiry-ts :as tuple] (.acquire pool :python)]+ (if (>= (System/nanoTime) expiry-ts)+ (do (analytics/inc! :metabase-sql-parsing/context-disposals-expired)+ (.dispose pool :python tuple)+ (recur))+ (do (analytics/inc! :metabase-sql-parsing/context-acquisitions)+ (->PooledContext context pool tuple (atom false)))))))++(defn python-context+ "Acquire a python context from the pool. Must be closed. Use in a `with-open` context."+ []+ (acquire-context @python-context-pool))diff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.cljnew file mode 100644index 00000000000..541231e50de--- /dev/null+++ b/src/metabase/sql_tools/common.clj@@ -0,0 +1,189 @@+(ns metabase.sql-tools.common+ {:clj-kondo/config '{:linters {:metabase/modules {:level :off}}}}+ (:require+ [clojure.set :as set]+ [clojure.string :as str]+ [metabase.driver.sql :as driver.sql]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.sql-tools.interface :as sql-tools]+ [metabase.util.humanization :as u.humanization]+ [metabase.util.malli :as mu]))++(defn normalize-table-spec+ "Return tables spec with normalized values."+ [driver {:keys [table schema]}]+ {:table (driver.sql/normalize-name driver table)+ :schema (some->> schema (driver.sql/normalize-name driver))})++(defn find-table-or-transform+ "Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.+ It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."+ [driver tables transforms {search-table :table raw-schema :schema}]+ (let [search-schema (or raw-schema+ (driver.sql/default-schema driver))+ normalize (partial driver.sql/normalize-name driver)+ matches? (fn [db-table db-schema]+ (and (= (normalize search-table) (normalize db-table))+ (= (some-> search-schema normalize) (some-> db-schema normalize))))]+ (or (some (fn [{:keys [name schema id]}]+ (when (matches? name schema)+ {:table id}))+ tables)+ (some (fn [{:keys [id] {:keys [name schema]} :target}]+ (when (matches? name schema)+ {:transform id}))+ transforms))))++(mu/defn table-name :- [:maybe :string]+ "Computes a table name from a table reference"+ [raw-col :- [:map+ [:database {:optional true} :string]+ [:schema {:optional true} :string]+ [:table {:optional true} :string]]]+ (when (:table raw-col)+ (->> [:database :schema :table]+ (keep raw-col)+ (str/join "."))))++(defn wrap-col+ "Wraps the argument in `{:col _}`"+ [col]+ {:col col})++(defmulti resolve-field+ "Resolves a field reference to one or more actual database fields.++ This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field+ references returned by sql-tools.macaw.references/field-references.++ Note: this currently sets :lib/desired-column-alias but no other :lib/* fields, because the callers of this function+ don't need the other fields. If we care about other :lib/* fields in the future, we can add them then."+ {:added "0.57.0" :arglists '([driver metadata-provider col-spec])}+ (fn [_driver _metadata-provider col-spec]+ (:type col-spec)))++(defmethod resolve-field :all-columns+ [driver metadata-provider col-spec]+ (or (some->> (:table col-spec)+ (find-table-or-transform+ driver (lib.metadata/tables metadata-provider) (lib.metadata/transforms metadata-provider))+ :table+ (lib.metadata/active-fields metadata-provider)+ (map #(-> (assoc % :lib/desired-column-alias (:name %))+ wrap-col)))+ [{:error (lib/missing-table-alias-error+ (table-name (:table col-spec)))}]))++(defmethod resolve-field :single-column+ [driver metadata-provider {:keys [alias] :as col-spec}]+ [(if-let [{:keys [name] :as found}+ (->> (:source-columns col-spec)+ (some (fn [source-col-set]+ ;; in cases like `select (select blah from ...) from ...`, if blah refers to a+ ;; column in both the inner query and the outer query, the column from the inner+ ;; query will be preferred. However, if blah doesn't refer to something in the+ ;; inner query, it can also refer to something in the outer query.+ ;; sql-tools.macaw.references/field-references organizes source-cols into a list of lists+ ;; to account for this.+ (->> (mapcat (fn [current-col]+ ;; :unknown-columns is a placeholder for "we know there are columns being+ ;; returned, but have no way of knowing what those are -- this is primarily+ ;; used for table-functions like `select * from my_func()`. If we encounter+ ;; something like that, assume that the query is valid and make up a matching+ ;; column to avoid false positives.+ (if (= (:type current-col) :unknown-columns)+ (let [name (:column col-spec)]+ [{:base-type :type/*+ :name name+ :display-name (->> name (u.humanization/name->human-readable-name :simple))+ :effective-type :type/*+ :semantic-type :Semantic/*}])+ (keep :col (resolve-field driver metadata-provider current-col))))+ source-col-set)+ (some #(when (= (driver.sql/normalize-name driver (:name %))+ (driver.sql/normalize-name driver (:column col-spec)))+ %))))))]+ {:col (assoc found :lib/desired-column-alias (or alias name))}+ {:error (lib/missing-column-error (:column col-spec))})])++(defn- get-name [m]+ (or (:alias m) (str (gensym "new-col"))))++(defn- get-display-name [m]+ (->> (get-name m)+ (u.humanization/name->human-readable-name :simple)))++(defmethod resolve-field :custom-field+ [_driver _metadata-provider col-spec]+ [{:col {:base-type :type/*+ :name (get-name col-spec)+ :lib/desired-column-alias (get-name col-spec)+ :display-name (get-display-name col-spec)+ :effective-type :type/*+ :semantic-type :Semantic/*}}])++(defn- lca [default-type & types]+ (let [ancestor-sets (for [t types+ :when t]+ (conj (set (ancestors t)) t))+ common-ancestors (when (seq ancestor-sets)+ (apply set/intersection ancestor-sets))]+ (if (seq common-ancestors)+ (apply (partial max-key (comp count ancestors)) common-ancestors)+ default-type)))++(defmethod resolve-field :composite-field+ [driver metadata-provider col-spec]+ (let [member-fields (mapcat #(->> (resolve-field driver metadata-provider %)+ (keep :col))+ (:member-fields col-spec))]+ [{:col {:name (get-name col-spec)+ :lib/desired-column-alias (get-name col-spec)+ :display-name (get-display-name col-spec)+ :base-type (apply lca :type/* (map :base-type member-fields))+ :effective-type (apply lca :type/* (map :effective-type member-fields))+ :semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}}]))++(defmethod resolve-field :unknown-columns+ [_driver _metadata-provider _col-spec]+ [])++(defn returned-columns+ "Given a native query, return columns it produces using field-references pipeline."+ [parser driver native-query]+ (let [{:keys [returned-fields]} (sql-tools/field-references-impl parser driver (lib/raw-native-query native-query))]+ (vec (mapcat #(->> (resolve-field driver native-query %)+ (keep :col))+ returned-fields))))++(defn- normalize-error+ "Normalize error names using driver-specific case normalization.+ This ensures error names match database metadata conventions."+ [driver error]+ (if-let [error-name (:name error)]+ (assoc error :name (driver.sql/normalize-name driver error-name))+ error))++(defn validate-query+ "Validate native query."+ [parser driver native-query]+ (let [{:keys [used-fields returned-fields errors]} (sql-tools/field-references-impl parser driver (lib/raw-native-query native-query))+ check-fields #(mapcat (fn [col-spec]+ (->> (resolve-field driver (lib/->metadata-provider native-query) col-spec)+ (keep :error)))+ %)]+ (->> (-> errors+ (into (check-fields used-fields))+ (into (check-fields returned-fields)))+ (map (partial normalize-error driver))+ set)))++(defn referenced-fields+ "Extract fields referenced (used) in a native query - fields in WHERE, JOIN ON, etc."+ [parser driver native-query]+ (let [{:keys [used-fields]} (sql-tools/field-references-impl parser driver (lib/raw-native-query native-query))]+ (into #{}+ (mapcat #(->> (resolve-field driver native-query %)+ (keep :col)))+ used-fields)))diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.cljnew file mode 100644index 00000000000..d37fdf60832--- /dev/null+++ b/src/metabase/sql_tools/core.clj@@ -0,0 +1,184 @@+(ns metabase.sql-tools.core+ "Public API for sql-tools SQL parsing functionality.++ This namespace provides parser-agnostic functions that delegate to the+ configured backend (Macaw or SQLGlot) via multimethods.++ Note: Implementation namespaces (macaw.core, sqlglot.core) are loaded via+ metabase.sql-tools.init to avoid circular dependencies."+ (:require+ [metabase.sql-tools.interface :as interface]+ [metabase.sql-tools.metrics :as metrics]+ [metabase.sql-tools.settings :as sql-tools.settings]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [potemkin :as p]))++;;; -------------------------------------------------- Schemas --------------------------------------------------++(mr/def ::table-spec+ "Schema for table identifier used in replacements."+ [:map+ [:table :string]+ [:schema {:optional true} [:maybe :string]]])++(mr/def ::column-spec+ "Schema for column identifier used in replacements."+ [:map+ [:column :string]+ [:table {:optional true} [:maybe :string]]+ [:schema {:optional true} [:maybe :string]]])++(mr/def ::table-replacement-value+ "Schema for table replacement target - can be a string or a map with schema/table."+ [:or+ :string+ [:map+ [:schema {:optional true} [:maybe :string]]+ [:table {:optional true} [:maybe :string]]]])++(mr/def ::replacements+ "Schema for replace-names replacements map."+ [:map+ [:schemas {:optional true} [:map-of :string :string]]+ [:tables {:optional true} [:map-of ::table-spec ::table-replacement-value]]+ [:columns {:optional true} [:map-of ::column-spec :string]]])++(mr/def ::replace-names-opts+ "Schema for replace-names options."+ [:map+ [:allow-unused? {:optional true} :boolean]])++(mr/def ::simple-query-result+ "Schema for simple-query? return value."+ [:map+ [:is_simple :boolean]+ [:reason {:optional true} :string]])++;; Re-export multimethod vars so implementations can use sql-tools/foo-impl+(p/import-vars+ [interface+ returned-columns-impl+ referenced-tables-impl+ referenced-fields-impl+ field-references-impl+ validate-query-impl+ replace-names-impl+ referenced-tables-raw-impl+ simple-query?-impl+ add-into-clause-impl])++(defn parser-backend+ "Get current value of sql-tools-parser-backend."+ []+ (sql-tools.settings/current-parser-backend))++(mu/defn returned-columns :- [:sequential :map]+ "Return appdb columns for the `native-query`."+ [driver :- :keyword+ native-query]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "returned-columns"]+ (interface/returned-columns-impl parser driver native-query))))++(mu/defn referenced-tables :- [:set :map]+ "Return tables referenced by the `native-query`"+ [driver :- :keyword+ native-query]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "referenced-tables"]+ (interface/referenced-tables-impl parser driver native-query))))++(mu/defn referenced-fields :- [:set :map]+ "Return appdb fields referenced (used) by the `native-query`.++ This includes fields in SELECT, WHERE, JOIN ON, GROUP BY, ORDER BY, etc.+ Returns a set of :metadata/column maps."+ [driver :- :keyword+ native-query]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "referenced-fields"]+ (interface/referenced-fields-impl parser driver native-query))))++(mu/defn field-references :- :metabase.sql-tools.macaw.references/field-references+ "Return field references for SQL string.++ Returns a map with:+ - :used-fields - set of field specs+ - :returned-fields - vector of field specs+ - :errors - set of validation errors"+ [driver :- :keyword+ sql-string :- :string]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "field-references"]+ (interface/field-references-impl parser driver sql-string))))++(mu/defn validate-query :- [:set :map]+ "Validate native query. Returns a set of validation errors (empty set if valid)."+ [driver :- :keyword+ native-query]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "validate-query"]+ (interface/validate-query-impl parser driver native-query))))++(mu/defn replace-names :- :string+ "Replace schema, table, and column names in a SQL query.++ `replacements` is a map with optional keys:+ - :schemas - map of old schema name -> new schema name+ - :tables - map of old table name -> new table name+ - :columns - map of {:table t :column c} -> new column name++ `opts` is an optional map with:+ - :allow-unused? - if true, don't error when a replacement isn't used++ Returns the modified SQL string.++ SECURITY: Replacement values are injected into the SQL AST as identifier names+ without sanitization. Callers MUST ensure replacement values are system-generated+ (e.g., workspace isolation schema names, database metadata). Never pass+ user-controlled input as replacement values."+ ([driver sql-string replacements]+ (replace-names driver sql-string replacements {}))+ ([driver :- [:maybe :keyword]+ sql-string :- :string+ replacements :- ::replacements+ opts :- ::replace-names-opts]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "replace-names"]+ (interface/replace-names-impl parser driver sql-string replacements opts)))))++(mu/defn referenced-tables-raw :- [:sequential ::table-spec]+ "Given a driver and sql string, returns a sequence of {:schema <name> :table <name>} maps."+ [driver :- :keyword+ sql-str :- :string]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "referenced-tables-raw"]+ (interface/referenced-tables-raw-impl parser driver sql-str))))++(mu/defn simple-query? :- ::simple-query-result+ "Check if SQL string is a simple SELECT (no LIMIT, OFFSET, or CTEs).+ Used by Workspaces to determine if automatic checkpoints can be inserted.++ Returns a map with:+ - `:is_simple` - boolean indicating if query is simple+ - `:reason` - string explaining why query is not simple (when false)"+ [sql-string :- :string]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "simple-query?"]+ (interface/simple-query?-impl parser sql-string))))++(mu/defn add-into-clause :- :string+ "Add an INTO clause to a SELECT statement.++ Transforms: 'SELECT * FROM products'+ Into: 'SELECT * INTO \"TABLE\" FROM products'++ Used by SQL Server compile-transform which requires SELECT INTO syntax+ instead of CREATE TABLE AS SELECT."+ [driver :- :keyword+ sql :- :string+ table-name :- :string]+ (let [parser (sql-tools.settings/current-parser-backend)]+ (metrics/with-operation-timing [parser "add-into-clause"]+ (interface/add-into-clause-impl parser driver sql table-name))))diff --git a/src/metabase/sql_tools/init.clj b/src/metabase/sql_tools/init.cljnew file mode 100644index 00000000000..eb9fa473d5d--- /dev/null+++ b/src/metabase/sql_tools/init.clj@@ -0,0 +1,7 @@+(ns metabase.sql-tools.init+ (:require+ ;; metabase.sql-tools.<parser> is required to ensure method's registration+ [metabase.sql-tools.macaw.core]+ [metabase.sql-tools.metrics]+ [metabase.sql-tools.settings]+ [metabase.sql-tools.sqlglot.core]))diff --git a/src/metabase/sql_tools/interface.clj b/src/metabase/sql_tools/interface.cljnew file mode 100644index 00000000000..78ee23f266c--- /dev/null+++ b/src/metabase/sql_tools/interface.clj@@ -0,0 +1,67 @@+(ns metabase.sql-tools.interface+ "Multimethod definitions for sql-tools parser implementations.++ This namespace contains only the multimethod definitions. Implementations+ are in metabase.sql-tools.macaw.core and metabase.sql-tools.sqlglot.core.")++(defn parser-driver-dispatch+ "Dispatch function for sql-tools multimethods. Returns the parser keyword (e.g. :macaw, :sqlglot)."+ [parser _driver & _args]+ parser)++(defmulti returned-columns-impl+ "Parser specific implementation of [[metabase.sql-tools.core/returned-columns]]. Do not use directly."+ {:arglists '([parser driver native-query])}+ parser-driver-dispatch)++(defmulti referenced-tables-impl+ "Parser specific implementation of [[metabase.sql-tools.core/referenced-tables]]. Do not use directly."+ {:arglists '([parser driver native-query])}+ parser-driver-dispatch)++(defmulti referenced-fields-impl+ "Parser specific implementation of [[metabase.sql-tools.core/referenced-fields]]. Do not use directly."+ {:arglists '([parser driver native-query])}+ parser-driver-dispatch)++(defmulti field-references-impl+ "Parser specific implementation of [[metabase.sql-tools.core/field-references]]. Do not use directly.++ Returns a map with:+ - :used-fields - set of field specs from WHERE, JOIN ON, GROUP BY, ORDER BY+ - :returned-fields - vector of field specs from SELECT clause (ordered)+ - :errors - set of validation errors"+ {:arglists '([parser driver sql-string])}+ parser-driver-dispatch)++(defmulti validate-query-impl+ "Parser specific implementation of [[metabase.sql-tools.core/validate-query]]. Do not use directly."+ {:arglists '([parser driver native-query])}+ parser-driver-dispatch)++(defmulti replace-names-impl+ "Parser specific implementation of [[metabase.sql-tools.core/replace-names]]. Do not use directly."+ {:arglists '([parser driver sql-string replacements opts])}+ parser-driver-dispatch)++(defmulti referenced-tables-raw-impl+ "Parser specific implementation of [[metabase.sql-tools.core/referenced-tables-raw]]. Do not use directly."+ {:arglists '([parser driver sql-str])}+ parser-driver-dispatch)++(defmulti simple-query?-impl+ "Parser specific implementation of [[metabase.sql-tools.core/simple-query?]]. Do not use directly.++ Returns a map with:+ - `:is_simple` - boolean indicating if query is simple+ - `:reason` - string explaining why query is not simple (optional)"+ {:arglists '([parser sql-string])}+ parser-driver-dispatch)++(defmulti add-into-clause-impl+ "Parser specific implementation of [[metabase.sql-tools.core/add-into-clause]]. Do not use directly.++ Transforms a SELECT statement to include an INTO clause for SQL Server style+ SELECT INTO syntax."+ {:arglists '([parser driver sql table-name])}+ parser-driver-dispatch)diff --git a/src/metabase/sql_tools/macaw/core.clj b/src/metabase/sql_tools/macaw/core.cljnew file mode 100644index 00000000000..a4a010d8e77--- /dev/null+++ b/src/metabase/sql_tools/macaw/core.clj@@ -0,0 +1,191 @@+(ns metabase.sql-tools.macaw.core+ (:require+ [clojure.string :as str]+ [macaw.core :as macaw]+ [metabase.driver :as driver]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.sql-tools.common :as sql-tools.common]+ [metabase.sql-tools.interface :as sql-tools]+ [metabase.sql-tools.macaw.references :as sql-tools.macaw.references]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]))++(set! *warn-on-reflection* true)++;;;; Macaw options and parsing++(def ^:private considered-drivers+ "The set of drivers for which we configure non-trivial macaw options."+ #{:h2 :mysql :postgres :redshift :sqlite :sqlserver})++(defn- macaw-options+ "Generate the options expected by Macaw based on the nature of the given driver."+ [driver]+ (merge+ ;; If this isn't a driver we've considered, fallback to Macaw's conservative defaults.+ (when (contains? considered-drivers driver)+ {;; According to the SQL-92 specification, non-quoted identifiers should be case-insensitive, and the majority of+ ;; engines are implemented this way.+ ;;+ ;; In practice there are exceptions, notably MySQL and SQL Server, where case sensitivity is a property of the+ ;; underlying resource referenced by the identifier, and the case-sensitivity does not depend on whether the+ ;; reference is quoted.+ ;;+ ;; For MySQL the case sensitivity of databases and tables depends on both the underlying file system, and a system+ ;; variable used to initialize the database. For SQL Server it depends on the collation settings of the collection+ ;; where the corresponding schema element is defined.+ ;;+ ;; For MySQL, columns and aliases can never be case-sensitive, and for SQL Server the default collation is case-+ ;; insensitive too, so it makes sense to just treat all databases as case-insensitive as a whole.+ ;;+ ;; In future, Macaw may support discriminating on the identifier type, in which case we could be more precise for+ ;; these databases. Being 100% correct would require querying system variables and schema configuration however,+ ;; which is likely a step too far in complexity.+ ;;+ ;; Currently, we go with :agnostic, as it is the most relaxed semantics (the case of both the identifiers and the+ ;; underlying schema is totally ignored, and correspondence is non-deterministic), but Macaw supports more nuanced+ ;; :lower and :upper configuration values which coerce the query identifiers to a given case then do an exact+ ;; comparison with the schema.+ :case-insensitive :agnostic+ ;; For both MySQL and SQL Server, whether identifiers are case-sensitive depends on database configuration only,+ ;; and quoting has no effect on this, so we disable this option for consistency with `:case-insensitive`.+ :quotes-preserve-case? (not (contains? #{:mysql :sqlserver} driver))+ :features {:postgres-syntax (isa? driver/hierarchy driver :postgres)+ :square-bracket-quotes (= :sqlserver driver)+ :unsupported-statements false+ :backslash-escape-char true+ ;; This will slow things down, but until we measure the difference, opt for correctness.+ :complex-parsing true}+ ;; 10 seconds+ :timeout 10000})+ {;; There is no plan to be exhaustive yet.+ ;; Note that while an allowed list would be more conservative, at the time of writing only 1 of the bundled+ ;; drivers use FINAL as a reserved word, and mentioning them all would be prohibitive.+ ;; In the future, we will use multimethods to define this explicitly per driver, or even discover it automatically+ ;; through the JDBC connection, where possible.+ :non-reserved-words (vec (remove nil? [(when-not (contains? #{:clickhouse} driver)+ :final)]))}))++(defn- parsed-query+ "Parse SQL using Macaw with driver-specific options."+ [sql driver & {:as opts}]+ (macaw/parsed-query sql (merge (macaw-options driver) opts)))++;;;; referenced-tables++(defn split-compound-table-spec+ "Macaw doesn't understand multi-part identifiers like BigQuery's `dataset.table` or+ `project.dataset.table`, returning them as a single compound `:table` value.+ This splits such compound names into separate `:schema` and `:table` parts.++ - `table` → `{:table \"table\"}`+ - `schema.table` → `{:schema \"schema\", :table \"table\"}`+ - `catalog.schema.table` → `{:schema \"schema\", :table \"table\"}`"+ [{:keys [table schema] :as table-spec}]+ (if (and table (nil? schema) (str/includes? table "."))+ (let [parts (str/split table #"\.")]+ (case (count parts)+ 1 table-spec+ 2 {:schema (first parts) :table (second parts)}+ ;; 3+ parts: take last two as schema.table (drop catalog/project prefix)+ {:schema (nth parts (- (count parts) 2))+ :table (last parts)}))+ table-spec))++(mu/defn- referenced-tables+ [driver :- :keyword+ query :- :metabase.lib.schema/native-only-query]+ (let [db-tables (lib.metadata/tables query)+ db-transforms (lib.metadata/transforms query)]+ (-> query+ lib/raw-native-query+ (parsed-query driver)+ (macaw/query->components {:strip-contexts? true})+ :tables+ (->> (map :component))+ (->> (map split-compound-table-spec))+ (->> (into #{}+ (keep #(->> (sql-tools.common/normalize-table-spec driver %)+ (sql-tools.common/find-table-or-transform driver db-tables db-transforms))))))))++(defmethod sql-tools/referenced-tables-impl :macaw+ [_parser driver query]+ (referenced-tables driver query))++;;;; field-references++(defmethod sql-tools/field-references-impl :macaw+ [_parser driver sql-string]+ (-> sql-string+ (parsed-query driver)+ macaw/->ast+ (->> (sql-tools.macaw.references/field-references driver))))++;;;; returned-columns++(defmethod sql-tools/returned-columns-impl :macaw+ [parser driver query]+ (sql-tools.common/returned-columns parser driver query))++;;;; referenced-fields++(defmethod sql-tools/referenced-fields-impl :macaw+ [parser driver query]+ (sql-tools.common/referenced-fields parser driver query))++(defmethod sql-tools/validate-query-impl :macaw+ [parser driver query]+ (sql-tools.common/validate-query parser driver query))++(defmethod sql-tools/referenced-tables-raw-impl :macaw+ [_parser _driver sql-str]+ (-> sql-str+ (macaw/parsed-query)+ (macaw/query->components {:strip-contexts? true})+ :tables+ (->> (map :component)+ (mapv (fn [{:keys [schema table]}]+ (cond-> {:table table}+ schema (assoc :schema schema)))))))++(defmethod sql-tools/simple-query?-impl :macaw+ [_parser sql-string]+ (try+ ;; BEWARE: No driver available, so we pass nil. This means macaw-options will be minimal.+ (let [^net.sf.jsqlparser.statement.select.PlainSelect parsed (parsed-query sql-string nil)]+ (cond+ (not (instance? net.sf.jsqlparser.statement.select.PlainSelect parsed))+ {:is_simple false+ :reason "Not a simple SELECT"}++ (.getLimit parsed)+ {:is_simple false+ :reason "Contains a LIMIT"}++ (.getOffset parsed)+ {:is_simple false+ :reason "Contains an OFFSET"}++ (seq (.getWithItemsList parsed))+ {:is_simple false+ :reason "Contains a CTE"}++ :else+ {:is_simple true}))+ (catch Exception e+ (log/debugf e "Failed to parse query: %s" (ex-message e))+ {:is_simple false})))++(defmethod sql-tools/add-into-clause-impl :macaw+ [_parser driver sql table-name]+ (let [^net.sf.jsqlparser.statement.select.Select parsed-query (parsed-query sql driver)+ ^net.sf.jsqlparser.statement.select.PlainSelect select-body (.getSelectBody parsed-query)]+ (.setIntoTables select-body [(net.sf.jsqlparser.schema.Table. ^String table-name)])+ (str parsed-query)))++(defmethod sql-tools/replace-names-impl :macaw+ [_parser driver sql-string replacements opts]+ ;; Note: :case-insensitive :agnostic causes ClassCastException in Macaw's replace-names+ ;; due to regex pattern handling. Omit it for now until Macaw is fixed.+ (macaw/replace-names sql-string replacements (merge (dissoc (macaw-options driver) :case-insensitive) opts)))diff --git a/src/metabase/driver/sql/references.clj b/src/metabase/sql_tools/macaw/references.cljsimilarity index 88%rename from src/metabase/driver/sql/references.cljrename to src/metabase/sql_tools/macaw/references.cljindex 9d4d8c69c2f..46a1d73ddb8 100644--- a/src/metabase/driver/sql/references.clj+++ b/src/metabase/sql_tools/macaw/references.clj@@ -1,11 +1,12 @@-(ns metabase.driver.sql.references+(ns metabase.sql-tools.macaw.references(:refer-clojure :exclude [every? mapv select-keys some])(:require- [clojure.string :as str][macaw.ast-types :as macaw.ast-types][metabase.driver :as driver]- [metabase.driver-api.core :as driver-api]- [metabase.driver.sql.normalize :as sql.normalize]+ [metabase.driver.sql :as driver.sql]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.validate :as lib.schema.validate]+ [metabase.sql-tools.common :as sql-tools.common][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.performance :refer [every? mapv select-keys some]]))@@ -62,12 +63,12 @@[:map[:used-fields [:set [:ref ::col-spec]]][:returned-fields [:sequential [:ref ::col-spec]]]- [:errors [:set [:ref driver-api/schema.validate.error]]]])+ [:errors [:set [:ref ::lib.schema.validate/error]]]])(defn- normalize-fields [driver m](update-vals m#(if (string? %)- (sql.normalize/normalize-name driver %)+ (driver.sql/normalize-name driver %)%)))(defn- col-fields [driver m]@@ -89,22 +90,11 @@sublist))sources))-(mu/defn table-name :- [:maybe :string]- "Computes a table name from a table reference"- [raw-col :- [:map- [:database {:optional true} :string]- [:schema {:optional true} :string]- [:table {:optional true} :string]]]- (when (:table raw-col)- (->> [:database :schema :table]- (keep raw-col)- (str/join "."))))-(defn- get-column [driver sources raw-col {:keys [return-table-matches?]}](if (and (nil? (:table raw-col))(nil? (:schema raw-col))(nil? (:database raw-col))- (sql.normalize/reserved-literal driver (:column raw-col)))+ (driver.sql/reserved-literal driver (:column raw-col)))[](let [{:keys [alias column table] :as expr} (col-fields driver raw-col)valid-sources (if table@@ -141,9 +131,9 @@:source-columns source-columns};; if we didn't find any potential sources, signal an error here:errors (when-not (some #(some identity %) valid-sources)- #{(if-let [name (table-name raw-col)]- (driver-api/missing-table-alias-error name)- (driver-api/missing-column-error column))})}]))))+ #{(if-let [name (sql-tools.common/table-name raw-col)]+ (lib/missing-table-alias-error name)+ (lib/missing-column-error column))})}]))))(defmulti find-used-fields"Finds the fields used in a given sql expression."@@ -220,7 +210,7 @@(defmethod find-returned-fields :default[driver sources withs {:keys [alias] :as expr}][{:col {:alias (when alias- (sql.normalize/normalize-name driver alias))+ (driver.sql/normalize-name driver alias)):type :custom-field:used-fields (into #{} (keep :col) (find-used-fields driver sources withs expr))}}])@@ -228,23 +218,18 @@[driver sources _withs expr](get-column driver sources expr {:return-table-matches? true}))-(defn wrap-col- "Wraps the argument in `{:col _}`"- [col]- {:col col})-(defmethod find-returned-fields [:sql :macaw.ast/wildcard][_driver sources _withs _expr](into []- (mapcat #(->> % :returned-fields (map wrap-col)))+ (mapcat #(->> % :returned-fields (map sql-tools.common/wrap-col)))(first sources)))(defmethod find-returned-fields [:sql :macaw.ast/table-wildcard][driver sources _withs expr](or (some->> (find-source (col-fields driver expr) sources):returned-fields- (map wrap-col))- [{:errors #{(driver-api/missing-table-alias-error (table-name expr))}}]))+ (map sql-tools.common/wrap-col))+ [{:errors #{(lib/missing-table-alias-error (sql-tools.common/table-name expr))}}]))(defmethod find-returned-fields [:sql :macaw.ast/set-operation][driver sources withs expr]@@ -295,14 +280,14 @@rec (partial find-used-fields driver with-select withs)](into #{}cat- [ ;; a select can't refer to its own aliases, so don't include them in sources here+ [;; a select can't refer to its own aliases, so don't include them in sources here(mapcat (partial find-used-fields driver with-outside withs)(:select expr))(rec (:where expr))(mapcat rec (:join expr))(mapcat rec (:group-by expr))(mapcat rec (:order-by expr))- (mapcat #(->> % :used-fields (map wrap-col))+ (mapcat #(->> % :used-fields (map sql-tools.common/wrap-col))local-sources)])))(defmethod find-returned-fields [:sql :macaw.ast/select]@@ -314,7 +299,7 @@;; :alias is a column alias, so this is presumably something like `select (select * from ...)`;; there should only be one field returned, and that field should have the appropriate alias(do (assert (= 1 (count returned-fields)))- (map #(assoc-in % [:col :alias] (sql.normalize/normalize-name driver (:alias expr)))+ (map #(assoc-in % [:col :alias] (driver.sql/normalize-name driver (:alias expr)))returned-fields))returned-fields)))@@ -336,7 +321,7 @@(comp cat (mapcat :errors))[returned-fields current-used-fields local-withs]):names (when-let [alias (:table-alias expr)]- {:table-alias (sql.normalize/normalize-name driver alias)})}))+ {:table-alias (driver.sql/normalize-name driver alias)})}))(defmethod field-references-impl [:sql :macaw.ast/join][driver outside-sources withs expr]@@ -352,14 +337,14 @@(comp cat (mapcat :errors))[used-fields returned-fields]):names (when-let [alias (:table-alias expr)]- {:table-alias (sql.normalize/normalize-name driver alias)})}))+ {:table-alias (driver.sql/normalize-name driver alias)})}))(defmethod field-references-impl [:sql :macaw.ast/table-function][driver _outside-sources _withs expr]{:used-fields #{}:returned-fields [{:type :unknown-columns}]:names (when-let [alias (:table-alias expr)]- {:table-alias (sql.normalize/normalize-name driver alias)})+ {:table-alias (driver.sql/normalize-name driver alias)}):errors #{}})(defmethod field-references-impl :default@@ -367,7 +352,7 @@{:used-fields #{}:returned-fields []:names nil- :errors #{(driver-api/syntax-error)}})+ :errors #{(lib/syntax-error)}})(mu/defn field-references :- [:ref ::field-references]"Takes a sql query in the macaw ast format and returns the fields referenced by a query.diff --git a/src/metabase/sql_tools/metrics.clj b/src/metabase/sql_tools/metrics.cljnew file mode 100644index 00000000000..659372b89f3--- /dev/null+++ b/src/metabase/sql_tools/metrics.clj@@ -0,0 +1,79 @@+(ns metabase.sql-tools.metrics+ "Prometheus metrics instrumentation for sql-tools operations.++ Provides recording functions and a `with-operation-timing` macro that wraps+ sql-tools public API calls with counters and histograms, labeled by parser+ backend and operation name."+ (:require+ [metabase.analytics.core :as analytics]+ [metabase.util :as u]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Known labels --------------------------------------------------++(def ^:private parsers ["macaw" "sqlglot"])++(def ^:private operations+ ["returned-columns"+ "referenced-tables"+ "referenced-fields"+ "field-references"+ "validate-query"+ "replace-names"+ "referenced-tables-raw"+ "simple-query?"+ "add-into-clause"])++(def ^:private operation-labels+ (vec (for [parser parsers+ operation operations]+ {:parser parser :operation operation})))++(defmethod analytics/known-labels :metabase-sql-tools/operations-total [_] operation-labels)+(defmethod analytics/known-labels :metabase-sql-tools/operations-completed [_] operation-labels)+(defmethod analytics/known-labels :metabase-sql-tools/operations-failed [_] operation-labels)++;;; -------------------------------------------------- Recording fns --------------------------------------------------++(defn record-operation-start!+ "Record the start of an sql-tools operation."+ [parser operation]+ (analytics/inc! :metabase-sql-tools/operations-total+ {:parser (name parser) :operation operation}))++(defn record-operation-completion!+ "Record the successful completion of an sql-tools operation."+ [parser operation duration-ms]+ (let [labels {:parser (name parser) :operation operation}]+ (analytics/inc! :metabase-sql-tools/operations-completed labels)+ (analytics/observe! :metabase-sql-tools/operation-duration-ms labels duration-ms)))++(defn record-operation-failure!+ "Record the failure of an sql-tools operation."+ [parser operation duration-ms]+ (let [labels {:parser (name parser) :operation operation}]+ (analytics/inc! :metabase-sql-tools/operations-failed labels)+ (analytics/observe! :metabase-sql-tools/operation-duration-ms labels duration-ms)))++;;; -------------------------------------------------- Timing macro --------------------------------------------------++(defmacro with-operation-timing+ "Execute body while timing an sql-tools operation. Automatically records+ start, completion or failure metrics labeled by parser and operation.++ Usage:+ (with-operation-timing [parser \"returned-columns\"]+ (interface/returned-columns-impl parser driver native-query))"+ [[parser operation] & body]+ `(let [parser# ~parser+ operation# ~operation+ start# (u/start-timer)]+ (record-operation-start! parser# operation#)+ (try+ (let [result# (do ~@body)]+ (record-operation-completion! parser# operation# (long (u/since-ms start#)))+ result#)+ (catch Throwable t#+ (record-operation-failure! parser# operation# (long (u/since-ms start#)))+ (throw t#)))))diff --git a/src/metabase/sql_tools/settings.clj b/src/metabase/sql_tools/settings.cljnew file mode 100644index 00000000000..b9bad486bc1--- /dev/null+++ b/src/metabase/sql_tools/settings.clj@@ -0,0 +1,54 @@+(ns metabase.sql-tools.settings+ "Settings for the sql-tools module.++ ## Swapping Between Macaw and SQLGlot++ The SQL parser backend can be switched using the `MB_SQL_TOOLS_PARSER_BACKEND` env var:++ ```bash+ # Use SQLGlot (default)+ MB_SQL_TOOLS_PARSER_BACKEND=sqlglot++ # Use Macaw (legacy)+ MB_SQL_TOOLS_PARSER_BACKEND=macaw+ ```++ This is useful for:+ - Running tests with a specific backend:+ `MB_SQL_TOOLS_PARSER_BACKEND=macaw bin/mage run-tests :some/test-ns`+ - Debugging parsing differences between backends+ - Falling back to Macaw if SQLGlot has issues++ **Note**: This is an internal flag only, not exposed in public docs.+ Once SQLGlot is stable, this env var will be removed and Macaw deprecated."+ (:require+ [metabase.settings.core :as setting :refer [defsetting]]+ [metabase.util.i18n :refer [deferred-tru tru]]))++(def ^:private available-parser-backends+ #{:macaw :sqlglot})++(def ^:dynamic *parser-backend-override*+ "Dynamic var for overriding the parser backend in tests.+ When bound, [[current-parser-backend]] returns this instead of the setting."+ nil)++(defsetting sql-tools-parser-backend+ (deferred-tru "Parser backend of `sql-tools` module.")+ :visibility :internal+ :export? false+ :type :keyword+ :default :sqlglot+ :setter (fn [new-value]+ (or (available-parser-backends (keyword new-value))+ (throw (ex-info (tru "Invalid sql-tools parser backend.")+ {:value (keyword new-value)+ :available available-parser-backends})))+ (setting/set-value-of-type! :keyword :sql-tools-parser-backend new-value)))++(defn current-parser-backend+ "Get the current parser backend. Checks [[*parser-backend-override*]] first,+ then falls back to the [[sql-tools-parser-backend]] setting."+ []+ (or *parser-backend-override*+ (sql-tools-parser-backend)))diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.cljnew file mode 100644index 00000000000..904d8942126--- /dev/null+++ b/src/metabase/sql_tools/sqlglot/core.clj@@ -0,0 +1,178 @@+(ns metabase.sql-tools.sqlglot.core+ (:require+ [clojure.string :as str]+ [metabase.driver.sql :as driver.sql]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.sql-parsing.core :as sql-parsing]+ [metabase.sql-tools.common :as sql-tools.common]+ [metabase.sql-tools.interface :as sql-tools]+ [metabase.util.log :as log])+ (:import+ (org.graalvm.polyglot PolyglotException)))++(set! *warn-on-reflection* true)++(defn- convert-error+ "Convert sql-parsing generic error format to lib.validate format.+ sql-parsing returns generic maps like {:type :syntax-error} or {:type :timeout :message \"...\"},+ and this function translates them to the lib.validate error format."+ [error]+ (case (:type error)+ :syntax-error (lib/syntax-error)+ :missing-column (lib/missing-column-error (:name error))+ :missing-table-alias (lib/missing-table-alias-error (:name error))+ :timeout (lib/validation-exception-error (:message error))+ ;; Fallback: pass through if already in correct format or unknown type+ error))++(defn driver->dialect+ "Map a Metabase driver keyword to a SQLGlot dialect string.+ Returns nil for drivers that should use SQLGlot's default dialect (e.g., H2)."+ [driver]+ (case driver+ nil nil+ :postgres "postgres"+ :mysql "mysql"+ :snowflake "snowflake"+ :bigquery "bigquery"+ :bigquery-cloud-sdk "bigquery"+ :redshift "redshift"+ :sqlserver "tsql"+ :sparksql "spark"+ :presto-jdbc "presto"+ :vertica nil+ :h2 nil+ ;; Default: try using the driver name as dialect+ (name driver)))++;;;; Tables++(defn- referenced-tables+ [driver query]+ (try+ (let [db-tables (lib.metadata/tables query)+ db-transforms (lib.metadata/transforms query)+ sql (lib/raw-native-query query)+ default-schema (driver.sql/default-schema driver)+ query-tables (sql-parsing/referenced-tables (driver->dialect driver) sql)]+ (into #{}+ (keep (fn [[_catalog table-schema table]]+ (sql-tools.common/find-table-or-transform+ driver db-tables db-transforms+ (sql-tools.common/normalize-table-spec+ driver {:table table+ :schema (or table-schema default-schema)}))))+ query-tables))+ (catch PolyglotException e+ ;; Return empty sequence on parse error to follow the Macaw implementation behavior.+ (if (str/starts-with? (str (.getMessage e)) "ParseError")+ #{}+ (throw e)))))++(defmethod sql-tools/referenced-tables-impl :sqlglot+ [_parser driver query]+ (referenced-tables driver query))++;;;; field-references++(def ^:private ^:const normalizable-keys+ "Keys that should have their string values normalized for case.+ Matches Macaw's col-fields which normalizes [:type :column :table :schema :database :alias]."+ #{:type :column :table :schema :database :alias})++(defn- normalize-field+ "Normalize identifier strings in a field spec using driver-specific case rules.+ Only normalizes specific keys (column, table, schema, etc.) to match Macaw's col-fields."+ [driver field]+ (if (map? field)+ (reduce-kv (fn [m k v]+ (assoc m k+ (cond+ (and (normalizable-keys k) (string? v))+ (driver.sql/normalize-name driver v)++ (map? v)+ (normalize-field driver v)++ (set? v)+ (set (map #(normalize-field driver %) v))++ (sequential? v)+ (mapv #(normalize-field driver %) v)++ :else v)))+ {}+ field)+ field))++(defmethod sql-tools/field-references-impl :sqlglot+ [_parser driver sql-string]+ (let [result (sql-parsing/field-references (driver->dialect driver) sql-string)]+ (-> result+ (update :used-fields #(set (map (partial normalize-field driver) %)))+ (update :returned-fields #(mapv (partial normalize-field driver) %))+ (update :errors #(set (map convert-error %))))))++;;;; referenced-fields++(defmethod sql-tools/referenced-fields-impl :sqlglot+ [parser driver query]+ (sql-tools.common/referenced-fields parser driver query))++;;;; Validation+;; SQLGlot validation uses the same pipeline as Macaw:+;; field-references (dispatched to :sqlglot) → resolve-field (Macaw's logic)+(defmethod sql-tools/validate-query-impl :sqlglot+ [parser driver query]+ (sql-tools.common/validate-query parser driver query))++;;;; returned-columns+;; SQLGlot returned-columns uses the same pipeline as Macaw:+;; field-references (dispatched to :sqlglot) → resolve-field (Macaw's logic)+(defmethod sql-tools/returned-columns-impl :sqlglot+ [parser driver query]+ (sql-tools.common/returned-columns parser driver query))++(defmethod sql-tools/referenced-tables-raw-impl :sqlglot+ [_parser driver sql-str]+ (try+ (let [dialect (driver->dialect driver)+ ;; sql-parsing/referenced-tables returns [[catalog schema table] ...]+ ;; Convert to [{:schema ... :table ...} ...] format.+ ;; Do NOT normalize case here — SQLGlot already applies dialect-appropriate case rules+ ;; (e.g., uppercase for Snowflake, lowercase for Postgres). Additional normalization+ ;; via normalize-table-spec would incorrectly lowercase Snowflake identifiers, breaking+ ;; AppDB lookups where identifiers are stored in their native case.+ ;; This matches the Macaw implementation which also returns raw identifiers.+ table-tuples (sql-parsing/referenced-tables dialect sql-str)]+ (mapv (fn [[_catalog schema table]]+ {:schema schema :table table})+ table-tuples))+ (catch PolyglotException e+ ;; Return empty sequence on parse error to follow the Macaw implementation behavior.+ (if (str/starts-with? (str (.getMessage e)) "ParseError")+ []+ (throw e)))))++(defmethod sql-tools/simple-query?-impl :sqlglot+ [_parser sql-string]+ (try+ ;; No dialect available from caller, use nil for SQLGlot's default dialect+ (sql-parsing/simple-query? nil sql-string)+ (catch Exception e+ (log/debugf e "Failed to parse query: %s" (ex-message e))+ {:is_simple false})))++(defmethod sql-tools/add-into-clause-impl :sqlglot+ [_parser driver sql table-name]+ (sql-parsing/add-into-clause (driver->dialect driver) sql table-name))++(defmethod sql-tools/replace-names-impl :sqlglot+ [_parser driver sql-string replacements _opts]+ ;; Convert map keys to list-of-pairs for JSON serialization+ ;; {:tables {{:table "a"} "b"}} -> {:tables [[{:table "a"} "b"]]}+ (let [replacements' (-> replacements+ (update :tables #(when % (vec %)))+ (update :columns #(when % (vec %))))]+ (sql-parsing/replace-names (driver->dialect driver) sql-string replacements')))diff --git a/src/metabase/transforms/api/transform.clj b/src/metabase/transforms/api/transform.cljindex 322939f8614..aed523ae748 100644--- a/src/metabase/transforms/api/transform.clj+++ b/src/metabase/transforms/api/transform.clj@@ -17,6 +17,7 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.schema :as qp.schema][metabase.request.core :as request]+ [metabase.sql-tools.core :as sql-tools][metabase.transforms.api.transform-job][metabase.transforms.api.transform-tag][metabase.transforms.canceling :as transforms.canceling]@@ -33,11 +34,8 @@[metabase.util.malli.schema :as ms][ring.util.response :as response][toucan2.core :as t2])- ;; TODO (Chris 2026-01-22) -- Remove jsqlparser imports/typehints to be SQL parser-agnostic(:import- (java.sql PreparedStatement)- ^{:clj-kondo/ignore [:metabase/no-jsqlparser-imports]}- (net.sf.jsqlparser.statement.select PlainSelect)))+ (java.sql PreparedStatement)))(comment metabase.transforms.api.transform-job/keep-memetabase.transforms.api.transform-tag/keep-me)@@ -541,36 +539,10 @@(run-transform! (api/write-check :model/Transform id)))(defn- simple-native-query?- "Checks if a native SQL query string is simple enough for automatic checkpoint insertion."+ "Checks if a native SQL query string is simple enough for automatic checkpoint insertion.+ Delegates to sql-tools which dispatches to the configured parser backend (macaw or sqlglot)."[sql-string]- (try- ;; BEWARE: The API endpoint (caller) does not have info on database engine this query should run on. Hence- ;; there's no way of providing appropriate [[metabase.driver.util/macaw-options]]. `nil` is best-effort- ;; adding at least default :non-resserved-words.- ;; TODO (Chris 2026-01-22) -- Remove jsqlparser typehints to be SQL parser-agnostic- (let [^PlainSelect parsed (driver.u/parsed-query sql-string nil)]- (cond- (not (instance? PlainSelect parsed))- {:is_simple false- :reason "Not a simple SELECT"}-- (.getLimit parsed)- {:is_simple false- :reason "Contains a LIMIT"}-- (.getOffset ^PlainSelect parsed)- {:is_simple false- :reason "Contains an OFFSET"}-- (seq (.getWithItemsList ^PlainSelect parsed))- {:is_simple false- :reason "Contains a CTE"}-- :else- {:is_simple true}))- (catch Exception e- (log/debugf e "Failed to parse query: %s" (ex-message e))- {:is_simple false})))+ (sql-tools/simple-query? sql-string))(api.macros/defendpoint :post "/is-simple-query" :- [:map[:is_simple :boolean]diff --git a/test/metabase/sql_parsing/core_test.clj b/test/metabase/sql_parsing/core_test.cljnew file mode 100644index 00000000000..8b0d0ac79d9--- /dev/null+++ b/test/metabase/sql_parsing/core_test.clj@@ -0,0 +1,484 @@+(ns metabase.sql-parsing.core-test+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [metabase.sql-parsing.core :as sql-parsing]+ [metabase.util :as u]))++(set! *warn-on-reflection* true)++;;; ------------------------------------------ referenced-tables API Tests -----------------------------------------++(deftest ^:parallel referenced-tables-basic-test+ (testing "Simple table extraction"+ (is (= [[nil nil "users"]]+ (sql-parsing/referenced-tables "postgres" "SELECT * FROM users"))))++ (testing "Multiple tables from JOIN"+ (is (= [[nil nil "orders"] [nil nil "users"]]+ (sql-parsing/referenced-tables+ "postgres"+ "SELECT * FROM users u LEFT JOIN orders o ON u.id = o.user_id"))))++ (testing "Schema-qualified tables are preserved"+ (is (= [[nil "other" "users"] [nil "public" "users"]]+ (sql-parsing/referenced-tables+ "postgres"+ "SELECT public.users.id, other.users.id+ FROM public.users u1+ LEFT JOIN other.users u2 ON u1.id = u2.id"))))++ (testing "CTE names are excluded"+ (is (= [[nil nil "users"]]+ (sql-parsing/referenced-tables+ "postgres"+ "WITH active AS (SELECT * FROM users WHERE active) SELECT * FROM active")))))++(deftest ^:parallel cte-test+ (testing "CTE (WITH clause) parsing"+ (is (= [[nil nil "users"]]+ (sql-parsing/referenced-tables+ "postgres"+ "WITH active_users AS (SELECT * FROM users WHERE active)+ SELECT * FROM active_users")))))++(deftest ^:parallel join-test+ (testing "JOIN parsing extracts all tables"+ (is (=? [[nil nil "orders"] [nil nil "users"]]+ (sql-parsing/referenced-tables+ "postgres"+ "SELECT u.name, o.total+ FROM users u+ JOIN orders o ON u.id = o.user_id")))))++(deftest ^:parallel subquery-test+ (testing "Subquery table extraction"+ (is (=? [[nil nil "users"]]+ (sql-parsing/referenced-tables+ "postgres"+ "SELECT * FROM (SELECT id FROM users) AS sub")))))++;;; ---------------------------------------- Catalog/Schema Extraction Tests ----------------------------------------+;;; These tests verify the 3-tuple (tables) and 4-tuple (fields) API per the design doc:+;;; - BigQuery: project.dataset.table → [project, dataset, table]+;;; - Snowflake: database.schema.table → [database, schema, table]+;;; - PostgreSQL: schema.table → [nil, schema, table]+;;; - MySQL: database.table → [nil, database, table] (MySQL's "database" = schema level)++(deftest ^:parallel catalog-schema-tables-bigquery-test+ (testing "BigQuery: project.dataset.table extracts all three levels"+ (is (= [["myproject" "analytics" "events"]]+ (sql-parsing/referenced-tables "bigquery" "SELECT * FROM myproject.analytics.events")))+ (is (= [["myproject" "analytics" "events"] ["myproject" "analytics" "users"]]+ (sql-parsing/referenced-tables "bigquery"+ "SELECT * FROM myproject.analytics.events e+ JOIN myproject.analytics.users u ON e.user_id = u.id")))))++(deftest ^:parallel catalog-schema-tables-snowflake-test+ (testing "Snowflake: database.schema.table extracts all three levels"+ ;; Snowflake uppercases identifiers by default+ (let [result (sql-parsing/referenced-tables "snowflake" "SELECT * FROM mydb.public.orders")+ normalized (mapv (fn [[c s t]] [(some-> c u/lower-case-en)+ (some-> s u/lower-case-en)+ (u/lower-case-en t)])+ result)]+ (is (= [["mydb" "public" "orders"]] normalized)))))++(deftest ^:parallel catalog-schema-tables-postgres-test+ (testing "PostgreSQL: schema.table (no cross-database queries)"+ (is (= [[nil "public" "users"]]+ (sql-parsing/referenced-tables "postgres" "SELECT * FROM public.users")))+ (is (= [[nil "other_schema" "accounts"] [nil "public" "users"]]+ (sql-parsing/referenced-tables "postgres"+ "SELECT * FROM public.users u+ JOIN other_schema.accounts a ON u.id = a.user_id")))))++(deftest ^:parallel catalog-schema-tables-mysql-test+ (testing "MySQL: database.table (MySQL 'database' maps to schema slot)"+ ;; In MySQL, CREATE SCHEMA = CREATE DATABASE, so there's no separate schema level+ ;; SQLGlot maps MySQL's database to the schema slot (slot 2)+ (is (= [[nil "mydb" "users"]]+ (sql-parsing/referenced-tables "mysql" "SELECT * FROM mydb.users")))+ (is (= [[nil nil "users"]]+ (sql-parsing/referenced-tables "mysql" "SELECT * FROM users")))))++(deftest ^:parallel catalog-schema-fields-bigquery-test+ (testing "BigQuery: fully-qualified field references"+ (is (= [["myproject" "analytics" "users" "email"]]+ (sql-parsing/referenced-fields "bigquery"+ "SELECT myproject.analytics.users.email+ FROM myproject.analytics.users")))))++(deftest ^:parallel catalog-schema-fields-postgres-test+ (testing "PostgreSQL: schema-qualified field references"+ (is (= [[nil "public" "orders" "total"]]+ (sql-parsing/referenced-fields "postgres" "SELECT public.orders.total FROM public.orders")))))++(deftest ^:parallel catalog-schema-fields-mysql-test+ (testing "MySQL: database-qualified field references"+ (is (= [[nil "mydb" "users" "id"]]+ (sql-parsing/referenced-fields "mysql" "SELECT mydb.users.id FROM mydb.users")))+ (is (= [[nil nil "users" "id"]]+ (sql-parsing/referenced-fields "mysql" "SELECT id FROM users")))))++;;; -------------------------------------------- UDTF (Table Function) Tests -----------------------------------------++;; Dialect-specific UDTF queries - each dialect may have slightly different syntax+;; for table-valued functions. The queries should be semantically equivalent.+;; We use a generic function name for dialects without well-known UDTFs to test+;; that unknown functions are handled gracefully.+(def udtf-queries+ {:postgres "SELECT * FROM generate_series(1, 10)"+ :mysql "SELECT * FROM my_table_func(1, 10)" ; MySQL's JSON_TABLE has complex syntax+ :snowflake "SELECT * FROM TABLE(FLATTEN(input => ARRAY_CONSTRUCT(1, 2, 3)))"+ :bigquery "SELECT * FROM UNNEST([1, 2, 3]) AS val"+ :redshift "SELECT * FROM generate_series(1, 10)"+ :duckdb "SELECT * FROM generate_series(1, 10)"})++;; UDTF queries mixed with real tables - tests that we correctly identify real tables+;; while gracefully handling table functions+(def udtf-with-table-queries+ {:postgres "SELECT o.* FROM orders o, generate_series(1, 10) g"+ :mysql "SELECT o.* FROM orders o, my_table_func(1, 10) t"+ :snowflake "SELECT o.* FROM orders o, TABLE(FLATTEN(input => ARRAY_CONSTRUCT(1))) f"+ :bigquery "SELECT o.* FROM orders o, UNNEST([1]) AS val"+ :redshift "SELECT o.* FROM orders o, generate_series(1, 10) g"+ :duckdb "SELECT o.* FROM orders o, generate_series(1, 10) g"})++(deftest ^:parallel udtf-referenced-tables-test+ (testing "UDTFs are handled by referenced-tables across dialects"+ (doseq [[dialect sql] udtf-queries]+ (testing (str "dialect: " dialect " - pure UDTF query")+ ;; Table functions shouldn't appear as referenced tables - they're not real tables+ (let [result (sql-parsing/referenced-tables (name dialect) sql)]+ (is (vector? result)+ (format "dialect %s: should return vector, got %s" dialect (type result)))+ (is (every? vector? result)+ (format "dialect %s: each element should be [catalog schema table] tuple" dialect)))))++ (doseq [[dialect sql] udtf-with-table-queries]+ (testing (str "dialect: " dialect " - UDTF mixed with real table")+ (let [result (sql-parsing/referenced-tables (name dialect) sql)]+ ;; Case-insensitive comparison since dialects like Snowflake uppercase identifiers+ ;; 3-tuple: [catalog schema table] - table is third element+ (is (some #(= "orders" (u/lower-case-en (nth % 2))) result)+ (format "dialect %s: should find 'orders' table in %s" dialect (pr-str result))))))))++;;; ------------------------------------------ Set Operation Tests (UNION, INTERSECT, EXCEPT) ------------------------------------------++;; Set operation queries by dialect - tests UNION, UNION ALL, INTERSECT, EXCEPT+;; All queries use tables "a" and "b" for consistent expected results+(def dialect->set-operation->query+ {:postgres+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}++ :mysql+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}++ :snowflake+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}++ :bigquery+ {:union "SELECT id, name FROM a UNION DISTINCT SELECT id, name FROM b"+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"+ :intersect "SELECT id FROM a INTERSECT DISTINCT SELECT id FROM b"+ :except "SELECT id FROM a EXCEPT DISTINCT SELECT id FROM b"+ :nested-union "SELECT * FROM (SELECT id FROM a UNION DISTINCT SELECT id FROM b) AS combined"+ :cte-with-union "WITH c AS (SELECT id FROM a UNION DISTINCT SELECT id FROM b) SELECT * FROM c"}++ :redshift+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}++ :duckdb+ {:union "SELECT id, name FROM a UNION SELECT id, name FROM b"+ :union-all "SELECT id, name FROM a UNION ALL SELECT id, name FROM b"+ :intersect "SELECT id FROM a INTERSECT SELECT id FROM b"+ :except "SELECT id FROM a EXCEPT SELECT id FROM b"+ :nested-union "SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS combined"+ :cte-with-union "WITH c AS (SELECT id FROM a UNION SELECT id FROM b) SELECT * FROM c"}})++(deftest ^:parallel set-operation-referenced-tables-test+ (testing "Set operations correctly identify all referenced tables across dialects"+ (doseq [[dialect ops] dialect->set-operation->query+ [op-type sql] ops]+ (testing (str "dialect: " (name dialect) " - " (name op-type))+ (let [result (sql-parsing/referenced-tables (name dialect) sql)+ ;; Normalize to lowercase for case-insensitive comparison (Snowflake uppercases)+ ;; 3-tuple: [catalog schema table]+ normalized (into #{} (map (fn [[catalog schema table]]+ [(some-> catalog u/lower-case-en)+ (some-> schema u/lower-case-en)+ (u/lower-case-en table)])+ result))]+ (is (= #{[nil nil "a"] [nil nil "b"]} normalized)+ (format "%s/%s should return [[nil nil a] [nil nil b]], got %s"+ (name dialect) (name op-type) normalized)))))))++(def ^:private set-operation->returned-columns+ {:union ["id" "name"]+ :union-all ["id" "name"]+ :intersect ["id"]+ :except ["id"]+ :nested-union ["id"]+ :cte-with-union ["id"]})++(deftest ^:parallel set-operation-validate-query-test+ (testing "Set operations validate correctly across dialects"+ (doseq [[dialect ops] dialect->set-operation->query+ [op-type sql] ops]+ (testing (str "dialect: " (name dialect) " - " (name op-type))+ (let [result (sql-parsing/validate-query (name dialect) sql "public" {})]+ (is (= "ok" (:status result))+ (format "%s/%s should validate OK, got %s"+ (name dialect) (name op-type) result)))))))++(deftest ^:parallel set-operation-returned-columns-lineage-test+ (testing "Set operations return correct column lineage across dialects"+ (doseq [[dialect ops] dialect->set-operation->query+ [op-type sql] ops+ :let [expected (get set-operation->returned-columns op-type)]]+ (testing (str "dialect: " (name dialect) ", operation: " (name op-type))+ (let [result (sql-parsing/returned-columns-lineage (name dialect) sql "public" {})+ columns (sort (map (comp u/lower-case-en first) result))]+ (is (= expected columns)+ (format "%s %s should return %s, got %s"+ (name dialect) (name op-type) (pr-str expected) columns)))))))++;;; -------------------------------------------- Schema Validation Tests (validate-query) ------------------------------------------------++;; validate-query validates SQL against a provided schema, detecting:+;; - Missing columns (column doesn't exist in any table)+;; - Missing table aliases (table wildcard references non-existent source)+;; - Unknown tables (table not in schema)++(def ^:private test-schema+ "Test schema for validation tests.+ Structure: {schema-name {table-name {column-name type}}}"+ {"PUBLIC" {"PRODUCTS" {"ID" "INT"+ "TITLE" "VARCHAR"+ "CATEGORY" "VARCHAR"+ "PRICE" "DECIMAL"}+ "ORDERS" {"ID" "INT"+ "USER_ID" "INT"+ "PRODUCT_ID" "INT"+ "TOTAL" "DECIMAL"}+ "USERS" {"ID" "INT"+ "NAME" "VARCHAR"+ "EMAIL" "VARCHAR"}}})++(deftest ^:parallel validate-query-valid-queries-test+ (testing "Valid queries against schema"+ (testing "simple select with existing columns"+ (is (= "ok" (:status (sql-parsing/validate-query nil "SELECT id, title FROM products" "PUBLIC" test-schema)))))++ (testing "wildcard select"+ (is (= "ok" (:status (sql-parsing/validate-query nil "SELECT * FROM products" "PUBLIC" test-schema)))))++ (testing "table-qualified columns"+ (is (= "ok" (:status (sql-parsing/validate-query nil "SELECT products.id, products.title FROM products" "PUBLIC" test-schema)))))++ (testing "join with valid columns"+ (is (= "ok" (:status (sql-parsing/validate-query nil+ "SELECT o.id, p.title FROM orders o JOIN products p ON o.product_id = p.id"+ "PUBLIC" test-schema)))))++ (testing "subquery"+ (is (= "ok" (:status (sql-parsing/validate-query nil+ "SELECT * FROM (SELECT id, title FROM products) AS sub"+ "PUBLIC" test-schema)))))))++(deftest ^:parallel validate-query-missing-column-test+ (testing "Missing column errors"+ (testing "non-existent column"+ (let [result (sql-parsing/validate-query nil "SELECT bad_column FROM products" "PUBLIC" test-schema)]+ (is (= "error" (:status result)))+ (is (= "column_not_resolved" (:type result)))+ (is (re-find #"(?i)bad_column" (:column result)))))++ (testing "column from wrong table"+ (let [result (sql-parsing/validate-query nil "SELECT email FROM products" "PUBLIC" test-schema)]+ (is (= "error" (:status result)))+ (is (= "column_not_resolved" (:type result)))))))++(deftest ^:parallel validate-query-missing-table-alias-test+ (testing "Missing table alias errors"+ (testing "table wildcard with non-existent table"+ (let [result (sql-parsing/validate-query nil "SELECT products.* FROM orders" "PUBLIC" test-schema)]+ (is (= "error" (:status result)))+ ;; SQLGlot reports this as unknown_table+ (is (contains? #{"unknown_table" "column_not_resolved"} (:type result)))))++ (testing "qualified column with non-existent alias"+ (let [result (sql-parsing/validate-query nil "SELECT p.id FROM products" "PUBLIC" test-schema)]+ (is (= "error" (:status result)))))))++(deftest ^:parallel validate-query-unknown-table-test+ ;; SQLGlot's qualify.qualify() does NOT detect unknown tables - it just doesn't qualify them.+ ;; Unknown table detection happens at the sql-tools layer by matching referenced tables+ ;; against Metabase's database metadata.+ (testing "Unknown tables are not detected by SQLGlot (known limitation)"+ (let [result (sql-parsing/validate-query nil "SELECT * FROM nonexistent" "PUBLIC" test-schema)]+ (is (= "ok" (:status result))+ "SQLGlot passes unknown tables through without error"))))++(deftest ^:parallel validate-query-syntax-error-test+ (testing "Syntax errors are caught"+ (let [result (sql-parsing/validate-query nil "SELECT * FORM products" "PUBLIC" test-schema)]+ (is (= "error" (:status result)))+ (is (= "invalid_expression" (:type result))))))++(deftest ^:parallel lenient-parsing-incomplete-limit-test+ (testing "SQLGlot's lenient parsing treats 'SELECT 1 LIMIT' as 'SELECT 1 AS LIMIT'"+ ;; This documents a known SQLGlot behavior where incomplete LIMIT clauses+ ;; are parsed as column aliases.+ (let [result (sql-parsing/referenced-tables "postgres" "SELECT 1 LIMIT")]+ (is (= [] result)+ "No tables are referenced in 'SELECT 1 LIMIT'")))++ (testing "To reliably trigger SQL errors, use nonexistent tables instead"+ ;; This is the recommended pattern for tests that need to trigger SQL failures+ (let [result (sql-parsing/referenced-tables "postgres" "SELECT * FROM nonexistent_table_xyz")]+ (is (= [[nil nil "nonexistent_table_xyz"]] result)+ "Nonexistent table reference is parsed and will fail at execution time"))))++(comment+ (require '[clojure.string :as str])+ (def query-corpus-path "/Users/bcm/dv/mb/query_corpus/")++ (def sentinel (re-pattern "\n-----end-query-----\n"))++ (def drivers ["mysql"])++ (let [driver (first drivers)+ corpus (slurp (str query-corpus-path driver ".log"))+ queries (sort (distinct (str/split corpus sentinel)))]++ (frequencies+ (doall+ (for [q queries]+ (try+ (sql-parsing/referenced-tables q driver)+ true+ (catch Exception _ false))))))++ ;; Load and run validation tests interactively+ (load-validation-test-cases)++ ;; Test a single case+ (validate-test-case {:name "test"+ :sql "SELECT * FROM users"+ :dialect "postgres"+ :expected :valid})++ ;; Run all valid query tests+ (doseq [tc (:valid-queries (load-validation-test-cases))]+ (let [result (validate-test-case tc)]+ (when-not (:passed result)+ result)))++ ;; Run all invalid query tests+ (doseq [tc (:invalid-queries (load-validation-test-cases))]+ (let [result (validate-test-case tc)]+ (when-not (:passed result)+ result))))++;;; -------------------------------------------- Referenced Fields Tests --------------------------------------------++(defn- load-referenced-fields-test-cases+ "Load referenced fields test cases from EDN file."+ []+ (let [resource (io/resource "metabase/sql_parsing/referenced_fields_test_cases.edn")]+ (when-not resource+ (throw (ex-info "Could not find referenced_fields_test_cases.edn" {})))+ (read-string (slurp resource))))++(defn- normalize-fields+ "Normalize field references for comparison (sort and ensure consistent format).+ Handles both 2-tuples (expected format in test cases) and 4-tuples (actual API format)."+ [fields]+ (vec (sort-by (fn [f]+ (if (= 4 (count f))+ ;; 4-tuple: [catalog schema table field]+ [(u/lower-case-en (nth f 2)) (u/lower-case-en (nth f 3))]+ ;; 2-tuple: [table field] (test case format)+ [(u/lower-case-en (first f)) (u/lower-case-en (second f))]))+ (map (fn [f]+ (if (= 4 (count f))+ ;; Convert 4-tuple to 2-tuple for comparison+ [(u/lower-case-en (nth f 2)) (u/lower-case-en (nth f 3))]+ ;; Already 2-tuple+ [(u/lower-case-en (first f)) (u/lower-case-en (second f))]))+ fields))))++(defn- fields-match?+ "Check if actual fields match expected fields (case-insensitive, order-independent)."+ [expected actual]+ (= (normalize-fields expected)+ (normalize-fields actual)))++(deftest ^:parallel referenced-fields-test+ (testing "Referenced fields extraction from SQL queries"+ (let [test-cases (:test-cases (load-referenced-fields-test-cases))]+ (doseq [test-case test-cases]+ (testing (:name test-case)+ (let [{:keys [sql dialect expected]} test-case+ actual (sql-parsing/referenced-fields dialect sql)]+ (is (fields-match? expected actual)+ (str "Test '" (:name test-case) "' failed"+ "\n SQL: " sql+ "\n Expected: " (pr-str (normalize-fields expected))+ "\n Actual: " (pr-str (normalize-fields actual))))))))))++(deftest ^:parallel referenced-fields-dialect-support-test+ (testing "Referenced fields works across different SQL dialects"+ (doseq [dialect ["postgres" "mysql" "snowflake" "bigquery" "redshift" "duckdb"]]+ (testing (str "dialect: " dialect)+ (let [result (sql-parsing/referenced-fields dialect "SELECT id, name FROM users WHERE active = true")]+ (is (seq result)+ (str "Should return fields for " dialect))+ (is (every? #(and (vector? %) (= 4 (count %))) result)+ (str "Each field should be [catalog schema table field] tuple for " dialect)))))))++(deftest ^:parallel referenced-fields-wildcard-test+ (testing "Wildcard handling in referenced fields"+ (testing "Unqualified wildcard - single table"+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT * FROM users")]+ (is (fields-match? [["users" "*"]] result))))++ (testing "Unqualified wildcard - multiple tables"+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT * FROM users u LEFT JOIN orders o ON u.id = o.user_id")]+ (is (some #(= ["orders" "*"] %) (normalize-fields result))+ "Should include orders wildcard")+ (is (some #(= ["users" "*"] %) (normalize-fields result))+ "Should include users wildcard")))++ (testing "Qualified wildcard"+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT u.* FROM users u")]+ (is (fields-match? [["users" "*"]] result))))++ (testing "Mixed wildcards and specific columns"+ (let [result (sql-parsing/referenced-fields "postgres" "SELECT u.*, t.total FROM users u, transactions t WHERE u.id = t.user_id")]+ (is (some #(= ["users" "*"] %) (normalize-fields result))+ "Should include users wildcard")+ (is (some #(= ["transactions" "total"] %) (normalize-fields result))+ "Should include transactions.total")))))diff --git a/test/metabase/sql_parsing/pool_test.clj b/test/metabase/sql_parsing/pool_test.cljnew file mode 100644index 00000000000..8d09a45451c--- /dev/null+++ b/test/metabase/sql_parsing/pool_test.clj@@ -0,0 +1,163 @@+(ns metabase.sql-parsing.pool-test+ (:require+ [clojure.test :refer :all]+ [metabase.sql-parsing.pool :as pool])+ (:import+ (io.aleph.dirigiste Pool)+ (java.util.concurrent CountDownLatch)))++(set! *warn-on-reflection* true)++(defn- mock-context+ "Create a mock 'context' object. Since Context is final, we just return a map with metadata.+ The pool doesn't care what type the objects are - it just stores and returns them."+ [id]+ {:context-id id+ :created-at (System/currentTimeMillis)})++(defn- counting-generator+ "Returns a generator function that counts how many contexts it has created."+ [counter-atom]+ (fn []+ (let [id (swap! counter-atom inc)]+ (mock-context id))))++(defn- failing-generator+ "Returns a generator that fails after creating n contexts."+ [counter-atom fail-after]+ (fn []+ (let [id (swap! counter-atom inc)]+ (when (> id fail-after)+ (throw (ex-info "Generator limit reached" {:count id :limit fail-after})))+ (mock-context id))))++;;; ------------------------------------------------- Test Helpers ---------------------------------------------------++(defn- test-pool+ "Create a pool for testing with sensible defaults.+ Returns [pool created-count-atom]."+ ([]+ (test-pool {}))+ ([config]+ (let [counter (atom 0)+ generator (counting-generator counter)+ pool (#'pool/make-python-context-pool generator config)]+ [pool counter])))++(defn- with-pool+ "Execute f with a context from pool. Uses with-open for proper resource management."+ [pool f]+ (with-open [^java.io.Closeable ctx (#'pool/acquire-context pool)]+ (f ctx)))++(defn- run-concurrent+ "Run n threads concurrently, each executing f with a context from pool.+ Returns vector of results."+ [pool n f]+ (let [latch (CountDownLatch. n)+ results (atom [])+ futures (doall+ (for [i (range n)]+ (future+ (.countDown latch)+ (.await latch)+ (let [result (with-pool pool (fn [ctx] (f i ctx)))]+ (swap! results conj result)))))]+ (doseq [fut futures] @fut)+ @results))++;;; ------------------------------------------------ Pool Size Tests -------------------------------------------------++(deftest pool-respects-maximum-size-test+ (testing "Pool never creates more than max contexts even under concurrent load"+ (let [[pool created-count] (test-pool {:max-size 3})+ num-threads 20+ results (run-concurrent pool num-threads+ (fn [_i ctx]+ (Thread/sleep (long (+ 10 (rand-int 50))))+ ctx))]++ (is (<= @created-count 3)+ (str "Pool created " @created-count " contexts but max is 3"))+ (is (= num-threads (count results))+ "All threads should have received a context"))))++(deftest pool-maintains-minimum-size-test+ (testing "Pool keeps at least 1 context available"+ (let [[pool created-count] (test-pool)]+ (with-pool pool (fn [_ctx] (is (= 1 @created-count) "First use creates one context")))+ (Thread/sleep (long 100))+ (with-pool pool (fn [_ctx] (is (= 1 @created-count) "Second use reuses the same context"))))))++;;; ----------------------------------------------- Context Expiry Tests ---------------------------------------------++(deftest pool-expires-old-contexts-test+ (testing "Pool disposes of contexts that have exceeded their TTL"+ (let [[pool _created-count] (test-pool {:ttl-minutes 10})+ [context expiry-ts :as tuple] (.acquire ^Pool pool :python)]+ (try+ (is (some? context) "Context should be created")+ (is (number? expiry-ts) "Expiry timestamp should be set")+ (is (< (System/nanoTime) expiry-ts) "Expiry should be in the future")++ (let [expiry-in-minutes (/ (- expiry-ts (System/nanoTime)) (* 1000000000 60))]+ (is (< 9 expiry-in-minutes 11) "Expiry should be approximately 10 minutes"))+ (finally+ (.release ^Pool pool :python tuple))))))++(deftest with-pooled-context-handles-expired-contexts-test+ (testing "with-pooled-context properly handles and replaces expired contexts"+ (let [[pool created-count] (test-pool)]+ (with-pool pool (fn [_ctx] (is (= 1 @created-count))))++ ;; Manually expire by disposing+ (let [tuple (.acquire ^Pool pool :python)]+ (.dispose ^Pool pool :python tuple))++ (with-pool pool (fn [_ctx] (is (= 2 @created-count) "Expired context should be replaced"))))))++;;; --------------------------------------------- Generator Failure Tests --------------------------------------------++(deftest pool-handles-generator-failures-test+ (testing "Pool gracefully handles when generator fails"+ (let [counter (atom 0)+ generator (failing-generator counter 1)+ pool (#'pool/make-python-context-pool generator)]++ (is (some? (with-pool pool identity)) "First context creation succeeds")++ ;; Dispose to force new creation+ (let [tuple (.acquire ^Pool pool :python)]+ (.dispose ^Pool pool :python tuple))++ (is (thrown? Exception (with-pool pool identity)) "Generator failure should propagate"))))++;;; ------------------------------------------- Concurrent Access Tests ----------------------------------------------++(deftest pool-handles-concurrent-access-safely-test+ (testing "Pool handles concurrent access without race conditions"+ (let [[pool created-count] (test-pool {:max-size 3})+ num-threads 50+ results (run-concurrent pool num-threads+ (fn [i _ctx]+ (Thread/sleep (long (rand-int 20)))+ i))]++ (is (= num-threads (count results)) "All threads should complete successfully")+ (is (<= @created-count 3) (str "Created " @created-count " contexts, expected <= 3")))))++;;; -------------------------------------------- Pool Behavior Tests -------------------------------------------------++(deftest pool-stores-and-returns-objects-test+ (testing "Pool correctly stores and returns arbitrary objects"+ (let [[pool created-count] (test-pool)+ result1 (with-pool pool :context)+ result2 (with-pool pool :context)]++ (is (map? result1))+ (is (= 1 (:context-id result1)))+ (is (= 1 @created-count))++ (is (map? result2))+ (is (= 1 (:context-id result2))) ;; Same context ID+ (is (= 1 @created-count)))))diff --git a/test/metabase/sql_parsing/referenced_fields_test_cases.edn b/test/metabase/sql_parsing/referenced_fields_test_cases.ednnew file mode 100644index 00000000000..6e35418a9d9--- /dev/null+++ b/test/metabase/sql_parsing/referenced_fields_test_cases.edn@@ -0,0 +1,244 @@+;; Referenced Fields Test Cases+;; Each test case has:+;; - :name - descriptive name for the test+;; - :sql - the SQL query to analyze+;; - :dialect - SQL dialect (postgres, mysql, snowflake, bigquery, redshift, duckdb)+;; - :expected - vector of [table field] pairs (sorted)++{:test-cases+ [;; Basic SELECT queries+ {:name "simple-select-with-columns"+ :sql "SELECT id, name FROM users"+ :dialect "postgres"+ :expected [["users" "id"] ["users" "name"]]}++ {:name "select-with-where-clause"+ :sql "SELECT id, name FROM users WHERE active = true"+ :dialect "postgres"+ :expected [["users" "active"] ["users" "id"] ["users" "name"]]}++ {:name "select-with-multiple-where-conditions"+ :sql "SELECT id, name FROM users WHERE active = true AND age > 18"+ :dialect "postgres"+ :expected [["users" "active"] ["users" "age"] ["users" "id"] ["users" "name"]]}++ {:name "select-with-order-by"+ :sql "SELECT id, name FROM users ORDER BY created_at DESC"+ :dialect "postgres"+ :expected [["users" "created_at"] ["users" "id"] ["users" "name"]]}++ {:name "select-with-limit"+ :sql "SELECT id, name FROM users LIMIT 10"+ :dialect "postgres"+ :expected [["users" "id"] ["users" "name"]]}++ ;; Wildcards+ {:name "unqualified-wildcard-single-table"+ :sql "SELECT * FROM users"+ :dialect "postgres"+ :expected [["users" "*"]]}++ {:name "unqualified-wildcard-with-where"+ :sql "SELECT * FROM users WHERE active = true"+ :dialect "postgres"+ :expected [["users" "*"] ["users" "active"]]}++ {:name "qualified-wildcard"+ :sql "SELECT u.* FROM users u"+ :dialect "postgres"+ :expected [["users" "*"]]}++ {:name "qualified-wildcard-with-specific-column"+ :sql "SELECT u.*, t.total FROM users u, transactions t WHERE u.id = t.user_id"+ :dialect "postgres"+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "*"] ["users" "id"]]}++ ;; JOINs+ {:name "inner-join-simple"+ :sql "SELECT u.id, o.total FROM users u INNER JOIN orders o ON u.id = o.user_id"+ :dialect "postgres"+ :expected [["orders" "total"] ["orders" "user_id"] ["users" "id"]]}++ {:name "left-join-with-aliases"+ :sql "SELECT t.id, u.email FROM transactions t LEFT JOIN users u ON t.user_id = u.id"+ :dialect "postgres"+ :expected [["transactions" "id"] ["transactions" "user_id"] ["users" "email"] ["users" "id"]]}++ {:name "multiple-joins"+ :sql "SELECT u.name, o.total, p.name FROM users u LEFT JOIN orders o ON u.id = o.user_id LEFT JOIN products p ON o.product_id = p.id"+ :dialect "postgres"+ :expected [["orders" "product_id"] ["orders" "total"] ["orders" "user_id"] ["products" "id"] ["products" "name"] ["users" "id"] ["users" "name"]]}++ {:name "join-with-wildcard"+ :sql "SELECT * FROM users u LEFT JOIN transactions t ON u.id = t.user_id"+ :dialect "postgres"+ :expected [["transactions" "*"] ["transactions" "user_id"] ["users" "*"] ["users" "id"]]}++ {:name "cross-join"+ :sql "SELECT u.id, t.total FROM users u, transactions t WHERE u.id = t.user_id"+ :dialect "postgres"+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "id"]]}++ ;; CTEs+ ;; Note: CTEs create scopes, and sqlglot attributes columns to the CTE scope in some cases+ {:name "simple-cte"+ :sql "WITH active_users AS (SELECT id FROM users WHERE active) SELECT * FROM active_users"+ :dialect "postgres"+ :expected [["users" "active"] ["users" "id"]]}++ {:name "cte-with-join"+ :sql "WITH active AS (SELECT id FROM users WHERE active = true) SELECT a.id, t.total FROM active a JOIN transactions t ON a.id = t.user_id"+ :dialect "postgres"+ ;; Note: sqlglot sees columns from CTE join and attributes some to transactions scope+ :expected [["transactions" "active"] ["transactions" "id"] ["transactions" "total"] ["transactions" "user_id"] ["users" "active"] ["users" "id"]]}++ {:name "multiple-ctes"+ :sql "WITH active AS (SELECT id FROM users WHERE active), recent AS (SELECT user_id, total FROM transactions WHERE created_at > '2024-01-01') SELECT * FROM active a JOIN recent r ON a.id = r.user_id"+ :dialect "postgres"+ :expected [["transactions" "created_at"] ["transactions" "total"] ["transactions" "user_id"] ["users" "active"] ["users" "id"]]}++ ;; Subqueries+ {:name "subquery-in-from"+ :sql "SELECT * FROM (SELECT id, name FROM users WHERE active = true) AS active_users"+ :dialect "postgres"+ :expected [["users" "active"] ["users" "id"] ["users" "name"]]}++ {:name "subquery-in-where"+ :sql "SELECT id, name FROM users WHERE id IN (SELECT user_id FROM transactions WHERE total > 100)"+ :dialect "postgres"+ ;; Note: sqlglot attributes some subquery columns to outer scope+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "id"] ["users" "name"] ["users" "total"] ["users" "user_id"]]}++ {:name "correlated-subquery"+ :sql "SELECT u.id, u.name FROM users u WHERE EXISTS (SELECT 1 FROM transactions t WHERE t.user_id = u.id)"+ :dialect "postgres"+ :expected [["transactions" "user_id"] ["users" "id"] ["users" "name"]]}++ ;; Aggregations+ ;; Note: The * in COUNT(*) is not a column reference, it's part of the aggregate function syntax+ {:name "count-aggregate"+ :sql "SELECT COUNT(*) FROM users"+ :dialect "postgres"+ :expected []}++ {:name "group-by-simple"+ :sql "SELECT country, COUNT(*) FROM users GROUP BY country"+ :dialect "postgres"+ :expected [["users" "country"]]}++ {:name "group-by-with-having"+ :sql "SELECT country, COUNT(*) as cnt FROM users GROUP BY country HAVING COUNT(*) > 10"+ :dialect "postgres"+ :expected [["users" "country"]]}++ {:name "multiple-aggregates"+ :sql "SELECT department, COUNT(*), AVG(salary), MAX(salary) FROM employees GROUP BY department"+ :dialect "postgres"+ :expected [["employees" "department"] ["employees" "salary"]]}++ ;; Window functions+ {:name "window-function-simple"+ :sql "SELECT name, salary, RANK() OVER (ORDER BY salary DESC) FROM employees"+ :dialect "postgres"+ :expected [["employees" "name"] ["employees" "salary"]]}++ {:name "window-with-partition"+ :sql "SELECT name, department, salary, AVG(salary) OVER (PARTITION BY department) FROM employees"+ :dialect "postgres"+ :expected [["employees" "department"] ["employees" "name"] ["employees" "salary"]]}++ ;; CASE expressions+ {:name "case-expression"+ :sql "SELECT name, CASE WHEN age < 18 THEN 'minor' ELSE 'adult' END AS age_group FROM users"+ :dialect "postgres"+ :expected [["users" "age"] ["users" "name"]]}++ {:name "case-with-multiple-conditions"+ :sql "SELECT name, CASE WHEN active = true AND age > 18 THEN 'active_adult' WHEN active = true THEN 'active_minor' ELSE 'inactive' END FROM users"+ :dialect "postgres"+ :expected [["users" "active"] ["users" "age"] ["users" "name"]]}++ ;; Set operations+ {:name "union-simple"+ :sql "SELECT id, name FROM users UNION SELECT id, name FROM archived_users"+ :dialect "postgres"+ :expected [["archived_users" "id"] ["archived_users" "name"] ["users" "id"] ["users" "name"]]}++ {:name "union-all"+ :sql "SELECT id FROM active_users UNION ALL SELECT id FROM inactive_users"+ :dialect "postgres"+ :expected [["active_users" "id"] ["inactive_users" "id"]]}++ ;; Schema-qualified tables+ {:name "schema-qualified-single"+ :sql "SELECT id, name FROM public.users"+ :dialect "postgres"+ :expected [["users" "id"] ["users" "name"]]}++ {:name "schema-qualified-multiple"+ :sql "SELECT u.id, o.total FROM public.users u JOIN sales.orders o ON u.id = o.user_id"+ :dialect "postgres"+ :expected [["orders" "total"] ["orders" "user_id"] ["users" "id"]]}++ ;; Complex queries+ {:name "complex-with-cte-join-subquery"+ :sql "WITH ranked AS (SELECT id, name, RANK() OVER (ORDER BY created_at DESC) as rank FROM users WHERE active = true) SELECT r.id, r.name, t.total FROM ranked r JOIN (SELECT user_id, SUM(amount) as total FROM transactions GROUP BY user_id) t ON r.id = t.user_id WHERE r.rank <= 10"+ :dialect "postgres"+ :expected [["transactions" "amount"] ["transactions" "user_id"] ["users" "active"] ["users" "created_at"] ["users" "id"] ["users" "name"]]}++ {:name "nested-subqueries"+ :sql "SELECT id, name FROM users WHERE id IN (SELECT user_id FROM transactions WHERE total > (SELECT AVG(total) FROM transactions))"+ :dialect "postgres"+ ;; Note: sqlglot attributes some subquery columns to outer scope+ :expected [["transactions" "total"] ["transactions" "user_id"] ["users" "id"] ["users" "name"] ["users" "total"] ["users" "user_id"]]}++ ;; Edge cases+ {:name "table-alias-same-as-table-name"+ :sql "SELECT users.id, users.name FROM users users"+ :dialect "postgres"+ :expected [["users" "id"] ["users" "name"]]}++ {:name "no-from-clause"+ :sql "SELECT 1 as one, 2 as two"+ :dialect "postgres"+ :expected []}++ {:name "select-from-values"+ :sql "SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS t(id, name)"+ :dialect "postgres"+ :expected []}++ ;; Different dialects+ {:name "mysql-backticks"+ :sql "SELECT `id`, `name` FROM `users` WHERE `active` = 1"+ :dialect "mysql"+ :expected [["users" "active"] ["users" "id"] ["users" "name"]]}++ {:name "bigquery-struct"+ :sql "SELECT id, name, address.city FROM users"+ :dialect "bigquery"+ ;; Note: Struct field access (address.city) is not parsed as a column reference+ :expected [["users" "id"] ["users" "name"]]}++ {:name "snowflake-uppercase"+ :sql "SELECT ID, NAME FROM USERS"+ :dialect "snowflake"+ :expected [["USERS" "ID"] ["USERS" "NAME"]]}++ ;; INSERT/UPDATE/DELETE (less common but valid)+ {:name "insert-with-select"+ :sql "INSERT INTO archive_users SELECT id, name, email FROM users WHERE active = false"+ :dialect "postgres"+ :expected [["users" "active"] ["users" "email"] ["users" "id"] ["users" "name"]]}++ {:name "update-with-join"+ :sql "UPDATE users SET last_purchase = t.max_date FROM (SELECT user_id, MAX(created_at) as max_date FROM transactions GROUP BY user_id) t WHERE users.id = t.user_id"+ :dialect "postgres"+ ;; Note: UPDATE statements may not be fully parsed by sqlglot for field extraction+ :expected []}++ {:name "delete-with-subquery"+ :sql "DELETE FROM users WHERE id IN (SELECT user_id FROM transactions WHERE total < 10)"+ :dialect "postgres"+ ;; Note: DELETE statements don't include the target table's columns in references+ :expected [["transactions" "total"] ["transactions" "user_id"]]}]}diff --git a/test/metabase/sql_tools/core_test.clj b/test/metabase/sql_tools/core_test.cljnew file mode 100644index 00000000000..11fa216c55d--- /dev/null+++ b/test/metabase/sql_tools/core_test.clj@@ -0,0 +1,91 @@+(ns metabase.sql-tools.core-test+ "Tests for sql-tools that run against both :macaw and :sqlglot backends.++ These tests verify that both parser implementations produce compatible results+ for common operations, ensuring we can switch backends without breaking the app."+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.driver :as driver]+ [metabase.lib.core :as lib]+ [metabase.sql-tools.core :as sql-tools]+ [metabase.sql-tools.test-util :as sql-tools.tu]+ [metabase.test :as mt]))++;;; ------------------------------------------------ validate-query ------------------------------------------------++(deftest ^:parallel validate-query-syntax-error-test+ (sql-tools.tu/test-parser-backends+ (mt/test-driver :h2+ (let [query (lib/native-query (mt/metadata-provider) "complete nonsense query")]+ (testing "Gibberish SQL returns syntax error"+ (is (= #{(lib/syntax-error)}+ (sql-tools/validate-query driver/*driver* query))))))))++(deftest ^:parallel validate-query-missing-column-test+ (sql-tools.tu/test-parser-backends+ (mt/test-driver :h2+ (let [query (lib/native-query (mt/metadata-provider) "select nonexistent from orders")]+ (testing "Reference to non-existent column returns missing-column error"+ (is (= #{(lib/missing-column-error "NONEXISTENT")}+ (sql-tools/validate-query driver/*driver* query))))))))++(deftest ^:parallel validate-query-valid-test+ (sql-tools.tu/test-parser-backends+ (mt/test-driver :h2+ (let [query (lib/native-query (mt/metadata-provider) "select id, total from orders")]+ (testing "Valid query returns empty error set"+ (is (= #{}+ (sql-tools/validate-query driver/*driver* query))))))))++;;; ---------------------------------------------- referenced-tables -----------------------------------------------++(deftest ^:parallel referenced-tables-basic-test+ (sql-tools.tu/test-parser-backends+ (mt/test-driver :h2+ (let [query (lib/native-query (mt/metadata-provider) "select id from orders")]+ (testing "Single table reference"+ (is (= #{{:table (mt/id :orders)}}+ (sql-tools/referenced-tables driver/*driver* query))))))))++(deftest ^:parallel referenced-tables-join-test+ (sql-tools.tu/test-parser-backends+ (mt/test-driver :h2+ (let [query (lib/native-query (mt/metadata-provider)+ "select o.id from orders o join products p on o.product_id = p.id")]+ (testing "Join references both tables"+ (is (= #{{:table (mt/id :orders)}+ {:table (mt/id :products)}}+ (sql-tools/referenced-tables driver/*driver* query))))))))++;;; ------------------------------------------------ replace-names -------------------------------------------------++(deftest ^:parallel replace-names-table-test+ (sql-tools.tu/test-parser-backends+ (testing "Basic table replacement"+ (is (= "SELECT * FROM new_orders"+ (sql-tools/replace-names :h2+ "SELECT * FROM orders"+ {:tables {{:table "orders"} "new_orders"}}))))))++(deftest ^:parallel replace-names-schema-test+ (sql-tools.tu/test-parser-backends+ (testing "Schema replacement"+ (is (= "SELECT * FROM new_schema.orders"+ (sql-tools/replace-names :h2+ "SELECT * FROM old_schema.orders"+ {:schemas {"old_schema" "new_schema"}}))))))++;;; -------------------------------------------- referenced-tables-raw ---------------------------------------------++(deftest ^:parallel referenced-tables-raw-test+ (sql-tools.tu/test-parser-backends+ (testing "Returns table names without resolving to IDs"+ ;; SQLGlot includes {:schema nil} while Macaw omits it - use =? for partial match+ (is (=? [{:table "orders"}]+ (sql-tools/referenced-tables-raw :h2 "SELECT * FROM orders"))))))++(deftest ^:parallel referenced-tables-raw-with-schema-test+ (sql-tools.tu/test-parser-backends+ (testing "Includes schema when present"+ (is (= [{:schema "public" :table "orders"}]+ (sql-tools/referenced-tables-raw :postgres "SELECT * FROM public.orders"))))))diff --git a/test/metabase/sql_tools/macaw/core_test.clj b/test/metabase/sql_tools/macaw/core_test.cljnew file mode 100644index 00000000000..134247ce372--- /dev/null+++ b/test/metabase/sql_tools/macaw/core_test.clj@@ -0,0 +1,39 @@+(ns metabase.sql-tools.macaw.core-test+ (:require+ [clojure.test :refer [deftest is testing]]+ ;; Load implementations for multimethod registration+ [metabase.sql-tools.init]+ [metabase.sql-tools.interface :as sql-tools]+ [metabase.sql-tools.macaw.core :as sql-tools.macaw]))++(deftest ^:parallel split-compound-table-spec-test+ (let [split sql-tools.macaw/split-compound-table-spec]+ (testing "simple table name is unchanged"+ (is (= {:table "orders" :schema nil}+ (split {:table "orders" :schema nil}))))+ (testing "two-part name splits into schema + table"+ (is (= {:schema "analytics" :table "events"}+ (split {:table "analytics.events" :schema nil}))))+ (testing "three-part name takes last two as schema + table"+ (is (= {:schema "analytics" :table "events"}+ (split {:table "myproject.analytics.events" :schema nil}))))+ (testing "already has schema — no splitting"+ (is (= {:table "events" :schema "analytics"}+ (split {:table "events" :schema "analytics"}))))+ (testing "compound name with existing schema is not split"+ (is (= {:table "analytics.events" :schema "other"}+ (split {:table "analytics.events" :schema "other"}))))))++(deftest ^:parallel referenced-tables-raw-bigquery-compound-names-test+ (testing "Macaw referenced-tables-raw splits BigQuery-style dotted table names"+ (is (= [{:schema "analytics" :table "events"}]+ (sql-tools/referenced-tables-raw-impl :macaw :bigquery-cloud-sdk+ "SELECT * FROM analytics.events")))+ (is (= #{{:schema "analytics" :table "events"}+ {:schema "analytics" :table "users"}}+ (set (sql-tools/referenced-tables-raw-impl :macaw :bigquery-cloud-sdk+ "SELECT * FROM analytics.events JOIN analytics.users ON events.user_id = users.id")))))+ (testing "simple table names are unaffected"+ (is (= [{:table "orders"}]+ (sql-tools/referenced-tables-raw-impl :macaw :postgres+ "SELECT * FROM orders")))))diff --git a/test/metabase/driver/sql/references_test.clj b/test/metabase/sql_tools/macaw/references_test.cljsimilarity index 99%rename from test/metabase/driver/sql/references_test.cljrename to test/metabase/sql_tools/macaw/references_test.cljindex 49e2e53fb27..44c382f2110 100644--- a/test/metabase/driver/sql/references_test.clj+++ b/test/metabase/sql_tools/macaw/references_test.clj@@ -1,19 +1,19 @@-(ns metabase.driver.sql.references-test+(ns metabase.sql-tools.macaw.references-test(:require[clojure.test :refer :all][macaw.core :as macaw]- [metabase.driver-api.core :as driver-api]- [metabase.driver.sql.references :as sql.references]))+ [metabase.lib.core :as lib]+ [metabase.sql-tools.macaw.references :as sql-tools.macaw]));; TODO (Chris 2026-01-22) -- Refactor to use driver.u/parsed-query instead of macaw/parsed-query#_{:clj-kondo/ignore [:discouraged-var]}(defn- ->references [query]- (->> query macaw/parsed-query macaw/->ast (sql.references/field-references :sql)))+ (->> query macaw/parsed-query macaw/->ast (sql-tools.macaw/field-references :sql)))(deftest ^:parallel garbage-test(is (= {:used-fields #{}:returned-fields []- :errors #{(driver-api/syntax-error)}}+ :errors #{(lib/syntax-error)}}(->references "nothing"))))(deftest ^:parallel basic-select-test@@ -274,7 +274,7 @@:source-columns [[{:type :all-columns,:table {:table "orders"}}]]}},:returned-fields []- :errors #{(driver-api/missing-table-alias-error "foo")}}+ :errors #{(lib/missing-table-alias-error "foo")}}(->references "select foo.* from (select a from orders)"))))(deftest ^:parallel bad-table-name-test@@ -288,7 +288,7 @@:alias nil,:type :single-column,:source-columns []}]- :errors #{(driver-api/missing-table-alias-error "bad")}}+ :errors #{(lib/missing-table-alias-error "bad")}}(->references "select bad.a from products"))))(deftest ^:parallel no-possible-sources-test@@ -302,7 +302,7 @@:alias nil:type :single-column:source-columns []}]- :errors #{(driver-api/missing-column-error "a")}}+ :errors #{(lib/missing-column-error "a")}}(->references "select a"))))(deftest ^:parallel select-constant-test@@ -922,6 +922,7 @@ SELECTcFROM c;"))))+;; composite field here(deftest ^:parallel recursive-cte-test(is (= {:used-fields#{{:column "id",diff --git a/test/metabase/sql_tools/macaw/replace_names_test.clj b/test/metabase/sql_tools/macaw/replace_names_test.cljnew file mode 100644index 00000000000..9be389be6a7--- /dev/null+++ b/test/metabase/sql_tools/macaw/replace_names_test.clj@@ -0,0 +1,112 @@+(ns metabase.sql-tools.macaw.replace-names-test+ "Tests for replace-names functionality via the sql-tools API.++ Note: Macaw's replace-names uses a specific format for replacement maps:+ - :tables - keys are {:schema s :table t} maps, values are new table names+ - :columns - keys are {:schema s :table t :column c} maps, values are new column names+ - :schemas - keys are schema name strings, values are new schema names"+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.sql-tools.core :as sql-tools]+ ;; Load implementations for multimethod registration+ [metabase.sql-tools.init]))++;;; Test cases adapted from Macaw's test suite to verify the sql-tools wrapper.+;;; Macaw keys use {:schema ... :table ... :column ...} maps, not plain strings.++(deftest ^:parallel basic-table-rename-test+ (testing "Basic table rename"+ (is (= "SELECT * FROM users"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT * FROM people"+ {:tables {{:table "people"} "users"}}+ {})))))++(deftest ^:parallel basic-column-rename-test+ (testing "Basic column rename"+ (is (= "SELECT user_id FROM orders"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT id FROM orders"+ {:columns {{:table "orders" :column "id"} "user_id"}}+ {})))))++(deftest ^:parallel schema-rename-test+ (testing "Schema rename"+ (is (= "SELECT * FROM private.orders"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT * FROM public.orders"+ {:schemas {"public" "private"}}+ {})))))++(deftest ^:parallel combined-renames-test+ (testing "Combined schema, table, and column renames"+ (is (= "SELECT amount FROM private.transactions"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT total FROM public.orders"+ {:schemas {"public" "private"}+ :tables {{:schema "public" :table "orders"} "transactions"}+ :columns {{:schema "public" :table "orders" :column "total"} "amount"}}+ {})))))++(deftest ^:parallel aliased-table-test+ (testing "Table with alias - alias is preserved, only table name changes"+ ;; AS is optional in table aliases per SQL standard+ (is (contains? #{"SELECT p.name FROM users p"+ "SELECT p.name FROM users AS p"}+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT p.name FROM people p"+ {:tables {{:table "people"} "users"}}+ {})))))++(deftest ^:parallel join-rename-test+ (testing "Renames in JOIN queries - alias preserved"+ ;; AS is optional in table aliases per SQL standard+ (is (contains? #{"SELECT p.id, o.total FROM users p JOIN orders o ON p.id = o.user_id"+ "SELECT p.id, o.total FROM users AS p JOIN orders AS o ON p.id = o.user_id"}+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT p.id, o.total FROM people p JOIN orders o ON p.id = o.user_id"+ {:tables {{:table "people"} "users"}}+ {})))))++(deftest ^:parallel multiple-table-renames-test+ (testing "Multiple table renames"+ (is (= "SELECT * FROM users, transactions"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT * FROM people, orders"+ {:tables {{:table "people"} "users"+ {:table "orders"} "transactions"}}+ {})))))++(deftest ^:parallel qualified-column-rename-test+ (testing "Qualified column in SELECT"+ (is (= "SELECT orders.amount FROM orders"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT orders.total FROM orders"+ {:columns {{:table "orders" :column "total"} "amount"}}+ {})))))++(deftest ^:parallel subquery-rename-test+ (testing "Renames in subqueries"+ ;; AS is optional in table aliases per SQL standard+ (is (contains? #{"SELECT * FROM (SELECT id FROM users) sub"+ "SELECT * FROM (SELECT id FROM users) AS sub"}+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT * FROM (SELECT id FROM people) sub"+ {:tables {{:table "people"} "users"}}+ {})))))++(deftest ^:parallel case-sensitive-match-test+ (testing "Case-sensitive matching - must match exact case"+ ;; Without case-insensitive option, matches are exact+ ;; PEOPLE won't match {:table "people"}+ (is (= "SELECT * FROM users"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT * FROM people"+ {:tables {{:table "people"} "users"}}+ {})))+ ;; Upper case needs upper case key+ (is (= "SELECT * FROM users"+ (sql-tools/replace-names-impl :macaw :postgres+ "SELECT * FROM PEOPLE"+ {:tables {{:table "PEOPLE"} "users"}}+ {})))))diff --git a/test/metabase/sql_tools/metrics_test.clj b/test/metabase/sql_tools/metrics_test.cljnew file mode 100644index 00000000000..e305904d36e--- /dev/null+++ b/test/metabase/sql_tools/metrics_test.clj@@ -0,0 +1,83 @@+(ns metabase.sql-tools.metrics-test+ (:require+ [clojure.test :refer :all]+ [metabase.analytics.prometheus :as prometheus]+ [metabase.analytics.prometheus-test :as prometheus-test]+ [metabase.sql-tools.metrics :as metrics]+ [metabase.test :as mt]))++(set! *warn-on-reflection* true)++(deftest with-operation-timing-success-test+ (testing "with-operation-timing increments counters on success"+ (mt/with-prometheus-system! [_ system]+ (metrics/with-operation-timing [:macaw "returned-columns"]+ :ok)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-total+ {:parser "macaw" :operation "returned-columns"})))+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-completed+ {:parser "macaw" :operation "returned-columns"})))+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-sql-tools/operations-failed+ {:parser "macaw" :operation "returned-columns"}))))))++(deftest with-operation-timing-failure-test+ (testing "with-operation-timing increments failure counter on exception"+ (mt/with-prometheus-system! [_ system]+ (is (thrown? Exception+ (metrics/with-operation-timing [:sqlglot "validate-query"]+ (throw (Exception. "boom")))))+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-total+ {:parser "sqlglot" :operation "validate-query"})))+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-sql-tools/operations-completed+ {:parser "sqlglot" :operation "validate-query"})))+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-failed+ {:parser "sqlglot" :operation "validate-query"}))))))++(deftest with-operation-timing-returns-value-test+ (testing "with-operation-timing returns the body's value"+ (mt/with-prometheus-system! [_ _system]+ (is (= :result+ (metrics/with-operation-timing [:macaw "simple-query?"]+ :result))))))++(deftest with-operation-timing-duration-test+ (testing "with-operation-timing records duration in histogram"+ (mt/with-prometheus-system! [_ system]+ (metrics/with-operation-timing [:macaw "replace-names"]+ (Thread/sleep 10))+ (let [histogram-val (mt/metric-value system :metabase-sql-tools/operation-duration-ms+ {:parser "macaw" :operation "replace-names"})]+ (is (pos? (:sum histogram-val)))))))++(deftest record-functions-test+ (testing "record-operation-start! increments operations-total"+ (mt/with-prometheus-system! [_ system]+ (metrics/record-operation-start! :macaw "field-references")+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-total+ {:parser "macaw" :operation "field-references"})))))+ (testing "record-operation-completion! increments completed and observes duration"+ (mt/with-prometheus-system! [_ system]+ (metrics/record-operation-completion! :sqlglot "referenced-tables" 42)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-completed+ {:parser "sqlglot" :operation "referenced-tables"})))+ (is (prometheus-test/approx= 42 (:sum (mt/metric-value system :metabase-sql-tools/operation-duration-ms+ {:parser "sqlglot" :operation "referenced-tables"}))))))+ (testing "record-operation-failure! increments failed and observes duration"+ (mt/with-prometheus-system! [_ system]+ (metrics/record-operation-failure! :macaw "add-into-clause" 100)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-tools/operations-failed+ {:parser "macaw" :operation "add-into-clause"})))+ (is (prometheus-test/approx= 100 (:sum (mt/metric-value system :metabase-sql-tools/operation-duration-ms+ {:parser "macaw" :operation "add-into-clause"})))))))++(deftest pool-metrics-registered-test+ (testing "Pool metrics are registered in prometheus"+ (mt/with-prometheus-system! [_ system]+ (prometheus/inc! :metabase-sql-parsing/context-timeouts)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-timeouts)))+ (prometheus/inc! :metabase-sql-parsing/context-acquisitions)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-acquisitions)))+ (prometheus/inc! :metabase-sql-parsing/context-creations)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-creations)))+ (prometheus/inc! :metabase-sql-parsing/context-disposals-expired)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-sql-parsing/context-disposals-expired))))))diff --git a/test/metabase/sql_tools/sqlglot/core_test.clj b/test/metabase/sql_tools/sqlglot/core_test.cljnew file mode 100644index 00000000000..4a39acd3abc--- /dev/null+++ b/test/metabase/sql_tools/sqlglot/core_test.clj@@ -0,0 +1,54 @@+(ns metabase.sql-tools.sqlglot.core-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.driver :as driver]+ [metabase.lib.core :as lib]+ [metabase.sql-tools.core :as sql-tools]+ [metabase.test :as mt]))++;;;; validate-query++(deftest validate-query-syntax-error-test+ (mt/test-driver+ :postgres+ (let [mp (mt/metadata-provider)+ query (lib/native-query mp "complete nonsense query")]+ (testing "Gibberish SQL returns syntax error"+ (is (= #{(lib/syntax-error)}+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))++(deftest validate-query-missing-table-alias-wildcard-test+ (mt/test-driver+ :postgres+ (let [mp (mt/metadata-provider)+ query (lib/native-query mp "select foo.* from (select id from orders)")]+ (testing "Wildcard with unknown table alias returns missing-table-alias error"+ (is (= #{(lib/missing-table-alias-error "foo")}+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))++(deftest validate-query-missing-table-alias-column-test+ (mt/test-driver+ :postgres+ (let [mp (mt/metadata-provider)+ query (lib/native-query mp "select bad.id from products")]+ (sql-tools/validate-query-impl :sqlglot :postgres query)+ (testing "Column with unknown table qualifier returns missing-table-alias error"+ (sql-tools/validate-query-impl :sqlglot :postgres query)))))++(deftest validate-query-missing-column-test+ (mt/test-driver+ :postgres+ (let [mp (mt/metadata-provider)+ query (lib/native-query mp "select nonexistent from orders")]+ (testing "Reference to non-existent column returns missing-column error"+ (is (= #{(lib/missing-column-error "nonexistent")}+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))++(deftest validate-query-valid-test+ (mt/test-driver+ :postgres+ (let [mp (mt/metadata-provider)+ query (lib/native-query mp "select id, total from orders")]+ (testing "Valid query returns empty error set"+ (is (= #{}+ (sql-tools/validate-query-impl :sqlglot driver/*driver* query)))))))diff --git a/test/metabase/sql_tools/sqlglot/references_test.clj b/test/metabase/sql_tools/sqlglot/references_test.cljnew file mode 100644index 00000000000..149f3820f21--- /dev/null+++ b/test/metabase/sql_tools/sqlglot/references_test.clj@@ -0,0 +1,2295 @@+;; SQLGlot version of references test+(ns metabase.sql-tools.sqlglot.references-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.sql-parsing.core :as sql-parsing]))++(defn- ->references [query]+ (sql-parsing/field-references "postgres" query))++(deftest ^:parallel garbage-test+ (is (= {:used-fields #{}+ :returned-fields []+ :errors #{(lib/syntax-error)}}+ (->references "nothing"))))++(deftest ^:parallel basic-select-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]+ :errors #{}}+ (->references "select * from (select a, b from products)"))))++(deftest ^:parallel basic-join-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields+ [{:type :all-columns, :table {:table "products"}}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]+ :errors #{}}+ (->references "select products.*, orders.id from products inner join orders on products.id = orders.product_id"))))++(deftest ^:parallel indeterminite-join-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "orders"}}+ {:type :all-columns, :table {:table "products"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "orders"}}+ {:type :all-columns, :table {:table "products"}}]]}]+ :errors #{}}+ (->references "select id from products inner join orders on products.id = orders.product_id"))))++(deftest ^:parallel table-wildcard-join-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:type :all-columns, :table {:table "orders"}}]+ :errors #{}}+ (->references "select orders.* from products inner join orders on products.id = orders.product_id"))))++(deftest ^:parallel wildcard-join-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:type :all-columns, :table {:table "orders"}}+ {:type :all-columns, :table {:table "products"}}]+ :errors #{}}+ (->references "select * from products inner join orders on products.id = orders.product_id"))))++(deftest ^:parallel basic-alias-test+ (is (= {:used-fields+ #{{:column "c",+ :alias "d",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}+ {:column "a",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}+ {:column "c",+ :alias "d",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}]+ :errors #{}}+ (->references "select p.a as b, p.c as d from products p"))))++(deftest ^:parallel basic-nested-query-test+ (is (= {:used-fields+ #{{:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}]+ :errors #{}}+ (->references "select a, b from (select a, b from products)"))))++(deftest ^:parallel renamed-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "a",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}]+ :errors #{}}+ (->references "select b as c from (select a as b from products)"))))++(deftest ^:parallel broken-nested-query-test+ (is (= {:used-fields+ #{{:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]]}]+ :errors #{}}+ (->references "select a from (select b from products)"))))++(deftest ^:parallel different-case-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}]+ :errors #{}}+ (->references "select A from (select a from orders)"))))++(deftest ^:parallel wildcard-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}]+ :errors #{}}+ (->references "select * from (select a from orders)"))))++(deftest ^:parallel table-wildcard-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}]+ :errors #{}}+ (->references "select o.* from (select a from orders) o"))))++(deftest ^:parallel bad-table-wildcard-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields []+ :errors #{(lib/missing-table-alias-error "foo")}}+ (->references "select foo.* from (select a from orders)"))))++(deftest ^:parallel bad-table-name-test+ (is (= #{(lib/missing-table-alias-error "bad")}+ (:errors (->references "select bad.a from products")))))++(deftest ^:parallel no-possible-sources-test+ (is (= {:used-fields+ #{{:column "a"+ :alias nil+ :type :single-column+ :source-columns []}}+ :returned-fields+ [{:column "a"+ :alias nil+ :type :single-column+ :source-columns []}]+ :errors #{(lib/missing-column-error "a")}}+ (->references "select a"))))++(deftest ^:parallel select-constant-test+ (is (= {:used-fields #{},+ :returned-fields+ [{:alias nil, :type :custom-field, :used-fields #{}}],+ :errors #{}}+ (->references "select 1"))))++(deftest ^:parallel basic-where-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]+ :errors #{}}+ (->references "select * from products where category = 'hello'"))))++(deftest ^:parallel basic-aggregation-test+ (is (=? {:used-fields #{},+ :returned-fields [{:alias nil, :type :custom-field, :used-fields #{}}]+ :errors #{}}+ (->references "select count(*) from products"))))++(deftest ^:parallel named-aggregation-test+ (is (=? {:used-fields #{},+ :returned-fields [{:alias "count", :type :custom-field, :used-fields #{}}]+ :errors #{}}+ (->references "select count(*) as count from products"))))++(deftest ^:parallel extra-names-test+ (is (= {:used-fields+ #{{:column "col",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:database "db",+ :schema "schema",+ :table "table"}}]]}},+ :returned-fields+ [{:column "col",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:database "db",+ :schema "schema",+ :table "table"}}]]}]+ :errors #{}}+ (->references "select db.schema.table.col from db.schema.table"))))++(deftest ^:parallel basic-grouping-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:alias "sum",+ :type :custom-field,+ :used-fields+ #{{:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:alias "sum",+ :type :custom-field,+ :used-fields+ #{{:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]+ :errors #{}}+ (->references "select sum(total) as sum from orders group by category"))))++(deftest ^:parallel grouping-order-by-test+ (is (= {:used-fields+ #{{:alias "cards_created",+ :type :custom-field,+ :used-fields+ #{{:column "creator_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}+ {:column "creator_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}+ {:column "creator_id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:alias "cards_created",+ :type :custom-field,+ :used-fields+ #{{:column "creator_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}]+ [{:type :all-columns, :table {:table "report_card"}}]]}},+ :returned-fields+ [{:column "creator_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}+ {:alias "cards_created",+ :type :custom-field,+ :used-fields+ #{{:column "creator_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}]+ :errors #{}}+ (->references "SELECT creator_id, COUNT(creator_id) AS cards_created FROM report_card GROUP BY creator_id ORDER BY cards_created DESC"))))++(deftest ^:parallel basic-arg-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]+ :errors #{}}+ (->references "select * from products where category = ?"))))++(deftest ^:parallel basic-case-test+ (is (=? {:used-fields+ #{{:column "tax",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "subtotal",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "tax",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "subtotal",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}}}]+ :errors #{}}+ (->references "select case when total < 0 then -subtotal else tax end from orders"))))++(deftest ^:parallel switch-case-test+ (is (=? {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields+ [{:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}}}]+ :errors #{}}+ (->references "select case category when 'Gizmo' then 'is gizmo' else 'is not gizmo' end from products"))))++(deftest ^:parallel basic-select-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]+ :errors #{}}+ (->references "select (select category from products where products.id = orders.product_id), * from orders"))))++(deftest ^:parallel named-select-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias "category2",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]+ :errors #{}}+ (->references "select (select category from products where products.id = orders.product_id) as category2, * from orders"))))++(deftest ^:parallel nested-select-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]+ :errors #{}}+ (->references "select (select (select category from products where products.id = orders.product_id)), * from orders"))))++(deftest ^:parallel select-subquery-with-normal-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]+ :errors #{}}+ (->references "select (select * from (select category from products where products.id = orders.product_id) sub), * from orders"))))++(deftest ^:parallel nested-select-subquery-with-reference-to-middle-select-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "products"}}]+ [{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "products"}}]+ [{:type :all-columns, :table {:table "orders"}}]]}]+ :errors #{}}+ (->references "select (select (select category) from products where products.id = orders.product_id) from orders"))))++(deftest ^:parallel nested-select-subquery-with-direct-match-in-outer-query+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}+ {:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "t1"}}]+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "t1"}}]+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}]+ :errors #{}}+ (->references "select (select a from t1) from (select a from t2)"))))++(deftest ^:parallel basic-exists-test+ (is (= {:used-fields+ #{{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}+ {:column "user_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "o", :table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}+ {:column "email",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}},+ :returned-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}+ {:column "email",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}]+ :errors #{}}+ (->references "SELECT u.name, u.email+FROM users u+WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))++(deftest ^:parallel basic-not-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]+ :errors #{}}+ (->references "select * from products where not (category = 'Gizmo')"))))++(deftest ^:parallel basic-is-null-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]+ :errors #{}}+ (->references "select * from products where category is null"))))++(deftest ^:parallel negated-is-null-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]+ :errors #{}}+ (->references "select * from products where category is not null"))))++(deftest ^:parallel basic-between-test+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "left",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "right",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields [{:type :all-columns, :table {:table "orders"}}]+ :errors #{}}+ (->references "select * from orders where created_at between left and right"))))++(deftest ^:parallel basic-select-table-alias-test+ (is (= {:used-fields #{},+ :returned-fields+ [{:type :custom-field,+ :alias "o",+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]+ :errors #{}}+ (->references "select o from (select * from orders) as o"))))++(deftest ^:parallel select-table-alias-with-alias-test+ (is (= {:used-fields #{},+ :returned-fields+ [{:type :custom-field,+ :alias "o2",+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]+ :errors #{}}+ (->references "select o as o2 from (select * from orders) as o"))))++(deftest ^:parallel table-function-test+ (is (= {:used-fields #{}, :returned-fields [{:type :unknown-columns}] :errors #{}}+ (->references "select i.* from generate_series(1, 500) as i"))))++(deftest ^:parallel basic-cte-test+ (is (= {:used-fields+ #{{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "active",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}},+ :returned-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}]+ :errors #{}}+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)+SELECT * FROM active_users"))))++(deftest ^:parallel unused-cte-test+ (is (= {:used-fields+ #{{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "active",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]+ :errors #{}}+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)+SELECT * FROM products"))))++(deftest ^:parallel dependent-cte-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]+ :errors #{}}+ (->references "+WITH orders_a AS (select * from orders),+orders_b AS (select * from orders_a)+SELECT id from orders_b"))))++(deftest ^:parallel shadowed-cte-test+ (is (= {:used-fields+ #{{:column "y",+ :alias "x",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "y",+ :alias "a",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "z",+ :alias "y",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "x",+ :alias "z",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "z",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "x",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}},+ :returned-fields+ [{:column "y",+ :alias "a",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "z",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "x",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}]+ :errors #{}}+ (->references "WITH c AS (+ WITH b AS (+ SELECT+ x as z,+ y as x,+ z as y+ FROM a+ )+ SELECT+ x as a,+ y as b,+ z as c+ FROM b+)+SELECT+ a,+ b,+ c+FROM c;"))))++;; composite field here+(deftest ^:parallel recursive-cte-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "h", :table "emp_hierarchy"}}]]}+ {:alias "name",+ :type :composite-field,+ :member-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}]}+ {:column "manager_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}+ {:column "level",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "h",+ :table "emp_hierarchy"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}+ {:alias "level",+ :type :composite-field,+ :member-fields+ [{:alias "level", :type :custom-field, :used-fields #{}}+ {:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "level",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "h",+ :table "emp_hierarchy"}}]]}}}]}+ {:column "manager_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "manager_id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:alias "level", :type :custom-field, :used-fields #{}}]+ [{:type :all-columns, :table {:table "employees"}}]]}},+ :returned-fields+ [{:alias "name",+ :type :composite-field,+ :member-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}]}+ {:alias "level",+ :type :composite-field,+ :member-fields+ [{:alias "level", :type :custom-field, :used-fields #{}}+ {:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "level",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "h",+ :table "emp_hierarchy"}}]]}}}]}]+ :errors #{}}+ (->references "WITH RECURSIVE emp_hierarchy AS (+ SELECT id, name, manager_id, 0 AS level+ FROM employees+ WHERE manager_id IS NULL+ UNION ALL+ SELECT e.id, e.name, e.manager_id, h.level + 1+ FROM employees e+ JOIN emp_hierarchy h ON e.manager_id = h.id+)+SELECT name, level FROM emp_hierarchy"))))++(deftest ^:parallel basic-union-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}},+ :returned-fields+ [{:alias "id",+ :type :composite-field,+ :member-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}]}+ {:alias "name",+ :type :composite-field,+ :member-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}]}]+ :errors #{}}+ (->references "SELECT id, name FROM users+UNION+SELECT id, name FROM archived_users"))))++(deftest ^:parallel row-number-test+ (is (= {:used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}},+ :returned-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}+ {:alias "rank",+ :type :custom-field,+ :used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}}}]+ :errors #{}}+ (->references "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) AS rank+FROM employees"))))++(deftest ^:parallel partition-by-test+ (is (= {:used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "department",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}},+ :returned-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:alias "dept_rank",+ :type :custom-field,+ :used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "department",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}}}]+ :errors #{}}+ (->references "SELECT name,+ RANK() OVER (PARTITION BY department ORDER BY salary DESC) AS dept_rank+FROM employees"))))++(deftest ^:parallel week-test+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}},+ :returned-fields+ [{:alias "created_at",+ :type :custom-field,+ :used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "count", :type :custom-field, :used-fields #{}}]+ :errors #{}}+ (->references "SELECT+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ ) AS \"created_at\",+ COUNT(*) AS \"count\"+FROM+ \"public\".\"orders\"+GROUP BY+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ORDER BY+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ ) ASC"))))++(deftest ^:parallel week-of-year-test+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}},+ :returned-fields+ [{:alias "created_at",+ :type :custom-field,+ :used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "count", :type :custom-field, :used-fields #{}}]+ :errors #{}}+ (->references "SELECT+ CEIL(+ (+ CAST(+ extract(+ doy+ from+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ ) AS integer+ ) / 7.0+ )+ ) AS \"created_at\",+ COUNT(*) AS \"count\"+FROM+ \"public\".\"orders\"+GROUP BY+ CEIL(+ (+ CAST(+ extract(+ doy+ from+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ ) AS integer+ ) / 7.0+ )+ )+ORDER BY+ CEIL(+ (+ CAST(+ extract(+ doy+ from+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ ) AS integer+ ) / 7.0+ )+ ) ASC"))))++;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}+(deftest ^:parallel complicated-test-1+ (is (= {:used-fields+ #{{:column "title",+ :alias "title",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "discount",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "rating",+ :alias "rating",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:alias "upper_category",+ :type :custom-field,+ :used-fields+ #{{:column "category",+ :alias "category",+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}}}+ {:column "ean",+ :alias "ean",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "created_at",+ :alias "created_at",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "product_id",+ :alias "product_id",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "category",+ :alias "Products__category",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "category",+ :alias "category",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "created_at",+ :alias "created_at",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "discount",+ :alias "discount",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "vendor",+ :alias "vendor",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "total",+ :alias "total",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "id",+ :alias "Products__id",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "id",+ :alias "id",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "price",+ :alias "price",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}},+ :returned-fields+ [{:alias "created_at",+ :type :custom-field,+ :used-fields+ #{{:column "created_at",+ :alias "created_at",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "upper_category",+ :type :custom-field,+ :used-fields+ #{{:column "category",+ :alias "category",+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}}}+ {:alias "sum",+ :type :custom-field,+ :used-fields+ #{{:column "total",+ :alias "total",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "sum_2",+ :type :custom-field,+ :used-fields+ #{{:column "discount",+ :alias "discount",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "count", :type :custom-field, :used-fields #{}}]+ :errors #{}}+ (->references "SELECT+ DATE_TRUNC('month', \"source\".\"created_at\") AS \"created_at\",+ \"source\".\"upper_category\" AS \"upper_category\",+ SUM(\"source\".\"total\") AS \"sum\",+ SUM(\"source\".\"discount\") AS \"sum_2\",+ COUNT(*) AS \"count\"+FROM+ (+ SELECT+ \"public\".\"orders\".\"product_id\" AS \"product_id\",+ \"public\".\"orders\".\"total\" AS \"total\",+ \"public\".\"orders\".\"discount\" AS \"discount\",+ \"public\".\"orders\".\"created_at\" AS \"created_at\",+ UPPER(\"Products\".\"category\") AS \"upper_category\",+ \"Products\".\"id\" AS \"Products__id\",+ \"Products\".\"category\" AS \"Products__category\"+ FROM+ \"public\".\"orders\"++LEFT JOIN (+ SELECT+ \"public\".\"products\".\"id\" AS \"id\",+ \"public\".\"products\".\"ean\" AS \"ean\",+ \"public\".\"products\".\"title\" AS \"title\",+ \"public\".\"products\".\"category\" AS \"category\",+ \"public\".\"products\".\"vendor\" AS \"vendor\",+ \"public\".\"products\".\"price\" AS \"price\",+ \"public\".\"products\".\"rating\" AS \"rating\",+ \"public\".\"products\".\"created_at\" AS \"created_at\"+ FROM+ \"public\".\"products\"+ ) AS \"Products\" ON \"public\".\"orders\".\"product_id\" = \"Products\".\"id\"++WHERE+ \"public\".\"orders\".\"discount\" > 0+ ) AS \"source\"+GROUP BY+ DATE_TRUNC('month', \"source\".\"created_at\"),+ \"source\".\"upper_category\"+ORDER BY+ DATE_TRUNC('month', \"source\".\"created_at\") ASC,+ \"source\".\"upper_category\" ASC"))))++;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}+(deftest ^:parallel complicated-test-2+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "is_converted",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "annual_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "base_fee",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "account_id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_alias",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_paid_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_active_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_active_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_paid_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_trialing_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "is_active_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "is_active_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_active_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_trialing_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:column "customer_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_trialing",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:column "is_active",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_paid_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_paid_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_paid_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}+ {:column "valid_to",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "valid_from",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}},+ :returned-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "customer_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "is_converted",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "is_active",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_alias",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "base_fee",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "annual_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}+ {:alias "is_trialing",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_trialing_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_active_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_trialing_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_active_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_active_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_active_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}+ {:alias "is_active_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "month",+ :type :custom-field,+ :used-fields #{}}}}]+ :errors #{}}+ (->references "with months as (+ select+ generate_series(date_trunc('month', '2024-01-01'::date),+ date_trunc('month', current_date) - interval '1 month',+ '1 month')::date as month++)+, monthly_account_subscription_status as (+ select+ a.id,+ m.month,++ max(case when ah.subscription_status is not null and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing,++ -- cloud+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_cloud,+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' then 1 else 0 end) as is_active_accounts_cloud,+ max(case when a.lifetime_value > 0 and a.deployment = 'cloud' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_cloud,++ -- self-hosted+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_self_hosted,+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' then 1 else 0 end) as is_active_accounts_self_hosted,+ max(case when a.lifetime_value > 0 and a.deployment = 'self-hosted' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_self_hosted,++ -- enterprise+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'enterprise' then 1 else 0 end) as is_active_accounts_enterprise,+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'enterprise' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_enterprise,++ -- starter+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'starter' then 1 else 0 end) as is_active_accounts_starter,+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'starter' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_starter,++ -- pro+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'pro' then 1 else 0 end) as is_active_accounts_pro,+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'pro' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_pro+ from dbt_models.account a+ left join dbt_models.account_history ah+ on a.id = ah.account_id+ left join months m+ on m.month between date_trunc('month', ah.valid_from) and date_trunc('month', ah.valid_to)+ group by 1,2+)+select+ a.id,+ a.customer_name,+ a.created_at,+ a.is_converted,+ a.lifetime_value,+ a.is_active,+ a.subscription_status,+ a.deployment,+ a.plan_name,+ a.plan_alias,+ a.base_fee,+ a.annual_value,++ -- monthly account status+ mas.month,+ mas.is_trialing,+ mas.is_trialing_accounts_cloud,+ mas.is_active_accounts_cloud,+ mas.is_paid_accounts_cloud,+ mas.is_trialing_accounts_self_hosted,+ mas.is_active_accounts_self_hosted,+ mas.is_paid_accounts_self_hosted,+ mas.is_active_accounts_enterprise,+ mas.is_paid_accounts_enterprise,+ mas.is_active_accounts_starter,+ mas.is_paid_accounts_starter,+ mas.is_active_accounts_pro,+ mas.is_paid_accounts_pro++from dbt_models.account a+left join monthly_account_subscription_status mas+ on a.id = mas.id+;"))))diff --git a/test/metabase/sql_tools/sqlglot/replace_names_test.clj b/test/metabase/sql_tools/sqlglot/replace_names_test.cljnew file mode 100644index 00000000000..4e350f991da--- /dev/null+++ b/test/metabase/sql_tools/sqlglot/replace_names_test.clj@@ -0,0 +1,121 @@+(ns metabase.sql-tools.sqlglot.replace-names-test+ "Tests for replace-names functionality using SQLGlot backend.++ These tests mirror the Macaw tests to ensure feature parity between backends.+ Tests call sql-parsing/replace-names directly since it doesn't need Metabase metadata."+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.sql-parsing.core :as sql-parsing]))++(defn- replace-names+ "Helper that mirrors sql-tools/replace-names API but calls SQLGlot directly."+ [driver sql-string replacements]+ ;; Convert map keys to list-of-pairs for JSON serialization+ (let [dialect (case driver+ :postgres "postgres"+ :mysql "mysql"+ :sqlserver "tsql"+ nil)+ replacements' (-> replacements+ (update :tables #(when % (vec %)))+ (update :columns #(when % (vec %))))]+ (sql-parsing/replace-names dialect sql-string replacements')))++(deftest ^:parallel basic-table-rename-test+ (testing "Basic table rename"+ (is (= "SELECT * FROM users"+ (replace-names :postgres+ "SELECT * FROM people"+ {:tables {{:table "people"} "users"}})))))++(deftest ^:parallel basic-column-rename-test+ (testing "Basic column rename"+ (is (= "SELECT user_id FROM orders"+ (replace-names :postgres+ "SELECT id FROM orders"+ {:columns {{:table "orders" :column "id"} "user_id"}})))))++(deftest ^:parallel schema-rename-test+ (testing "Schema rename"+ (is (= "SELECT * FROM private.orders"+ (replace-names :postgres+ "SELECT * FROM public.orders"+ {:schemas {"public" "private"}})))))++(deftest ^:parallel combined-renames-test+ (testing "Combined schema, table, and column renames"+ (is (= "SELECT amount FROM private.transactions"+ (replace-names :postgres+ "SELECT total FROM public.orders"+ {:schemas {"public" "private"}+ :tables {{:schema "public" :table "orders"} "transactions"}+ :columns {{:schema "public" :table "orders" :column "total"} "amount"}})))))++(deftest ^:parallel aliased-table-test+ (testing "Table with alias - alias is preserved, only table name changes"+ (is (= "SELECT p.name FROM users AS p"+ (replace-names :postgres+ "SELECT p.name FROM people p"+ {:tables {{:table "people"} "users"}})))))++(deftest ^:parallel join-rename-test+ (testing "Renames in JOIN queries - alias preserved"+ (is (= "SELECT p.id, o.total FROM users AS p JOIN orders AS o ON p.id = o.user_id"+ (replace-names :postgres+ "SELECT p.id, o.total FROM people p JOIN orders o ON p.id = o.user_id"+ {:tables {{:table "people"} "users"}})))))++(deftest ^:parallel multiple-table-renames-test+ (testing "Multiple table renames"+ (is (= "SELECT * FROM users, transactions"+ (replace-names :postgres+ "SELECT * FROM people, orders"+ {:tables {{:table "people"} "users"+ {:table "orders"} "transactions"}})))))++(deftest ^:parallel qualified-column-rename-test+ (testing "Qualified column in SELECT"+ (is (= "SELECT orders.amount FROM orders"+ (replace-names :postgres+ "SELECT orders.total FROM orders"+ {:columns {{:table "orders" :column "total"} "amount"}})))))++(deftest ^:parallel subquery-rename-test+ (testing "Renames in subqueries"+ (is (= "SELECT * FROM (SELECT id FROM users) AS sub"+ (replace-names :postgres+ "SELECT * FROM (SELECT id FROM people) sub"+ {:tables {{:table "people"} "users"}})))))++(deftest ^:parallel case-sensitive-match-test+ (testing "Case-sensitive matching - must match exact case"+ ;; Lower case matches lower case+ (is (= "SELECT * FROM users"+ (replace-names :postgres+ "SELECT * FROM people"+ {:tables {{:table "people"} "users"}})))+ ;; Upper case needs upper case key+ (is (= "SELECT * FROM users"+ (replace-names :postgres+ "SELECT * FROM PEOPLE"+ {:tables {{:table "PEOPLE"} "users"}})))))++(deftest ^:parallel table-rename-with-schema-map-value-test+ (testing "Table rename using map value with schema and table (workspace isolation pattern)"+ ;; This is the pattern used by workspaces to isolate tables into a new schema+ (is (= "SELECT * FROM ws_isolated_123.public__orders"+ (replace-names :postgres+ "SELECT * FROM orders"+ {:tables {{:table "orders"} {:schema "ws_isolated_123" :table "public__orders"}}})))+ ;; Qualified source table+ (is (= "SELECT * FROM ws_isolated_123.public__orders"+ (replace-names :postgres+ "SELECT * FROM public.orders"+ {:tables {{:schema "public" :table "orders"} {:schema "ws_isolated_123" :table "public__orders"}}})))))++(deftest ^:parallel table-rename-add-schema-only-test+ (testing "Table rename that only adds schema (qualify unqualified reference)"+ (is (= "SELECT * FROM public.orders"+ (replace-names :postgres+ "SELECT * FROM orders"+ {:tables {{:table "orders"} {:schema "public" :table "orders"}}})))))diff --git a/test/metabase/sql_tools/sqlglot/returned_columns_test.clj b/test/metabase/sql_tools/sqlglot/returned_columns_test.cljnew file mode 100644index 00000000000..a546312f621--- /dev/null+++ b/test/metabase/sql_tools/sqlglot/returned_columns_test.clj@@ -0,0 +1,480 @@+(ns metabase.sql-tools.sqlglot.returned-columns-test+ "Tests for SQLGlot returned-columns implementation.+ Ported from metabase.sql-tools.macaw.references-test.++ Note: These tests verify that returned-columns produces correct output,+ not that it matches Macaw's exact format. The key assertions are:+ - Correct column names+ - Correct column count+ - Type information when schema allows"+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.sql-tools.core :as sql-tools]+ [metabase.test :as mt]+ [metabase.util :as u]))++;;; Test Helpers++(defn- returned-column-names+ "Extract just the column names from returned-columns result.+ Normalizes to lowercase for cross-database comparison."+ [columns]+ (mapv (comp u/lower-case-en :name) columns))++(defn- returned-column-aliases+ "Extract just the column aliases from returned-columns result.+ Normalizes to lowercase for cross-database comparison."+ [columns]+ (mapv (comp u/lower-case-en :lib/desired-column-alias) columns))++(defn- make-query+ "Create a native query with the test metadata provider."+ [sql]+ (lib/native-query (mt/metadata-provider) sql))++(defn- returned-columns+ "Get returned columns for a SQL query using the test metadata provider."+ [sql]+ (let [driver (:engine (lib.metadata/database (mt/metadata-provider)))]+ (sql-tools/returned-columns-impl :sqlglot driver (make-query sql))))++;;; Basic SELECT Tests++(deftest ^:parallel basic-select-columns-test+ (testing "Simple SELECT returns expected columns"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT ID, NAME FROM PEOPLE")]+ (is (= 2 (count result)))+ ;; SQLGlot normalizes to lowercase+ (is (= ["id" "name"] (returned-column-names result)))+ (is (every? #(= :metadata/column (:lib/type %)) result))))))++(deftest ^:parallel select-star-test+ (testing "SELECT * returns all columns from table"+ (mt/test-driver :h2+ ;; Note: SQLGlot with lineage expands * to actual column names+ (let [result (returned-columns "SELECT * FROM PEOPLE")]+ (is (pos? (count result)))+ (is (every? #(= :metadata/column (:lib/type %)) result))))))++(deftest ^:parallel select-with-alias-test+ (testing "SELECT with alias preserves alias in output"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT ID AS user_id FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "id" (first (returned-column-names result))))+ (is (= "user_id" (first (returned-column-aliases result))))))))++;;; JOIN Tests++(deftest ^:parallel basic-join-test+ (testing "JOIN query returns columns from both tables"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT PEOPLE.NAME, ORDERS.TOTAL+ FROM PEOPLE+ INNER JOIN ORDERS ON PEOPLE.ID = ORDERS.USER_ID")]+ (is (= 2 (count result)))+ (is (= #{"name" "total"} (set (returned-column-names result))))))))++(deftest ^:parallel join-with-table-wildcard-test+ (testing "JOIN with table.* returns columns from that table"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ORDERS.*+ FROM PEOPLE+ INNER JOIN ORDERS ON PEOPLE.ID = ORDERS.USER_ID")]+ (is (pos? (count result)))+ ;; All returned columns should be from ORDERS+ (is (every? some? result))))))++;;; Subquery Tests++(deftest ^:parallel subquery-test+ (testing "Subquery columns are traced correctly"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT a, b FROM (SELECT ID AS a, NAME AS b FROM PEOPLE)")]+ (is (= 2 (count result)))+ (is (= ["a" "b"] (returned-column-aliases result)))))))++(deftest ^:parallel nested-subquery-test+ (testing "Nested subqueries trace through multiple levels"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT x FROM (SELECT a AS x FROM (SELECT ID AS a FROM PEOPLE))")]+ (is (= 1 (count result)))+ (is (= "x" (first (returned-column-aliases result))))))))++;;; CTE Tests++(deftest ^:parallel basic-cte-test+ (testing "CTE columns are traced correctly"+ (mt/test-driver :h2+ (let [result (returned-columns+ "WITH active_users AS (SELECT ID, NAME FROM PEOPLE WHERE ID > 0)+ SELECT * FROM active_users")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++;;; Aggregate Tests++(deftest ^:parallel aggregate-test+ (testing "Aggregate functions return custom columns (pure? = false)"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT COUNT(*) AS cnt FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "cnt" (first (returned-column-names result))))+ ;; Aggregates should have :type/* since they're computed+ (is (= :type/* (:base-type (first result))))))))++(deftest ^:parallel group-by-aggregate-test+ (testing "GROUP BY with aggregates"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT STATE, COUNT(*) AS total+ FROM PEOPLE+ GROUP BY STATE")]+ (is (= 2 (count result)))+ (is (= #{"state" "total"} (set (returned-column-names result))))))))++;;; Error Handling Tests++(deftest ^:parallel invalid-sql-test+ (testing "Invalid SQL returns empty result (no crash)"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT * FORM PEOPLE")]+ ;; Should return empty, not throw+ (is (vector? result))))))++(deftest ^:parallel missing-table-test+ (testing "Reference to non-existent table returns empty or partial result"+ (mt/test-driver :h2+ ;; This might fail during qualification or return empty+ (let [result (returned-columns "SELECT * FROM nonexistent_table")]+ (is (vector? result))))))++;;; Expression Tests++(deftest ^:parallel expression-column-test+ (testing "Expressions in SELECT create computed columns"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT ID + 1 AS incremented FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "incremented" (first (returned-column-names result))))+ ;; Computed columns have :type/* base type+ (is (= :type/* (:base-type (first result))))))))++(deftest ^:parallel case-expression-test+ (testing "CASE expressions create computed columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT CASE WHEN ID > 5 THEN 'high' ELSE 'low' END AS category+ FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "category" (first (returned-column-names result))))))))++;;; UNION Tests++(deftest ^:parallel union-test+ (testing "UNION combines columns from both branches"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, NAME FROM PEOPLE+ UNION+ SELECT ID, NAME FROM PEOPLE WHERE ID > 5")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++;;; Window Function Tests++(deftest ^:parallel window-function-test+ (testing "Window functions create computed columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT NAME, ROW_NUMBER() OVER (ORDER BY ID) AS rn+ FROM PEOPLE")]+ (is (= 2 (count result)))+ (is (= #{"name" "rn"} (set (returned-column-names result))))))))++;;; More Subquery Tests (ported from references_test.clj)++(deftest ^:parallel renamed-nested-query-test+ (testing "Alias chain through subquery: a -> b -> c"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT b AS c FROM (SELECT ID AS b FROM PEOPLE)")]+ (is (= 1 (count result)))+ (is (= "c" (first (returned-column-aliases result))))))))++(deftest ^:parallel different-case-nested-query-test+ (testing "Case-insensitive column matching through subquery"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT A FROM (SELECT id AS a FROM PEOPLE)")]+ (is (= 1 (count result)))+ ;; Should match despite case difference+ (is (= "a" (first (returned-column-aliases result))))))))++(deftest ^:parallel wildcard-nested-query-test+ (testing "SELECT * from subquery expands to subquery columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT * FROM (SELECT ID, NAME FROM PEOPLE)")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++(deftest ^:parallel table-wildcard-nested-query-test+ (testing "SELECT alias.* from subquery with alias"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT p.* FROM (SELECT ID, NAME FROM PEOPLE) p")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++;;; WHERE Clause Tests++(deftest ^:parallel basic-where-test+ (testing "WHERE clause doesn't affect returned columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT * FROM PEOPLE WHERE STATE = 'CA'")]+ ;; WHERE doesn't change output columns+ (is (pos? (count result)))+ (is (every? #(= :metadata/column (:lib/type %)) result))))))++(deftest ^:parallel where-is-null-test+ (testing "WHERE with IS NULL"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, NAME FROM PEOPLE WHERE NAME IS NOT NULL")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++(deftest ^:parallel where-between-test+ (testing "WHERE with BETWEEN"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, NAME FROM PEOPLE WHERE ID BETWEEN 1 AND 100")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++;;; Constant/Literal Tests++(deftest ^:parallel select-constant-test+ (testing "SELECT constant returns computed column"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT 1 AS one")]+ (is (= 1 (count result)))+ (is (= "one" (first (returned-column-aliases result))))+ ;; Constants are computed (pure? = false)+ (is (= :type/* (:base-type (first result))))))))++(deftest ^:parallel select-string-constant-test+ (testing "SELECT string constant"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT 'hello' AS greeting")]+ (is (= 1 (count result)))+ (is (= "greeting" (first (returned-column-names result))))))))++;;; More CTE Tests++(deftest ^:parallel unused-cte-test+ (testing "Unused CTE doesn't affect output"+ (mt/test-driver :h2+ (let [result (returned-columns+ "WITH unused AS (SELECT ID FROM PEOPLE)+ SELECT NAME FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "name" (u/lower-case-en (:name (first result)))))))))++(deftest ^:parallel dependent-cte-test+ (testing "CTE that references another CTE"+ (mt/test-driver :h2+ (let [result (returned-columns+ "WITH cte1 AS (SELECT ID, NAME FROM PEOPLE),+ cte2 AS (SELECT ID FROM cte1)+ SELECT * FROM cte2")]+ (is (= 1 (count result)))+ (is (= "id" (u/lower-case-en (:name (first result)))))))))++(deftest ^:parallel multiple-cte-test+ (testing "Multiple CTEs with different columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "WITH names AS (SELECT NAME FROM PEOPLE),+ ids AS (SELECT ID FROM PEOPLE)+ SELECT * FROM names")]+ (is (= 1 (count result)))+ (is (= "name" (u/lower-case-en (:name (first result)))))))))++;;; Scalar Subquery Tests++(deftest ^:parallel scalar-subquery-test+ (testing "Scalar subquery in SELECT"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, (SELECT COUNT(*) FROM ORDERS) AS order_count+ FROM PEOPLE")]+ (is (= 2 (count result)))+ (is (= #{"id" "order_count"} (set (returned-column-names result))))))))++;;; More Aggregate Tests++(deftest ^:parallel multiple-aggregates-test+ (testing "Multiple aggregates in one query"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT COUNT(*) AS cnt, SUM(ID) AS total, AVG(ID) AS avg_id+ FROM PEOPLE")]+ (is (= 3 (count result)))+ (is (= #{"cnt" "total" "avg_id"} (set (returned-column-names result))))+ ;; All aggregates are computed+ (is (every? #(= :type/* (:base-type %)) result))))))++(deftest ^:parallel aggregate-with-distinct-test+ (testing "COUNT DISTINCT"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT COUNT(DISTINCT STATE) AS unique_states FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "unique_states" (first (returned-column-names result))))))))++(deftest ^:parallel having-clause-test+ (testing "GROUP BY with HAVING"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT STATE, COUNT(*) AS cnt+ FROM PEOPLE+ GROUP BY STATE+ HAVING COUNT(*) > 1")]+ (is (= 2 (count result)))+ (is (= #{"state" "cnt"} (set (returned-column-names result))))))))++;;; ORDER BY Tests++(deftest ^:parallel order-by-test+ (testing "ORDER BY doesn't affect returned columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, NAME FROM PEOPLE ORDER BY NAME DESC")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++(deftest ^:parallel order-by-expression-test+ (testing "ORDER BY expression not in SELECT"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT NAME FROM PEOPLE ORDER BY ID")]+ ;; ID is not in output, only NAME+ (is (= 1 (count result)))+ (is (= "name" (u/lower-case-en (:name (first result)))))))))++;;; LIMIT/OFFSET Tests++(deftest ^:parallel limit-test+ (testing "LIMIT doesn't affect returned columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, NAME FROM PEOPLE LIMIT 10")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++;;; DISTINCT Tests++(deftest ^:parallel select-distinct-test+ (testing "SELECT DISTINCT returns same columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT DISTINCT STATE FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "state" (u/lower-case-en (:name (first result)))))))))++;;; Qualified Column Names++(deftest ^:parallel fully-qualified-column-test+ (testing "Fully qualified column names"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT PEOPLE.ID, PEOPLE.NAME FROM PEOPLE")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++;;; More Window Function Tests++(deftest ^:parallel window-partition-by-test+ (testing "Window function with PARTITION BY"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT NAME, STATE,+ RANK() OVER (PARTITION BY STATE ORDER BY NAME) AS state_rank+ FROM PEOPLE")]+ (is (= 3 (count result)))+ (is (= #{"name" "state" "state_rank"} (set (returned-column-names result))))))))++(deftest ^:parallel multiple-window-functions-test+ (testing "Multiple window functions"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID,+ ROW_NUMBER() OVER (ORDER BY ID) AS rn,+ LAG(ID) OVER (ORDER BY ID) AS prev_id+ FROM PEOPLE")]+ (is (= 3 (count result)))+ (is (= #{"id" "rn" "prev_id"} (set (returned-column-names result))))))))++;;; UNION ALL / INTERSECT / EXCEPT++(deftest ^:parallel union-all-test+ (testing "UNION ALL preserves all rows but same columns"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, NAME FROM PEOPLE WHERE ID < 5+ UNION ALL+ SELECT ID, NAME FROM PEOPLE WHERE ID >= 5")]+ (is (= 2 (count result)))+ (is (= #{"id" "name"} (set (returned-column-names result))))))))++;;; Coalesce / NVL / IFNULL++(deftest ^:parallel coalesce-test+ (testing "COALESCE creates computed column"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT COALESCE(NAME, 'Unknown') AS name_or_default FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "name_or_default" (first (returned-column-names result))))+ (is (= :type/* (:base-type (first result))))))))++;;; String Functions++(deftest ^:parallel string-concat-test+ (testing "String concatenation"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT NAME || ' from ' || STATE AS description FROM PEOPLE")]+ (is (= 1 (count result)))+ (is (= "description" (first (returned-column-names result))))))))++;;; Math Expressions++(deftest ^:parallel math-expression-test+ (testing "Math expression in SELECT"+ (mt/test-driver :h2+ (let [result (returned-columns+ "SELECT ID, ID * 2 AS doubled, ID + 10 AS plus_ten FROM PEOPLE")]+ (is (= 3 (count result)))+ (is (= #{"id" "doubled" "plus_ten"} (set (returned-column-names result))))+ ;; ID is pure, expressions are computed+ (is (= :type/* (:base-type (second result))))))))++;;; NULL Handling++(deftest ^:parallel null-literal-test+ (testing "NULL literal in SELECT"+ (mt/test-driver :h2+ (let [result (returned-columns "SELECT NULL AS nothing")]+ (is (= 1 (count result)))+ (is (= "nothing" (first (returned-column-names result))))))))diff --git a/test/metabase/sql_tools/test_util.clj b/test/metabase/sql_tools/test_util.cljnew file mode 100644index 00000000000..2a4bdbebc41--- /dev/null+++ b/test/metabase/sql_tools/test_util.clj@@ -0,0 +1,35 @@+(ns metabase.sql-tools.test-util+ "Utilities for testing sql-tools with multiple parser backends.++ Use [[test-parser-backends]] to run tests against both :macaw and :sqlglot:++ (deftest my-test+ (test-parser-backends+ (is (= expected (sql-tools/validate-query ...)))))++ Set MB_SQL_TOOLS_PARSER_BACKEND to run just one backend:++ MB_SQL_TOOLS_PARSER_BACKEND=macaw bin/mage run-tests ..."+ (:require+ [clojure.test :as t]+ [colorize.core :as colorize]+ [environ.core :as env]+ [metabase.sql-tools.settings :as sql-tools.settings]))++(def ^:private all-backends [:macaw :sqlglot])++(defn parser-backends-to-test+ "Returns [[all-backends]], or just the one specified by MB_SQL_TOOLS_PARSER_BACKEND."+ []+ (if-let [explicit (env/env :mb-sql-tools-parser-backend)]+ [(keyword explicit)]+ all-backends))++(defmacro test-parser-backends+ "Run body against both parser backends (or just one if env var is set)."+ {:style/indent 0}+ [& body]+ `(doseq [backend# (parser-backends-to-test)]+ (t/testing (colorize/magenta (name backend#))+ (binding [sql-tools.settings/*parser-backend-override* backend#]+ ~@body))))