The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve checker (#72747)
dpsuttona497238a754f
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj b/enterprise/backend/src/metabase_enterprise/dependencies/analysis.cljindex 2b6656a89f9..6e32099704c 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj@@ -54,11 +54,15 @@driver (:engine (lib.metadata/database metadata-provider))query (lib/query metadata-provider query)output-fields (returned-columns driver query)+ ;; Group by the user-visible column name (the alias when present),+ ;; not the underlying column's `:name` — otherwise `t1.name AS a` and+ ;; `t2.name AS b` look like duplicates because both have `:name "name"`.+ output-name #(or (:lib/desired-column-alias %) (:name %))duplicated-fields (->> output-fields- (group-by :name)+ (group-by output-name)vals(keep #(when (> (count %) 1)- (lib/duplicate-column-error (-> % first :name))))+ (lib/duplicate-column-error (output-name (first %)))))seq)](cond-> (check-query driver query)duplicated-fields (into duplicated-fields))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.cljindex 231bc3588a7..3f7b3723061 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj@@ -256,37 +256,41 @@errors)errors))))+(defn- has-table-template-tags?+ "Returns true if the query has any table-type template tags."+ [query]+ (some #(= (:type %) :table) (lib/all-template-tags query)))+(mu/defn validate-native-query"Compiles a (native) query and validates that the fields and tables it refers to really exist.- Returns a set of errors, each enriched with source entity information when possible."+ Returns a set of errors, each enriched with source entity information when possible.+ Returns empty set for queries with table-type template tags since the actual+ table (and therefore columns) is unknown at check time."[driver :- :keywordquery :- ::lib.schema/query]- (into #{}- (comp- ;; Strip :unknown source attribution — consumers should not distinguish between- ;; "we tried and couldn't determine the source" vs "no source info available".- (map #(cond-> % (= :unknown (:source-entity-type %)) (dissoc :source-entity-type :source-entity-id)))- ;; Remove errors referencing table placeholder names- (remove #(some-> (:name %) (str/starts-with? table-placeholder-prefix))))- (if (has-substitutable-template-tags? query)- (if-let [compiled (compile-toplevel-query query)]- (let [errors (validate-with-sources driver compiled true)]+ (if (has-table-template-tags? query)+ #{}+ (into #{}+ (comp+ ;; Strip :unknown source attribution — consumers should not distinguish between+ ;; "we tried and couldn't determine the source" vs "no source info available".+ (map #(cond-> % (= :unknown (:source-entity-type %)) (dissoc :source-entity-type :source-entity-id)))+ ;; Remove errors referencing table placeholder names+ (remove #(some-> (:name %) (str/starts-with? table-placeholder-prefix))))+ (if (has-substitutable-template-tags? query)+ (if-let [compiled (compile-toplevel-query query)]+ (let [errors (validate-with-sources driver compiled true)]+ (if (empty? errors)+ errors+ (fallback-enrich driver compiled errors)))+ ;; Fallback: cards with placeholder collision+ (driver/validate-native-query-fields driver (compile-query query)))+ (let [compiled (compile-query query)+ errors (validate-with-sources driver compiled false)](if (empty? errors)errors- (fallback-enrich driver compiled errors)))- ;; Fallback: cards with placeholder collision- (driver/validate-native-query-fields driver (compile-query query)))- (let [compiled (compile-query query)- errors (validate-with-sources driver compiled false)]- (if (empty? errors)- errors- (fallback-enrich driver compiled errors))))))--(defn- has-table-template-tags?- "Returns true if the query has any table-type template tags."- [query]- (some #(= (:type %) :table) (lib/all-template-tags query)))+ (fallback-enrich driver compiled errors)))))))(mu/defn native-result-metadata"Compiles a (native) query and calculates its result metadata.diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.cljindex ef990e6d4d7..a39b564f3be 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.dependencies.native-validation-test(:require[clojure.test :refer :all]+ [metabase-enterprise.dependencies.analysis :as deps.analysis][metabase-enterprise.dependencies.native-validation :as deps.native-validation][metabase-enterprise.dependencies.test-util :as deps.tu][metabase.driver.sql :as driver.sql]@@ -516,3 +517,95 @@{:source-entity-type :card:source-entity-id 1234})})(deps.native-validation/validate-native-query driver query)))))))++;;; ===========================================================================+;;; Transform duplicate-column detection+;;;+;;; The :transform check in deps.analysis groups output fields by their+;;; user-visible name (the alias when present), not the underlying column's+;;; :name. Otherwise `t1.name AS user_name` and `t2.name AS venue_name` would+;;; look like duplicates because both columns have :name "name".+;;; ===========================================================================++(defn- mp-with-transform [native-sql]+ (let [query (lib/->legacy-MBQL (lib/native-query meta/metadata-provider native-sql))+ transform {:id 1+ :name "tx"+ :source {:type :query :query query}+ :target {:type :table :schema "public" :name "out"}}]+ (lib.tu/mock-metadata-provider meta/metadata-provider {:transforms [transform]})))++(deftest ^:parallel transform-aliased-name-columns-not-duplicate-test+ (testing "two `xxx.name` columns aliased to distinct names are NOT flagged as duplicates"+ (let [mp (mp-with-transform+ "SELECT u.NAME AS user_name, v.NAME AS venue_name+ FROM CHECKINS c+ JOIN USERS u ON c.USER_ID = u.ID+ JOIN VENUES v ON c.VENUE_ID = v.ID")+ errors (deps.analysis/check-entity mp :transform 1)+ dupes (filter #(= :duplicate-column (:type %)) errors)]+ (is (empty? dupes)+ (str "Expected no :duplicate-column errors, got: " (pr-str dupes))))))++(deftest ^:parallel transform-real-duplicate-still-flagged-test+ (testing "a genuine output-name collision IS still flagged"+ (let [mp (mp-with-transform+ "SELECT TOTAL, SUBTOTAL AS total FROM ORDERS")+ errors (deps.analysis/check-entity mp :transform 1)+ dupes (filter #(= :duplicate-column (:type %)) errors)]+ (is (= [{:type :duplicate-column :name "TOTAL"}] dupes)+ (str "Expected one :duplicate-column error, got: " (pr-str dupes))))))++;;; ===========================================================================+;;; Derived-table / CTE column resolution+;;;+;;; Columns defined in inline derived tables (subqueries in FROM) or by+;;; window functions in CTEs must resolve correctly when the outer SELECT+;;; aliases them to a different name. A mutation bug in sql_tools.py's+;;; _find_returned_fields caused the source column's alias to be overwritten+;;; with the outer alias, breaking resolution for later references to the+;;; same column.+;;; ===========================================================================++(deftest ^:parallel derived-table-column-with-alias-test+ (testing "column from derived table resolves when aliased to a different name"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (validates? mp driver 4 #{})+ (testing "simple derived table"+ (is (empty? (deps.native-validation/validate-native-query driver+ (fake-query mp "SELECT datum AS b FROM (SELECT 1 AS datum) DQ")))))+ (testing "multiple references to derived-table column with different aliases"+ (is (empty? (deps.native-validation/validate-native-query driver+ (fake-query mp "SELECT datum AS b, datum - 1 AS d FROM (SELECT 1 AS datum) DQ")))))+ (testing "derived table from GENERATE_SERIES with expressions"+ (is (empty? (deps.native-validation/validate-native-query driver+ (fake-query mp+ "SELECT TO_CHAR(datum, 'yyyymmdd') AS a, datum AS b, datum + 1 AS c+ FROM (SELECT SEQUENCE.DAY AS datum+ FROM GENERATE_SERIES(0, 100) AS SEQUENCE (DAY)+ GROUP BY SEQUENCE.DAY) DQ"))))))))++(deftest ^:parallel cte-window-function-column-test+ (testing "column defined by window function in CTE resolves in outer query"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (is (empty? (deps.native-validation/validate-native-query driver+ (fake-query mp+ "WITH ranked AS (+ SELECT ID, TOTAL,+ LAG(TOTAL) OVER (ORDER BY ID) AS prev_total+ FROM ORDERS)+ SELECT prev_total AS source, TOTAL AS target+ FROM ranked")))))))++(deftest ^:parallel table-template-tag-skipped-test+ (testing "queries with table-type template tags are skipped (table is dynamic)"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))+ query (-> (lib/native-query mp "SELECT * FROM {{my_table}}")+ (lib/with-template-tags {"my_table" {:type :table+ :name "my_table"+ :display-name "My Table"+ :table-id 1}}))]+ (is (empty? (deps.native-validation/validate-native-query driver query))))))diff --git a/resources/python-sources/sql_tools.py b/resources/python-sources/sql_tools.pyindex c4e7bf7f90e..e67af0a934d 100644--- a/resources/python-sources/sql_tools.py+++ b/resources/python-sources/sql_tools.py@@ -1013,6 +1013,11 @@ class FieldReferenceWalker:alias = expr.aliasfor result in inner_results:if "col" in result:+ # Shallow-copy before overwriting alias — the same dict may+ # be referenced as a source-column by other scopes. Mutating+ # it replaces the inner alias (e.g. "datum") with the outer+ # alias (e.g. "b"), breaking column resolution upstream.+ result["col"] = dict(result["col"])result["col"]["alias"] = aliasreturn inner_results@@ -1374,8 +1379,10 @@ class FieldReferenceWalker:"source_columns": source_ref}}]- # Return custom_field as-is to preserve structure- return [{"col": source_column}]+ # Copy before returning — callers may mutate the alias+ # (e.g. _find_returned_fields sets alias for outer SELECT AS),+ # and source_column is shared with the source's returned_fields.+ return [{"col": dict(source_column)}]# For composite_field, etc.: return as-is to preserve structureelse:return [{"col": source_column}]