Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Error source for native entities (#70021)

lbrdnke741639713b6
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index a7c939bd67d..5970b7e728e 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2220,6 +2220,7 @@
request
revisions
settings
+ sql-tools
task
transforms
transforms-base
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
index e29de2e0b67..625b94fff1f 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
@@ -1,10 +1,15 @@
(ns metabase-enterprise.dependencies.native-validation
(:require
+ [clojure.string :as str]
[metabase.driver :as driver]
+ [metabase.driver.sql :as driver.sql]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.query-processor.compile :as qp.compile]
+ [metabase.sql-tools.core :as sql-tools]
+ [metabase.util :as u]
[metabase.util.malli :as mu]))
(mu/defn- compile-query :- ::lib.schema/native-only-query
@@ -21,15 +26,233 @@
compiled (qp.compile/compile-with-inline-parameters with-params)]
(lib/native-query with-params (:query compiled))))
+(defn- has-card-template-tags?
+ "Returns true if the query has any card-type template tags."
+ [query]
+ (some #(= (:type %) :card) (lib/all-template-tags query)))
+
+(def ^:private card-placeholder-prefix
+ "Prefix used to generate placeholder table names for card references.
+ Card ref {{#N}} becomes mb__validat_card__N in the SQL sent to SQLGlot."
+ "mb__validat_card__")
+
+(defn- parse-card-placeholder
+ "If table-name is a card placeholder (e.g. \"mb__VALIDAT_card__42\" or \"MB__VALIDAT_CARD__42\"),
+ return the card ID. Returns nil otherwise. Case-insensitive to handle driver normalization."
+ [table-name]
+ (let [lower (u/lower-case-en (str table-name))]
+ (when (str/starts-with? lower card-placeholder-prefix)
+ (parse-long (subs lower (count card-placeholder-prefix))))))
+
+(defn- compile-toplevel-query
+ "Compile a query replacing card references with placeholder table names.
+ Card refs like {{#1}} are replaced with mb__validat_card__1.
+
+ Returns the compiled native-only-query, or nil if the original SQL contains
+ the placeholder prefix (collision guard)."
+ [query]
+ (let [stage (first (:stages query))
+ sql (:native stage)
+ ttags (:template-tags stage)
+ card-tags (into {} (filter #(= (:type (val %)) :card)) ttags)]
+ (when-not (str/includes? sql card-placeholder-prefix)
+ (let [modified-sql
+ (reduce (fn [s [tag-name tag]]
+ (str/replace s
+ (str "{{" tag-name "}}")
+ (str card-placeholder-prefix (:card-id tag))))
+ sql
+ card-tags)
+
+ non-card-tags
+ (into {} (remove #(= (:type (val %)) :card)) ttags)
+
+ modified-query
+ (-> query
+ (assoc-in [:stages 0 :native] modified-sql)
+ (assoc-in [:stages 0 :template-tags] non-card-tags))]
+ (compile-query modified-query)))))
+
+(defn- table-deps
+ "Returns the set of table dep maps (e.g. {:table \"products\" :schema \"public\"})
+ referenced directly in the compiled native query.
+ Excludes card placeholder tables (mb__validat_card__N)."
+ [driver compiled]
+ (into #{}
+ (filter (fn [dep]
+ (and (:table dep)
+ (not (parse-card-placeholder (:table dep))))))
+ (driver/native-query-deps driver compiled)))
+
+(defn- extract-source-entity
+ "Extract source entity info from a :single-column col-spec's source-columns.
+ When card-placeholders? is true, recognizes card placeholder table names (mb__validat_card__N).
+ Returns:
+ - {:kind :table, :spec table-spec} for a single real table source
+ - {:kind :card, :card-id N} for a single card placeholder source (only when card-placeholders?)
+ - :multiple for ambiguous sources (multiple sources of any kind)
+ - nil otherwise (no recognizable sources, or not a :single-column spec)"
+ [card-placeholders? col-spec]
+ (let [card-id-fn (if card-placeholders?
+ (fn [source] (some-> source :table :table parse-card-placeholder))
+ (constantly nil))]
+ (when (= (:type col-spec) :single-column)
+ (let [first-sources (first (:source-columns col-spec))
+ all-col-sources (filter #(= (:type %) :all-columns) first-sources)]
+ (cond
+ (empty? all-col-sources) nil
+
+ (> (count all-col-sources) 1)
+ (let [cards (keep card-id-fn all-col-sources)
+ tables (remove card-id-fn all-col-sources)]
+ (cond
+ (and (= 1 (count cards)) (empty? tables))
+ {:kind :card, :card-id (first cards)}
+ (and (empty? cards) (= 1 (count tables)))
+ {:kind :table, :spec (:table (first tables))}
+ :else :multiple))
+
+ :else
+ (let [source (first all-col-sources)
+ card-id (card-id-fn source)]
+ (if card-id
+ {:kind :card, :card-id card-id}
+ {:kind :table, :spec (:table source)})))))))
+
+(defn- resolve-table-id
+ "Map a SQL table spec {:table name, :schema schema} to a Metabase table ID.
+ Returns table ID or nil if not found."
+ [driver mp table-spec]
+ (:table (sql-tools/find-table-or-transform
+ driver
+ (lib.metadata/tables mp)
+ (lib.metadata/transforms mp)
+ table-spec)))
+
+(defn- card-column-exists?
+ "Check if a column name exists in a card's result-metadata (normalized comparison)."
+ [driver mp card-id col-name]
+ (when-let [card (lib.metadata/card mp card-id)]
+ (let [normalized (driver.sql/normalize-name driver col-name)]
+ (some #(= (driver.sql/normalize-name driver (:name %)) normalized)
+ (:result-metadata card)))))
+
+(defn- enrich-error
+ "Enrich a :missing-column error with source entity information from its col-spec.
+ When card-placeholders? is true, checks card placeholder columns against card metadata:
+ valid columns return nil (suppressed), invalid ones get source attribution.
+ Suppresses :missing-table-alias errors for card placeholder table names.
+ For other error types, returns the error unchanged."
+ [driver mp card-placeholders? col-spec error]
+ (cond
+ ;; Suppress missing-table-alias for card placeholders (e.g. SELECT * FROM mb__validat_card__1)
+ (and card-placeholders?
+ (= (:type error) :missing-table-alias)
+ (parse-card-placeholder (:name error)))
+ nil
+
+ (not= (:type error) :missing-column)
+ error
+
+ :else
+ (let [source (extract-source-entity card-placeholders? col-spec)]
+ (cond
+ ;; Table source
+ (and (map? source) (= (:kind source) :table))
+ (if-let [table-id (resolve-table-id driver mp (:spec source))]
+ (assoc error :source-entity-type :table :source-entity-id table-id)
+ error)
+
+ ;; Card placeholder source — check column against card metadata
+ (and (map? source) (= (:kind source) :card))
+ (let [card-id (:card-id source)]
+ (if (card-column-exists? driver mp card-id (:name error))
+ nil ;; column is valid, suppress error
+ (assoc error :source-entity-type :card :source-entity-id card-id)))
+
+ ;; Multiple/ambiguous sources
+ (= source :multiple)
+ (assoc error :source-entity-type :unknown)
+
+ ;; nil — unknown structure, leave unenriched for fallback
+ :else error))))
+
+(defn- validate-with-sources
+ "Validate a compiled native query, enriching errors with source entity information
+ derived from the col-spec's source-columns.
+ When card-placeholders? is true, card placeholder sources are recognized and valid
+ columns are filtered out (enrich-error returns nil)."
+ [driver compiled card-placeholders?]
+ (let [sql (lib/raw-native-query compiled)
+ mp (lib/->metadata-provider compiled)
+ {:keys [used-fields returned-fields errors]} (sql-tools/field-references driver sql)
+ check-fields (fn [fields]
+ (mapcat (fn [col-spec]
+ (->> (sql-tools/resolve-field driver mp col-spec)
+ (keep :error)
+ (keep (partial enrich-error driver mp card-placeholders? col-spec))))
+ fields))]
+ (into #{}
+ (map (partial driver.sql/normalize-error driver))
+ (concat errors
+ (check-fields used-fields)
+ (check-fields returned-fields)))))
+
+(defn- fallback-enrich
+ "Second-pass enrichment for errors that validate-with-sources couldn't attribute
+ (e.g., subquery sources). Falls back to the table-count approach."
+ [driver compiled errors]
+ (if (every? :source-entity-type errors)
+ errors
+ (let [mp (lib/->metadata-provider compiled)
+ tables (table-deps driver compiled)
+ source (cond
+ (empty? tables)
+ nil
+
+ (= 1 (count tables))
+ (let [table-spec (first tables)]
+ (if-let [table-id (if (int? (:table table-spec))
+ (:table table-spec)
+ (resolve-table-id driver mp table-spec))]
+ {:source-entity-type :table
+ :source-entity-id table-id}
+ {:source-entity-type :unknown}))
+
+ :else
+ {:source-entity-type :unknown})]
+ (if source
+ (into #{} (map (fn [error]
+ (if (or (:source-entity-type error)
+ (not= (:type error) :missing-column))
+ error
+ (merge error source))))
+ errors)
+ errors))))
+
(mu/defn validate-native-query
"Compiles a (native) query and validates that the fields and tables it refers to really exist.
- Returns either nil or a list of errors."
+ Returns a set of errors, each enriched with source entity information when possible."
[driver :- :keyword
query :- ::lib.schema/query]
- (->> query
- compile-query
- (driver/validate-native-query-fields driver)))
+ (into #{}
+ ;; Strip :unknown source attribution — consumers should not distinguish between
+ ;; "we tried and couldn't determine the source" vs "no source info available".
+ (map #(cond-> % (= :unknown (:source-entity-type %)) (dissoc :source-entity-type :source-entity-id)))
+ (if (has-card-template-tags? query)
+ (if-let [compiled (compile-toplevel-query query)]
+ (let [errors (validate-with-sources driver compiled true)]
+ (if (empty? errors)
+ errors
+ (fallback-enrich driver compiled errors)))
+ ;; Fallback: cards with placeholder collision
+ (driver/validate-native-query-fields driver (compile-query query)))
+ (let [compiled (compile-query query)
+ errors (validate-with-sources driver compiled false)]
+ (if (empty? errors)
+ errors
+ (fallback-enrich driver compiled errors))))))
(mu/defn native-result-metadata
"Compiles a (native) query and calculates its result metadata"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
index aa6fdb25899..752c6f00540 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
@@ -197,7 +197,7 @@
:base-provider provider
:graph graph
:include-native? true)]
- (is (= #{:card :transform} (set (keys errors))))
+ (is (= #{:card} (set (keys errors))))
;; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL
;; queries that consume the transform's table.
;; We only check :missing-column errors since SQLGlot and Macaw produce different
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
index dd95d5fdaf8..ef990e6d4d7 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
@@ -8,6 +8,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
[metabase.util :as u]))
(defn- fake-query
@@ -102,14 +103,18 @@
(is (= #{(lib/syntax-error)} result)
(str "Expected syntax-error or empty set, got: " result))))
(testing "bad table wildcard"
- ;; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)
- (is (= (normalize-error-names driver #{(lib/missing-table-alias-error "products")})
+ ;; missing-table-alias errors don't get source attribution — the alias itself is unresolved
+ (is (= (normalize-error-names driver
+ #{(lib/missing-table-alias-error "products")})
(deps.native-validation/validate-native-query
driver
(fake-query mp "select products.* from orders")))))
(testing "bad col reference"
;; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)
- (is (= (normalize-error-names driver #{(lib/missing-column-error "bad")})
+ (is (= (normalize-error-names driver
+ #{(merge (lib/missing-column-error "bad")
+ {:source-entity-type :table
+ :source-entity-id (meta/id :products)})})
(deps.native-validation/validate-native-query
driver
(fake-query mp "select bad from products"))))))))
@@ -131,15 +136,23 @@
(testing "Valid query - selecting existing columns from subquery"
(validates? mp driver 10 empty?))
(testing "Invalid query - selecting non-existent column from subquery"
- (validates? mp driver 11 #{(lib/missing-column-error "CATEGORY")})
- (validates? mp driver 12 #{(lib/missing-column-error "CATEGORY")}))
+ (validates? mp driver 11 #{(merge (lib/missing-column-error "CATEGORY")
+ {:source-entity-type :table
+ :source-entity-id (meta/id :people)})})
+ (validates? mp driver 12 #{(merge (lib/missing-column-error "CATEGORY")
+ {:source-entity-type :table
+ :source-entity-id (meta/id :people)})}))
(testing "Nested subqueries"
(validates? mp driver 13 empty?)
- (validates? mp driver 14 #{(lib/missing-column-error "CATEGORY")}))
+ (validates? mp driver 14 #{(merge (lib/missing-column-error "CATEGORY")
+ {:source-entity-type :table
+ :source-entity-id (meta/id :people)})}))
(testing "SELECT * from subquery expands to subquery columns"
(validates? mp driver 15 empty?)
(validates? mp driver 16 empty?)
- (validates? mp driver 17 #{(lib/missing-column-error "EMAIL")})))))
+ (validates? mp driver 17 #{(merge (lib/missing-column-error "EMAIL")
+ {:source-entity-type :table
+ :source-entity-id (meta/id :people)})})))))
(deftest ^:parallel validate-card-reference-after-expansion-test
(testing "Validation of queries after card references have been expanded"
@@ -149,17 +162,55 @@
(validates? mp driver 18 empty?))
(testing "Card reference expanded to subquery - invalid column"
(validates? mp driver 19
- #{(lib/missing-column-error "DESCRIPTION")}))
+ #{(merge (lib/missing-column-error "DESCRIPTION")
+ {:source-entity-type :card
+ :source-entity-id 1})}))
(testing "Card reference with alias - valid column"
(validates? mp driver 20 empty?))
(testing "Card reference with alias - invalid column"
(validates? mp driver 21
- #{(lib/missing-column-error "PASSWORD")}))
+ #{(merge (lib/missing-column-error "PASSWORD")
+ {:source-entity-type :card
+ :source-entity-id 1})}))
(testing "Wildcard selection from card reference"
(validates? mp driver 22 empty?))
(testing "Invalid column from aliased card"
(validates? mp driver 23
- #{(lib/missing-column-error "LATITUDE")})))))
+ #{(merge (lib/missing-column-error "LATITUDE")
+ {:source-entity-type :card
+ :source-entity-id 1})})))))
+
+(deftest ^:parallel validate-card-reference-multi-card-test
+ (testing "Source attribution with multiple card references"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "Multi-card query, unqualified - source is unknown"
+ (validates? mp driver 24
+ #{(lib/missing-column-error "BAD")}))
+ (testing "Multi-card query, qualified - source attributed to specific card"
+ (validates? mp driver 25
+ #{(merge (lib/missing-column-error "BAD")
+ {:source-entity-type :card
+ :source-entity-id 1})})))))
+
+(deftest ^:parallel validate-card-reference-mixed-table-card-test
+ (testing "Source attribution with mixed table and card references"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "Mixed table+card, unqualified - source is unknown"
+ (validates? mp driver 26
+ #{(lib/missing-column-error "BAD")}))
+ (testing "Mixed table+card, qualified to table - source attributed to table"
+ (validates? mp driver 27
+ #{(merge (lib/missing-column-error "BAD")
+ {:source-entity-type :table
+ :source-entity-id (meta/id :products)})}))
+ (testing "Mixed table+card, missing column + unknown alias"
+ (validates? mp driver 28
+ #{(merge (lib/missing-column-error "BAD")
+ {:source-entity-type :table
+ :source-entity-id (meta/id :products)})
+ (lib/missing-table-alias-error "xix")})))))
(defn- check-result-metadata [driver mp query expected]
(is (=? (normalize-result-metadata expected)
@@ -264,3 +315,204 @@
driver mp
"this is not a query"
[])))))
+
+(deftest ^:parallel validate-native-query-source-attribution-test
+ (testing "errors include source entity info for table-only queries"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "single table query - error attributed to that table"
+ (is (= (normalize-error-names driver
+ #{{:type :missing-column
+ :name "bad"
+ :source-entity-type :table
+ :source-entity-id (meta/id :products)}})
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "select bad from products")))))
+ (testing "multi-table query, unqualified - source is unknown"
+ (is (= (normalize-error-names driver
+ #{{:type :missing-column
+ :name "bad"}})
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "select bad from products join orders on products.id = orders.product_id")))))
+ (testing "multi-table query, qualified - source attributed to specific table"
+ (is (= (normalize-error-names driver
+ #{{:type :missing-column
+ :name "bad"
+ :source-entity-type :table
+ :source-entity-id (meta/id :products)}})
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "select products.bad from products join orders on products.id = orders.product_id")))))
+ (testing "multi-table query, qualified with alias - source attributed to specific table"
+ (is (= (normalize-error-names driver
+ #{{:type :missing-column
+ :name "bad"
+ :source-entity-type :table
+ :source-entity-id (meta/id :products)}})
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "select p.bad from products p join orders o on p.id = o.product_id")))))
+ (testing "no errors - returns empty set"
+ (is (= #{}
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "select * from products"))))))))
+
+(deftest ^:parallel fallback-single-table-cte-test
+ (testing "CTE with single table - fallback attributes missing column to that table"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (is (= (normalize-error-names driver
+ #{{:type :missing-column
+ :name "bad"
+ :source-entity-type :table
+ :source-entity-id (meta/id :products)}})
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "WITH cte AS (SELECT id, title FROM products) SELECT bad FROM cte")))))))
+
+(deftest ^:parallel fallback-multi-table-cte-test
+ (testing "CTE with multiple tables - fallback produces unknown source"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (is (= (normalize-error-names driver
+ #{{:type :missing-column
+ :name "bad"}})
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "WITH cte AS (SELECT p.id, o.total FROM products p JOIN orders o ON p.id = o.product_id) SELECT bad FROM cte")))))))
+
+(deftest ^:parallel fallback-zero-tables-test
+ (testing "CTE with no real table references - error has no source attribution"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (is (= (normalize-error-names driver
+ #{{:type :missing-column
+ :name "bad"}})
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "WITH cte AS (SELECT 1 AS id) SELECT bad FROM cte")))))))
+
+(deftest ^:parallel card-source-attribution-test
+ (testing "errors are attributed to the correct card"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "mixed table+card, qualified to card - attributed to card"
+ (validates? mp driver 29
+ #{(merge (lib/missing-column-error "BAD")
+ {:source-entity-type :card
+ :source-entity-id 1})}))
+ (testing "multi-card, qualified to second card - attributed to card 2"
+ (validates? mp driver 30
+ #{(merge (lib/missing-column-error "BAD")
+ {:source-entity-type :card
+ :source-entity-id 2})})))))
+
+(deftest ^:parallel transitive-card-chain-test
+ (testing "Transitive card chain: c32 -> c31 -> c1, where c31's result-metadata lost CATEGORY"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (is (nil? (some #{"CATEGORY"} (map :name (:result-metadata (lib.metadata/card mp 31)))))
+ "Card 31 should not have CATEGORY in its result-metadata")
+ (validates? mp driver 32
+ #{(merge (lib/missing-column-error "CATEGORY")
+ {:source-entity-type :card
+ :source-entity-id 31})}))))
+
+(deftest ^:parallel cross-reference-mbql-and-native-cards-test
+ (testing "Native query referencing an MBQL card"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "error attributed to MBQL card"
+ (validates? mp driver 34
+ #{(merge (lib/missing-column-error "CATEGORY")
+ {:source-entity-type :card
+ :source-entity-id 33})}))
+ (testing "mixed MBQL + native card refs - errors attributed to correct cards"
+ (validates? mp driver 36
+ #{(merge (lib/missing-column-error "CATEGORY")
+ {:source-entity-type :card
+ :source-entity-id 33})
+ (merge (lib/missing-column-error "BAD")
+ {:source-entity-type :card
+ :source-entity-id 35})})))))
+
+(deftest ^:parallel card-with-special-char-column-test
+ (testing "Card reference where card has special characters in column names"
+ (let [base-mp (deps.tu/mock-metadata-provider {:snippets []})
+ driver (:engine (lib.metadata/database base-mp))
+ card (deps.tu/mock-card base-mp
+ {:id 100
+ :query "SELECT 1"
+ :details {:result-metadata [{:name "FIRST NAME" :base-type :type/Text}
+ {:name "ORDER-ID" :base-type :type/Integer}
+ {:name "NORMAL" :base-type :type/Text}]}})
+ mp (deps.tu/mock-metadata-provider {:cards [card] :snippets []})
+ mkquery (fn [sql]
+ (let [ttags (lib/extract-template-tags mp sql)]
+ (fake-query mp sql ttags)))]
+ (testing "valid column produces no errors"
+ (is (= #{}
+ (deps.native-validation/validate-native-query
+ driver
+ (mkquery "SELECT NORMAL FROM {{#100}}")))))
+ (testing "invalid column produces error attributed to card"
+ (is (= (normalize-error-names driver
+ #{(merge (lib/missing-column-error "BAD")
+ {:source-entity-type :card
+ :source-entity-id 100})})
+ (deps.native-validation/validate-native-query
+ driver
+ (mkquery "SELECT BAD FROM {{#100}}"))))))))
+
+(deftest ^:parallel placeholder-collision-test
+ (testing "When SQL contains the placeholder prefix as a real table, validation runs without source attribution"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))
+ ttags (lib/extract-template-tags mp "SELECT BAD FROM {{#1}}")
+ query (fake-query mp
+ "SELECT BAD FROM mb__validat_card__99 JOIN {{#1}}"
+ ttags)]
+ (is (= (normalize-error-names driver
+ #{(lib/missing-column-error "BAD")})
+ (deps.native-validation/validate-native-query driver query))))))
+
+(deftest ^:parallel placeholder-no-card-refs-test
+ (testing "Queries with tables named as placeholder produce expected validation results"
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 100 :engine :h2}
+ :tables [{:name "MB__VALIDAT_CARD__10330"
+ :schema "PUBLIC"
+ :id 333
+ :db-id 1}]
+ :fields [{:table-id 333
+ :name "ID"
+ :base-type :type/Integer}]})
+ driver (:engine (lib.metadata/database mp))]
+ (testing "Selection of column present in placeholder named table returns no error"
+ (let [query (fake-query mp
+ "SELECT ID FROM MB__VALIDAT_CARD__10330")]
+ (is (= #{}
+ (deps.native-validation/validate-native-query driver query)))))
+ (testing "Selection of column missing from placeholder named table returns an error with source"
+ (let [query (fake-query mp
+ "SELECT BAD FROM MB__VALIDAT_CARD__10330")]
+ (is (= (normalize-error-names driver
+ #{(merge (lib/missing-column-error "BAD")
+ {:source-entity-type :table
+ :source-entity-id 333})})
+ (deps.native-validation/validate-native-query driver query))))))))
+
+(deftest ^:parallel select-from-non-existent-card-test
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "Select from non-existent card produces an error"
+ (let [query (fake-query mp
+ "SELECT ID FROM {{#1234-missing-card}}")]
+ (is (= (normalize-error-names driver
+ #{(merge (lib/missing-column-error "ID")
+ {:source-entity-type :card
+ :source-entity-id 1234})})
+ (deps.native-validation/validate-native-query driver query)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
index ef87562760b..29e04d9bbde 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
@@ -121,5 +121,47 @@
{:id 22
:query "SELECT * FROM {{#1}}"}
{:id 23
- :query "SELECT p.LATITUDE FROM {{#1}} AS p"}])
+ :query "SELECT p.LATITUDE FROM {{#1}} AS p"}
+ {:id 24
+ :query "SELECT BAD FROM {{#1}} AS c1 JOIN {{#2}} AS c2 ON c1.ID = c2.ID"}
+ {:id 25
+ :query "SELECT c1.BAD FROM {{#1}} AS c1 JOIN {{#2}} AS c2 ON c1.ID = c2.ID"}
+ {:id 26
+ :query "SELECT BAD FROM products JOIN {{#1}} AS c1 ON products.ID = c1.ID"}
+ {:id 27
+ :query "SELECT products.BAD FROM products JOIN {{#1}} AS c1 ON products.ID = c1.ID"}
+ {:id 28
+ :query "SELECT xix.x, products.BAD FROM products JOIN {{#1}} AS c1 ON products.ID = c1.ID"}
+ ;; Mixed table+card, qualified to card alias
+ {:id 29
+ :query "SELECT c1.BAD FROM products JOIN {{#1-how-are-you}} AS c1 ON products.ID = c1.ID"}
+ ;; Multi-card, qualified to second card
+ {:id 30
+ :query "SELECT c2.BAD FROM {{#1-im-fine}} AS c1 JOIN {{#2-thank-you}} AS c2 ON c1.ID = c2.ID"}
+ ;; Card 31: middle of transitive chain, passes through card 1's columns
+ ;; but with CATEGORY removed (simulates upstream card 1 was changed)
+ {:id 31
+ :query "SELECT * FROM {{#1}}"
+ :details {:result-metadata
+ (vec (remove #(= (:name %) "CATEGORY")
+ (:result-metadata initial-card-a)))}}
+ ;; Card 32: end of chain, selects CATEGORY which card 31 no longer has
+ {:id 32
+ :query "SELECT CATEGORY FROM {{#31}}"}
+ ;; Card 33: MBQL card with subset of products columns (simulates MBQL intermediary)
+ {:id 33
+ :query (lib/query initial-mock-mp (meta/table-metadata :products))
+ :details {:result-metadata
+ (vec (filter #(#{"ID" "TITLE"} (:name %))
+ (:result-metadata initial-card-a)))}}
+ ;; Card 34: native referencing MBQL card 33 — CATEGORY not in card 33's metadata
+ {:id 34
+ :query "SELECT CATEGORY FROM {{#33}}"}
+ ;; Card 35: native card with orders columns as result-metadata
+ {:id 35
+ :query "SELECT * FROM orders"
+ :details {:result-metadata (:result-metadata initial-card-b)}}
+ ;; Card 36: native referencing both MBQL card 33 and native card 35
+ {:id 36
+ :query "SELECT c1.CATEGORY, c2.BAD FROM {{#33}} AS c1 JOIN {{#35}} AS c2 ON c1.ID = c2.ID"}])
:snippets snippets})))
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index 49159e0fa31..1df1a58c13e 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -209,18 +209,6 @@
;; honeysql, and accepts a keyword too. This way we delegate proper escaping and qualification to honeysql.
(driver/drop-table! driver (:id database) (qualified-name target)))
-(defmulti default-schema
- "Returns the default schema for a given database driver.
-
- Drivers that support any of the `:transforms/...` features must implement this method."
- {:added "0.57.0" :arglists '([driver])}
- driver/dispatch-on-initialized-driver
- :hierarchy #'driver/hierarchy)
-
-(defmethod default-schema :sql
- [_]
- "public")
-
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Dependencies |
;;; +----------------------------------------------------------------------------------------------------------------+
@@ -252,4 +240,4 @@
(p/import-vars
[sql.params.substitution ->prepared-substitution PreparedStatementSubstitution]
- [sql.normalize normalize-name reserved-literal])
+ [sql.normalize default-schema normalize-error normalize-name reserved-literal])
diff --git a/src/metabase/driver/sql/normalize.clj b/src/metabase/driver/sql/normalize.clj
index 591b989c2aa..fde7437e2de 100644
--- a/src/metabase/driver/sql/normalize.clj
+++ b/src/metabase/driver/sql/normalize.clj
@@ -30,6 +30,26 @@
(str/replace quote-quote quote)))
(normalize-unquoted-name driver name-str))))
+(defn normalize-error
+ "Normalize error names using driver-specific case normalization.
+ This ensures error names match database metadata conventions."
+ [driver error]
+ (if-let [error-name (:name error)]
+ (assoc error :name (normalize-name driver error-name))
+ error))
+
+(defmulti default-schema
+ "Returns the default schema for a given database driver.
+
+ Drivers that support any of the `:transforms/...` features must implement this method."
+ {:added "0.57.0" :arglists '([driver])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod default-schema :sql
+ [_]
+ "public")
+
(defmulti reserved-literal
"Checks whether a particular name is actually a literal value in a given sql dialect.
diff --git a/src/metabase/graph/core.cljc b/src/metabase/graph/core.cljc
index 337ce1c0e2c..3e4b3c84c6f 100644
--- a/src/metabase/graph/core.cljc
+++ b/src/metabase/graph/core.cljc
@@ -141,12 +141,13 @@
[f :- [:-> ::node :any]
children :- ::child-map]
(let [all-nodes (all-map-nodes children)
- full-parent-map (->> children
- (mapcat (fn [[parent current-children]]
- (map (fn [child]
- {child #{parent}})
- current-children)))
- (apply merge-with into))]
+ full-parent-map (or (->> children
+ (mapcat (fn [[parent current-children]]
+ (map (fn [child]
+ {child #{parent}})
+ current-children)))
+ (apply merge-with into))
+ {})]
(loop [nodes-remaining all-nodes
parent-map full-parent-map
ignored #{}
diff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.clj
index 541231e50de..67dbd1ca1a4 100644
--- a/src/metabase/sql_tools/common.clj
+++ b/src/metabase/sql_tools/common.clj
@@ -3,26 +3,31 @@
(:require
[clojure.set :as set]
[clojure.string :as str]
- [metabase.driver.sql :as driver.sql]
+ [metabase.driver.sql.normalize :as sql.normalize]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.sql-tools.interface :as sql-tools]
[metabase.util.humanization :as u.humanization]
[metabase.util.malli :as mu]))
+(defn normalize-name
+ "Normalize a name by per driver rules."
+ [driver name*]
+ (sql.normalize/normalize-name driver name*))
+
(defn normalize-table-spec
"Return tables spec with normalized values."
[driver {:keys [table schema]}]
- {:table (driver.sql/normalize-name driver table)
- :schema (some->> schema (driver.sql/normalize-name driver))})
+ {:table (normalize-name driver table)
+ :schema (some->> schema (normalize-name driver))})
(defn find-table-or-transform
"Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.
It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."
[driver tables transforms {search-table :table raw-schema :schema}]
(let [search-schema (or raw-schema
- (driver.sql/default-schema driver))
- normalize (partial driver.sql/normalize-name driver)
+ (sql.normalize/default-schema driver))
+ normalize (partial normalize-name driver)
matches? (fn [db-table db-schema]
(and (= (normalize search-table) (normalize db-table))
(= (some-> search-schema normalize) (some-> db-schema normalize))))]
@@ -101,8 +106,8 @@
:semantic-type :Semantic/*}])
(keep :col (resolve-field driver metadata-provider current-col))))
source-col-set)
- (some #(when (= (driver.sql/normalize-name driver (:name %))
- (driver.sql/normalize-name driver (:column col-spec)))
+ (some #(when (= (normalize-name driver (:name %))
+ (normalize-name driver (:column col-spec)))
%))))))]
{:col (assoc found :lib/desired-column-alias (or alias name))}
{:error (lib/missing-column-error (:column col-spec))})])
@@ -157,14 +162,6 @@
(keep :col))
returned-fields))))
-(defn- normalize-error
- "Normalize error names using driver-specific case normalization.
- This ensures error names match database metadata conventions."
- [driver error]
- (if-let [error-name (:name error)]
- (assoc error :name (driver.sql/normalize-name driver error-name))
- error))
-
(defn validate-query
"Validate native query."
[parser driver native-query]
@@ -176,7 +173,7 @@
(->> (-> errors
(into (check-fields used-fields))
(into (check-fields returned-fields)))
- (map (partial normalize-error driver))
+ (map (partial sql.normalize/normalize-error driver))
set)))
(defn referenced-fields
diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.clj
index d37fdf60832..aa462227433 100644
--- a/src/metabase/sql_tools/core.clj
+++ b/src/metabase/sql_tools/core.clj
@@ -7,6 +7,7 @@
Note: Implementation namespaces (macaw.core, sqlglot.core) are loaded via
metabase.sql-tools.init to avoid circular dependencies."
(:require
+ [metabase.sql-tools.common :as common]
[metabase.sql-tools.interface :as interface]
[metabase.sql-tools.metrics :as metrics]
[metabase.sql-tools.settings :as sql-tools.settings]
@@ -182,3 +183,15 @@
(let [parser (sql-tools.settings/current-parser-backend)]
(metrics/with-operation-timing [parser "add-into-clause"]
(interface/add-into-clause-impl parser driver sql table-name))))
+
+(defn find-table-or-transform
+ "Given a table and schema parsed from a native query, find the matching table or transform.
+ Returns {:table table-id} or {:transform transform-id}, or nil."
+ [driver tables transforms spec]
+ (common/find-table-or-transform driver tables transforms spec))
+
+(defn resolve-field
+ "Resolve a field reference to one or more actual database fields.
+ See [[metabase.sql-tools.common/resolve-field]] for details."
+ [driver mp col-spec]
+ (common/resolve-field driver mp col-spec))
diff --git a/src/metabase/sql_tools/init.clj b/src/metabase/sql_tools/init.clj
index eb9fa473d5d..2c56dbd708a 100644
--- a/src/metabase/sql_tools/init.clj
+++ b/src/metabase/sql_tools/init.clj
@@ -1,5 +1,7 @@
(ns metabase.sql-tools.init
(:require
+ ;; The common implements shims provided through sql-tools.core. Following ensures method registration.
+ [metabase.sql-tools.common]
;; metabase.sql-tools.<parser> is required to ensure method's registration
[metabase.sql-tools.macaw.core]
[metabase.sql-tools.metrics]
diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.clj
index 904d8942126..91b09dd1502 100644
--- a/src/metabase/sql_tools/sqlglot/core.clj
+++ b/src/metabase/sql_tools/sqlglot/core.clj
@@ -1,7 +1,7 @@
(ns metabase.sql-tools.sqlglot.core
(:require
[clojure.string :as str]
- [metabase.driver.sql :as driver.sql]
+ [metabase.driver.sql.normalize :as sql.normalize]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.sql-parsing.core :as sql-parsing]
@@ -54,7 +54,7 @@
(let [db-tables (lib.metadata/tables query)
db-transforms (lib.metadata/transforms query)
sql (lib/raw-native-query query)
- default-schema (driver.sql/default-schema driver)
+ default-schema (sql.normalize/default-schema driver)
query-tables (sql-parsing/referenced-tables (driver->dialect driver) sql)]
(into #{}
(keep (fn [[_catalog table-schema table]]
@@ -90,7 +90,7 @@
(assoc m k
(cond
(and (normalizable-keys k) (string? v))
- (driver.sql/normalize-name driver v)
+ (sql.normalize/normalize-name driver v)
(map? v)
(normalize-field driver v)