The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Error source for native entities (#70021)
lbrdnke741639713b6
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex a7c939bd67d..5970b7e728e 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2220,6 +2220,7 @@requestrevisionssettings+ sql-toolstasktransformstransforms-basediff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.cljindex e29de2e0b67..625b94fff1f 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj@@ -1,10 +1,15 @@(ns metabase-enterprise.dependencies.native-validation(:require+ [clojure.string :as str][metabase.driver :as driver]+ [metabase.driver.sql :as driver.sql][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.compile :as qp.compile]+ [metabase.sql-tools.core :as sql-tools]+ [metabase.util :as u][metabase.util.malli :as mu]))(mu/defn- compile-query :- ::lib.schema/native-only-query@@ -21,15 +26,233 @@compiled (qp.compile/compile-with-inline-parameters with-params)](lib/native-query with-params (:query compiled))))+(defn- has-card-template-tags?+ "Returns true if the query has any card-type template tags."+ [query]+ (some #(= (:type %) :card) (lib/all-template-tags query)))++(def ^:private card-placeholder-prefix+ "Prefix used to generate placeholder table names for card references.+ Card ref {{#N}} becomes mb__validat_card__N in the SQL sent to SQLGlot."+ "mb__validat_card__")++(defn- parse-card-placeholder+ "If table-name is a card placeholder (e.g. \"mb__VALIDAT_card__42\" or \"MB__VALIDAT_CARD__42\"),+ return the card ID. Returns nil otherwise. Case-insensitive to handle driver normalization."+ [table-name]+ (let [lower (u/lower-case-en (str table-name))]+ (when (str/starts-with? lower card-placeholder-prefix)+ (parse-long (subs lower (count card-placeholder-prefix))))))++(defn- compile-toplevel-query+ "Compile a query replacing card references with placeholder table names.+ Card refs like {{#1}} are replaced with mb__validat_card__1.++ Returns the compiled native-only-query, or nil if the original SQL contains+ the placeholder prefix (collision guard)."+ [query]+ (let [stage (first (:stages query))+ sql (:native stage)+ ttags (:template-tags stage)+ card-tags (into {} (filter #(= (:type (val %)) :card)) ttags)]+ (when-not (str/includes? sql card-placeholder-prefix)+ (let [modified-sql+ (reduce (fn [s [tag-name tag]]+ (str/replace s+ (str "{{" tag-name "}}")+ (str card-placeholder-prefix (:card-id tag))))+ sql+ card-tags)++ non-card-tags+ (into {} (remove #(= (:type (val %)) :card)) ttags)++ modified-query+ (-> query+ (assoc-in [:stages 0 :native] modified-sql)+ (assoc-in [:stages 0 :template-tags] non-card-tags))]+ (compile-query modified-query)))))++(defn- table-deps+ "Returns the set of table dep maps (e.g. {:table \"products\" :schema \"public\"})+ referenced directly in the compiled native query.+ Excludes card placeholder tables (mb__validat_card__N)."+ [driver compiled]+ (into #{}+ (filter (fn [dep]+ (and (:table dep)+ (not (parse-card-placeholder (:table dep))))))+ (driver/native-query-deps driver compiled)))++(defn- extract-source-entity+ "Extract source entity info from a :single-column col-spec's source-columns.+ When card-placeholders? is true, recognizes card placeholder table names (mb__validat_card__N).+ Returns:+ - {:kind :table, :spec table-spec} for a single real table source+ - {:kind :card, :card-id N} for a single card placeholder source (only when card-placeholders?)+ - :multiple for ambiguous sources (multiple sources of any kind)+ - nil otherwise (no recognizable sources, or not a :single-column spec)"+ [card-placeholders? col-spec]+ (let [card-id-fn (if card-placeholders?+ (fn [source] (some-> source :table :table parse-card-placeholder))+ (constantly nil))]+ (when (= (:type col-spec) :single-column)+ (let [first-sources (first (:source-columns col-spec))+ all-col-sources (filter #(= (:type %) :all-columns) first-sources)]+ (cond+ (empty? all-col-sources) nil++ (> (count all-col-sources) 1)+ (let [cards (keep card-id-fn all-col-sources)+ tables (remove card-id-fn all-col-sources)]+ (cond+ (and (= 1 (count cards)) (empty? tables))+ {:kind :card, :card-id (first cards)}+ (and (empty? cards) (= 1 (count tables)))+ {:kind :table, :spec (:table (first tables))}+ :else :multiple))++ :else+ (let [source (first all-col-sources)+ card-id (card-id-fn source)]+ (if card-id+ {:kind :card, :card-id card-id}+ {:kind :table, :spec (:table source)})))))))++(defn- resolve-table-id+ "Map a SQL table spec {:table name, :schema schema} to a Metabase table ID.+ Returns table ID or nil if not found."+ [driver mp table-spec]+ (:table (sql-tools/find-table-or-transform+ driver+ (lib.metadata/tables mp)+ (lib.metadata/transforms mp)+ table-spec)))++(defn- card-column-exists?+ "Check if a column name exists in a card's result-metadata (normalized comparison)."+ [driver mp card-id col-name]+ (when-let [card (lib.metadata/card mp card-id)]+ (let [normalized (driver.sql/normalize-name driver col-name)]+ (some #(= (driver.sql/normalize-name driver (:name %)) normalized)+ (:result-metadata card)))))++(defn- enrich-error+ "Enrich a :missing-column error with source entity information from its col-spec.+ When card-placeholders? is true, checks card placeholder columns against card metadata:+ valid columns return nil (suppressed), invalid ones get source attribution.+ Suppresses :missing-table-alias errors for card placeholder table names.+ For other error types, returns the error unchanged."+ [driver mp card-placeholders? col-spec error]+ (cond+ ;; Suppress missing-table-alias for card placeholders (e.g. SELECT * FROM mb__validat_card__1)+ (and card-placeholders?+ (= (:type error) :missing-table-alias)+ (parse-card-placeholder (:name error)))+ nil++ (not= (:type error) :missing-column)+ error++ :else+ (let [source (extract-source-entity card-placeholders? col-spec)]+ (cond+ ;; Table source+ (and (map? source) (= (:kind source) :table))+ (if-let [table-id (resolve-table-id driver mp (:spec source))]+ (assoc error :source-entity-type :table :source-entity-id table-id)+ error)++ ;; Card placeholder source — check column against card metadata+ (and (map? source) (= (:kind source) :card))+ (let [card-id (:card-id source)]+ (if (card-column-exists? driver mp card-id (:name error))+ nil ;; column is valid, suppress error+ (assoc error :source-entity-type :card :source-entity-id card-id)))++ ;; Multiple/ambiguous sources+ (= source :multiple)+ (assoc error :source-entity-type :unknown)++ ;; nil — unknown structure, leave unenriched for fallback+ :else error))))++(defn- validate-with-sources+ "Validate a compiled native query, enriching errors with source entity information+ derived from the col-spec's source-columns.+ When card-placeholders? is true, card placeholder sources are recognized and valid+ columns are filtered out (enrich-error returns nil)."+ [driver compiled card-placeholders?]+ (let [sql (lib/raw-native-query compiled)+ mp (lib/->metadata-provider compiled)+ {:keys [used-fields returned-fields errors]} (sql-tools/field-references driver sql)+ check-fields (fn [fields]+ (mapcat (fn [col-spec]+ (->> (sql-tools/resolve-field driver mp col-spec)+ (keep :error)+ (keep (partial enrich-error driver mp card-placeholders? col-spec))))+ fields))]+ (into #{}+ (map (partial driver.sql/normalize-error driver))+ (concat errors+ (check-fields used-fields)+ (check-fields returned-fields)))))++(defn- fallback-enrich+ "Second-pass enrichment for errors that validate-with-sources couldn't attribute+ (e.g., subquery sources). Falls back to the table-count approach."+ [driver compiled errors]+ (if (every? :source-entity-type errors)+ errors+ (let [mp (lib/->metadata-provider compiled)+ tables (table-deps driver compiled)+ source (cond+ (empty? tables)+ nil++ (= 1 (count tables))+ (let [table-spec (first tables)]+ (if-let [table-id (if (int? (:table table-spec))+ (:table table-spec)+ (resolve-table-id driver mp table-spec))]+ {:source-entity-type :table+ :source-entity-id table-id}+ {:source-entity-type :unknown}))++ :else+ {:source-entity-type :unknown})]+ (if source+ (into #{} (map (fn [error]+ (if (or (:source-entity-type error)+ (not= (:type error) :missing-column))+ error+ (merge error source))))+ errors)+ errors))))+(mu/defn validate-native-query"Compiles a (native) query and validates that the fields and tables it refers to really exist.- Returns either nil or a list of errors."+ Returns a set of errors, each enriched with source entity information when possible."[driver :- :keywordquery :- ::lib.schema/query]- (->> query- compile-query- (driver/validate-native-query-fields driver)))+ (into #{}+ ;; Strip :unknown source attribution — consumers should not distinguish between+ ;; "we tried and couldn't determine the source" vs "no source info available".+ (map #(cond-> % (= :unknown (:source-entity-type %)) (dissoc :source-entity-type :source-entity-id)))+ (if (has-card-template-tags? query)+ (if-let [compiled (compile-toplevel-query query)]+ (let [errors (validate-with-sources driver compiled true)]+ (if (empty? errors)+ errors+ (fallback-enrich driver compiled errors)))+ ;; Fallback: cards with placeholder collision+ (driver/validate-native-query-fields driver (compile-query query)))+ (let [compiled (compile-query query)+ errors (validate-with-sources driver compiled false)]+ (if (empty? errors)+ errors+ (fallback-enrich driver compiled errors))))))(mu/defn native-result-metadata"Compiles a (native) query and calculates its result metadata"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.cljindex aa6fdb25899..752c6f00540 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj@@ -197,7 +197,7 @@:base-provider provider:graph graph:include-native? true)]- (is (= #{:card :transform} (set (keys errors))))+ (is (= #{:card} (set (keys errors))));; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL;; queries that consume the transform's table.;; We only check :missing-column errors since SQLGlot and Macaw produce differentdiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.cljindex dd95d5fdaf8..ef990e6d4d7 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj@@ -8,6 +8,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu][metabase.util :as u]))(defn- fake-query@@ -102,14 +103,18 @@(is (= #{(lib/syntax-error)} result)(str "Expected syntax-error or empty set, got: " result))))(testing "bad table wildcard"- ;; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)- (is (= (normalize-error-names driver #{(lib/missing-table-alias-error "products")})+ ;; missing-table-alias errors don't get source attribution — the alias itself is unresolved+ (is (= (normalize-error-names driver+ #{(lib/missing-table-alias-error "products")})(deps.native-validation/validate-native-querydriver(fake-query mp "select products.* from orders")))))(testing "bad col reference";; Normalize expected value using driver conventions (H2 uppercases, Postgres lowercases)- (is (= (normalize-error-names driver #{(lib/missing-column-error "bad")})+ (is (= (normalize-error-names driver+ #{(merge (lib/missing-column-error "bad")+ {:source-entity-type :table+ :source-entity-id (meta/id :products)})})(deps.native-validation/validate-native-querydriver(fake-query mp "select bad from products"))))))))@@ -131,15 +136,23 @@(testing "Valid query - selecting existing columns from subquery"(validates? mp driver 10 empty?))(testing "Invalid query - selecting non-existent column from subquery"- (validates? mp driver 11 #{(lib/missing-column-error "CATEGORY")})- (validates? mp driver 12 #{(lib/missing-column-error "CATEGORY")}))+ (validates? mp driver 11 #{(merge (lib/missing-column-error "CATEGORY")+ {:source-entity-type :table+ :source-entity-id (meta/id :people)})})+ (validates? mp driver 12 #{(merge (lib/missing-column-error "CATEGORY")+ {:source-entity-type :table+ :source-entity-id (meta/id :people)})}))(testing "Nested subqueries"(validates? mp driver 13 empty?)- (validates? mp driver 14 #{(lib/missing-column-error "CATEGORY")}))+ (validates? mp driver 14 #{(merge (lib/missing-column-error "CATEGORY")+ {:source-entity-type :table+ :source-entity-id (meta/id :people)})}))(testing "SELECT * from subquery expands to subquery columns"(validates? mp driver 15 empty?)(validates? mp driver 16 empty?)- (validates? mp driver 17 #{(lib/missing-column-error "EMAIL")})))))+ (validates? mp driver 17 #{(merge (lib/missing-column-error "EMAIL")+ {:source-entity-type :table+ :source-entity-id (meta/id :people)})})))))(deftest ^:parallel validate-card-reference-after-expansion-test(testing "Validation of queries after card references have been expanded"@@ -149,17 +162,55 @@(validates? mp driver 18 empty?))(testing "Card reference expanded to subquery - invalid column"(validates? mp driver 19- #{(lib/missing-column-error "DESCRIPTION")}))+ #{(merge (lib/missing-column-error "DESCRIPTION")+ {:source-entity-type :card+ :source-entity-id 1})}))(testing "Card reference with alias - valid column"(validates? mp driver 20 empty?))(testing "Card reference with alias - invalid column"(validates? mp driver 21- #{(lib/missing-column-error "PASSWORD")}))+ #{(merge (lib/missing-column-error "PASSWORD")+ {:source-entity-type :card+ :source-entity-id 1})}))(testing "Wildcard selection from card reference"(validates? mp driver 22 empty?))(testing "Invalid column from aliased card"(validates? mp driver 23- #{(lib/missing-column-error "LATITUDE")})))))+ #{(merge (lib/missing-column-error "LATITUDE")+ {:source-entity-type :card+ :source-entity-id 1})})))))++(deftest ^:parallel validate-card-reference-multi-card-test+ (testing "Source attribution with multiple card references"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "Multi-card query, unqualified - source is unknown"+ (validates? mp driver 24+ #{(lib/missing-column-error "BAD")}))+ (testing "Multi-card query, qualified - source attributed to specific card"+ (validates? mp driver 25+ #{(merge (lib/missing-column-error "BAD")+ {:source-entity-type :card+ :source-entity-id 1})})))))++(deftest ^:parallel validate-card-reference-mixed-table-card-test+ (testing "Source attribution with mixed table and card references"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "Mixed table+card, unqualified - source is unknown"+ (validates? mp driver 26+ #{(lib/missing-column-error "BAD")}))+ (testing "Mixed table+card, qualified to table - source attributed to table"+ (validates? mp driver 27+ #{(merge (lib/missing-column-error "BAD")+ {:source-entity-type :table+ :source-entity-id (meta/id :products)})}))+ (testing "Mixed table+card, missing column + unknown alias"+ (validates? mp driver 28+ #{(merge (lib/missing-column-error "BAD")+ {:source-entity-type :table+ :source-entity-id (meta/id :products)})+ (lib/missing-table-alias-error "xix")})))))(defn- check-result-metadata [driver mp query expected](is (=? (normalize-result-metadata expected)@@ -264,3 +315,204 @@driver mp"this is not a query"[])))))++(deftest ^:parallel validate-native-query-source-attribution-test+ (testing "errors include source entity info for table-only queries"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "single table query - error attributed to that table"+ (is (= (normalize-error-names driver+ #{{:type :missing-column+ :name "bad"+ :source-entity-type :table+ :source-entity-id (meta/id :products)}})+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "select bad from products")))))+ (testing "multi-table query, unqualified - source is unknown"+ (is (= (normalize-error-names driver+ #{{:type :missing-column+ :name "bad"}})+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "select bad from products join orders on products.id = orders.product_id")))))+ (testing "multi-table query, qualified - source attributed to specific table"+ (is (= (normalize-error-names driver+ #{{:type :missing-column+ :name "bad"+ :source-entity-type :table+ :source-entity-id (meta/id :products)}})+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "select products.bad from products join orders on products.id = orders.product_id")))))+ (testing "multi-table query, qualified with alias - source attributed to specific table"+ (is (= (normalize-error-names driver+ #{{:type :missing-column+ :name "bad"+ :source-entity-type :table+ :source-entity-id (meta/id :products)}})+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "select p.bad from products p join orders o on p.id = o.product_id")))))+ (testing "no errors - returns empty set"+ (is (= #{}+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "select * from products"))))))))++(deftest ^:parallel fallback-single-table-cte-test+ (testing "CTE with single table - fallback attributes missing column to that table"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (is (= (normalize-error-names driver+ #{{:type :missing-column+ :name "bad"+ :source-entity-type :table+ :source-entity-id (meta/id :products)}})+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "WITH cte AS (SELECT id, title FROM products) SELECT bad FROM cte")))))))++(deftest ^:parallel fallback-multi-table-cte-test+ (testing "CTE with multiple tables - fallback produces unknown source"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (is (= (normalize-error-names driver+ #{{:type :missing-column+ :name "bad"}})+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "WITH cte AS (SELECT p.id, o.total FROM products p JOIN orders o ON p.id = o.product_id) SELECT bad FROM cte")))))))++(deftest ^:parallel fallback-zero-tables-test+ (testing "CTE with no real table references - error has no source attribution"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (is (= (normalize-error-names driver+ #{{:type :missing-column+ :name "bad"}})+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "WITH cte AS (SELECT 1 AS id) SELECT bad FROM cte")))))))++(deftest ^:parallel card-source-attribution-test+ (testing "errors are attributed to the correct card"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "mixed table+card, qualified to card - attributed to card"+ (validates? mp driver 29+ #{(merge (lib/missing-column-error "BAD")+ {:source-entity-type :card+ :source-entity-id 1})}))+ (testing "multi-card, qualified to second card - attributed to card 2"+ (validates? mp driver 30+ #{(merge (lib/missing-column-error "BAD")+ {:source-entity-type :card+ :source-entity-id 2})})))))++(deftest ^:parallel transitive-card-chain-test+ (testing "Transitive card chain: c32 -> c31 -> c1, where c31's result-metadata lost CATEGORY"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (is (nil? (some #{"CATEGORY"} (map :name (:result-metadata (lib.metadata/card mp 31)))))+ "Card 31 should not have CATEGORY in its result-metadata")+ (validates? mp driver 32+ #{(merge (lib/missing-column-error "CATEGORY")+ {:source-entity-type :card+ :source-entity-id 31})}))))++(deftest ^:parallel cross-reference-mbql-and-native-cards-test+ (testing "Native query referencing an MBQL card"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "error attributed to MBQL card"+ (validates? mp driver 34+ #{(merge (lib/missing-column-error "CATEGORY")+ {:source-entity-type :card+ :source-entity-id 33})}))+ (testing "mixed MBQL + native card refs - errors attributed to correct cards"+ (validates? mp driver 36+ #{(merge (lib/missing-column-error "CATEGORY")+ {:source-entity-type :card+ :source-entity-id 33})+ (merge (lib/missing-column-error "BAD")+ {:source-entity-type :card+ :source-entity-id 35})})))))++(deftest ^:parallel card-with-special-char-column-test+ (testing "Card reference where card has special characters in column names"+ (let [base-mp (deps.tu/mock-metadata-provider {:snippets []})+ driver (:engine (lib.metadata/database base-mp))+ card (deps.tu/mock-card base-mp+ {:id 100+ :query "SELECT 1"+ :details {:result-metadata [{:name "FIRST NAME" :base-type :type/Text}+ {:name "ORDER-ID" :base-type :type/Integer}+ {:name "NORMAL" :base-type :type/Text}]}})+ mp (deps.tu/mock-metadata-provider {:cards [card] :snippets []})+ mkquery (fn [sql]+ (let [ttags (lib/extract-template-tags mp sql)]+ (fake-query mp sql ttags)))]+ (testing "valid column produces no errors"+ (is (= #{}+ (deps.native-validation/validate-native-query+ driver+ (mkquery "SELECT NORMAL FROM {{#100}}")))))+ (testing "invalid column produces error attributed to card"+ (is (= (normalize-error-names driver+ #{(merge (lib/missing-column-error "BAD")+ {:source-entity-type :card+ :source-entity-id 100})})+ (deps.native-validation/validate-native-query+ driver+ (mkquery "SELECT BAD FROM {{#100}}"))))))))++(deftest ^:parallel placeholder-collision-test+ (testing "When SQL contains the placeholder prefix as a real table, validation runs without source attribution"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))+ ttags (lib/extract-template-tags mp "SELECT BAD FROM {{#1}}")+ query (fake-query mp+ "SELECT BAD FROM mb__validat_card__99 JOIN {{#1}}"+ ttags)]+ (is (= (normalize-error-names driver+ #{(lib/missing-column-error "BAD")})+ (deps.native-validation/validate-native-query driver query))))))++(deftest ^:parallel placeholder-no-card-refs-test+ (testing "Queries with tables named as placeholder produce expected validation results"+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 100 :engine :h2}+ :tables [{:name "MB__VALIDAT_CARD__10330"+ :schema "PUBLIC"+ :id 333+ :db-id 1}]+ :fields [{:table-id 333+ :name "ID"+ :base-type :type/Integer}]})+ driver (:engine (lib.metadata/database mp))]+ (testing "Selection of column present in placeholder named table returns no error"+ (let [query (fake-query mp+ "SELECT ID FROM MB__VALIDAT_CARD__10330")]+ (is (= #{}+ (deps.native-validation/validate-native-query driver query)))))+ (testing "Selection of column missing from placeholder named table returns an error with source"+ (let [query (fake-query mp+ "SELECT BAD FROM MB__VALIDAT_CARD__10330")]+ (is (= (normalize-error-names driver+ #{(merge (lib/missing-column-error "BAD")+ {:source-entity-type :table+ :source-entity-id 333})})+ (deps.native-validation/validate-native-query driver query))))))))++(deftest ^:parallel select-from-non-existent-card-test+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "Select from non-existent card produces an error"+ (let [query (fake-query mp+ "SELECT ID FROM {{#1234-missing-card}}")]+ (is (= (normalize-error-names driver+ #{(merge (lib/missing-column-error "ID")+ {:source-entity-type :card+ :source-entity-id 1234})})+ (deps.native-validation/validate-native-query driver query)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.cljindex ef87562760b..29e04d9bbde 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj@@ -121,5 +121,47 @@{:id 22:query "SELECT * FROM {{#1}}"}{:id 23- :query "SELECT p.LATITUDE FROM {{#1}} AS p"}])+ :query "SELECT p.LATITUDE FROM {{#1}} AS p"}+ {:id 24+ :query "SELECT BAD FROM {{#1}} AS c1 JOIN {{#2}} AS c2 ON c1.ID = c2.ID"}+ {:id 25+ :query "SELECT c1.BAD FROM {{#1}} AS c1 JOIN {{#2}} AS c2 ON c1.ID = c2.ID"}+ {:id 26+ :query "SELECT BAD FROM products JOIN {{#1}} AS c1 ON products.ID = c1.ID"}+ {:id 27+ :query "SELECT products.BAD FROM products JOIN {{#1}} AS c1 ON products.ID = c1.ID"}+ {:id 28+ :query "SELECT xix.x, products.BAD FROM products JOIN {{#1}} AS c1 ON products.ID = c1.ID"}+ ;; Mixed table+card, qualified to card alias+ {:id 29+ :query "SELECT c1.BAD FROM products JOIN {{#1-how-are-you}} AS c1 ON products.ID = c1.ID"}+ ;; Multi-card, qualified to second card+ {:id 30+ :query "SELECT c2.BAD FROM {{#1-im-fine}} AS c1 JOIN {{#2-thank-you}} AS c2 ON c1.ID = c2.ID"}+ ;; Card 31: middle of transitive chain, passes through card 1's columns+ ;; but with CATEGORY removed (simulates upstream card 1 was changed)+ {:id 31+ :query "SELECT * FROM {{#1}}"+ :details {:result-metadata+ (vec (remove #(= (:name %) "CATEGORY")+ (:result-metadata initial-card-a)))}}+ ;; Card 32: end of chain, selects CATEGORY which card 31 no longer has+ {:id 32+ :query "SELECT CATEGORY FROM {{#31}}"}+ ;; Card 33: MBQL card with subset of products columns (simulates MBQL intermediary)+ {:id 33+ :query (lib/query initial-mock-mp (meta/table-metadata :products))+ :details {:result-metadata+ (vec (filter #(#{"ID" "TITLE"} (:name %))+ (:result-metadata initial-card-a)))}}+ ;; Card 34: native referencing MBQL card 33 — CATEGORY not in card 33's metadata+ {:id 34+ :query "SELECT CATEGORY FROM {{#33}}"}+ ;; Card 35: native card with orders columns as result-metadata+ {:id 35+ :query "SELECT * FROM orders"+ :details {:result-metadata (:result-metadata initial-card-b)}}+ ;; Card 36: native referencing both MBQL card 33 and native card 35+ {:id 36+ :query "SELECT c1.CATEGORY, c2.BAD FROM {{#33}} AS c1 JOIN {{#35}} AS c2 ON c1.ID = c2.ID"}]):snippets snippets})))diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex 49159e0fa31..1df1a58c13e 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -209,18 +209,6 @@;; honeysql, and accepts a keyword too. This way we delegate proper escaping and qualification to honeysql.(driver/drop-table! driver (:id database) (qualified-name target)))-(defmulti default-schema- "Returns the default schema for a given database driver.-- Drivers that support any of the `:transforms/...` features must implement this method."- {:added "0.57.0" :arglists '([driver])}- driver/dispatch-on-initialized-driver- :hierarchy #'driver/hierarchy)--(defmethod default-schema :sql- [_]- "public")-;;; +----------------------------------------------------------------------------------------------------------------+;;; | Dependencies |;;; +----------------------------------------------------------------------------------------------------------------+@@ -252,4 +240,4 @@(p/import-vars[sql.params.substitution ->prepared-substitution PreparedStatementSubstitution]- [sql.normalize normalize-name reserved-literal])+ [sql.normalize default-schema normalize-error normalize-name reserved-literal])diff --git a/src/metabase/driver/sql/normalize.clj b/src/metabase/driver/sql/normalize.cljindex 591b989c2aa..fde7437e2de 100644--- a/src/metabase/driver/sql/normalize.clj+++ b/src/metabase/driver/sql/normalize.clj@@ -30,6 +30,26 @@(str/replace quote-quote quote)))(normalize-unquoted-name driver name-str))))+(defn normalize-error+ "Normalize error names using driver-specific case normalization.+ This ensures error names match database metadata conventions."+ [driver error]+ (if-let [error-name (:name error)]+ (assoc error :name (normalize-name driver error-name))+ error))++(defmulti default-schema+ "Returns the default schema for a given database driver.++ Drivers that support any of the `:transforms/...` features must implement this method."+ {:added "0.57.0" :arglists '([driver])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod default-schema :sql+ [_]+ "public")+(defmulti reserved-literal"Checks whether a particular name is actually a literal value in a given sql dialect.diff --git a/src/metabase/graph/core.cljc b/src/metabase/graph/core.cljcindex 337ce1c0e2c..3e4b3c84c6f 100644--- a/src/metabase/graph/core.cljc+++ b/src/metabase/graph/core.cljc@@ -141,12 +141,13 @@[f :- [:-> ::node :any]children :- ::child-map](let [all-nodes (all-map-nodes children)- full-parent-map (->> children- (mapcat (fn [[parent current-children]]- (map (fn [child]- {child #{parent}})- current-children)))- (apply merge-with into))]+ full-parent-map (or (->> children+ (mapcat (fn [[parent current-children]]+ (map (fn [child]+ {child #{parent}})+ current-children)))+ (apply merge-with into))+ {})](loop [nodes-remaining all-nodesparent-map full-parent-mapignored #{}diff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.cljindex 541231e50de..67dbd1ca1a4 100644--- a/src/metabase/sql_tools/common.clj+++ b/src/metabase/sql_tools/common.clj@@ -3,26 +3,31 @@(:require[clojure.set :as set][clojure.string :as str]- [metabase.driver.sql :as driver.sql]+ [metabase.driver.sql.normalize :as sql.normalize][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.sql-tools.interface :as sql-tools][metabase.util.humanization :as u.humanization][metabase.util.malli :as mu]))+(defn normalize-name+ "Normalize a name by per driver rules."+ [driver name*]+ (sql.normalize/normalize-name driver name*))+(defn normalize-table-spec"Return tables spec with normalized values."[driver {:keys [table schema]}]- {:table (driver.sql/normalize-name driver table)- :schema (some->> schema (driver.sql/normalize-name driver))})+ {:table (normalize-name driver table)+ :schema (some->> schema (normalize-name driver))})(defn find-table-or-transform"Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."[driver tables transforms {search-table :table raw-schema :schema}](let [search-schema (or raw-schema- (driver.sql/default-schema driver))- normalize (partial driver.sql/normalize-name driver)+ (sql.normalize/default-schema driver))+ normalize (partial normalize-name driver)matches? (fn [db-table db-schema](and (= (normalize search-table) (normalize db-table))(= (some-> search-schema normalize) (some-> db-schema normalize))))]@@ -101,8 +106,8 @@:semantic-type :Semantic/*}])(keep :col (resolve-field driver metadata-provider current-col))))source-col-set)- (some #(when (= (driver.sql/normalize-name driver (:name %))- (driver.sql/normalize-name driver (:column col-spec)))+ (some #(when (= (normalize-name driver (:name %))+ (normalize-name driver (:column col-spec)))%))))))]{:col (assoc found :lib/desired-column-alias (or alias name))}{:error (lib/missing-column-error (:column col-spec))})])@@ -157,14 +162,6 @@(keep :col))returned-fields))))-(defn- normalize-error- "Normalize error names using driver-specific case normalization.- This ensures error names match database metadata conventions."- [driver error]- (if-let [error-name (:name error)]- (assoc error :name (driver.sql/normalize-name driver error-name))- error))-(defn validate-query"Validate native query."[parser driver native-query]@@ -176,7 +173,7 @@(->> (-> errors(into (check-fields used-fields))(into (check-fields returned-fields)))- (map (partial normalize-error driver))+ (map (partial sql.normalize/normalize-error driver))set)))(defn referenced-fieldsdiff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.cljindex d37fdf60832..aa462227433 100644--- a/src/metabase/sql_tools/core.clj+++ b/src/metabase/sql_tools/core.clj@@ -7,6 +7,7 @@Note: Implementation namespaces (macaw.core, sqlglot.core) are loaded viametabase.sql-tools.init to avoid circular dependencies."(:require+ [metabase.sql-tools.common :as common][metabase.sql-tools.interface :as interface][metabase.sql-tools.metrics :as metrics][metabase.sql-tools.settings :as sql-tools.settings]@@ -182,3 +183,15 @@(let [parser (sql-tools.settings/current-parser-backend)](metrics/with-operation-timing [parser "add-into-clause"](interface/add-into-clause-impl parser driver sql table-name))))++(defn find-table-or-transform+ "Given a table and schema parsed from a native query, find the matching table or transform.+ Returns {:table table-id} or {:transform transform-id}, or nil."+ [driver tables transforms spec]+ (common/find-table-or-transform driver tables transforms spec))++(defn resolve-field+ "Resolve a field reference to one or more actual database fields.+ See [[metabase.sql-tools.common/resolve-field]] for details."+ [driver mp col-spec]+ (common/resolve-field driver mp col-spec))diff --git a/src/metabase/sql_tools/init.clj b/src/metabase/sql_tools/init.cljindex eb9fa473d5d..2c56dbd708a 100644--- a/src/metabase/sql_tools/init.clj+++ b/src/metabase/sql_tools/init.clj@@ -1,5 +1,7 @@(ns metabase.sql-tools.init(:require+ ;; The common implements shims provided through sql-tools.core. Following ensures method registration.+ [metabase.sql-tools.common];; metabase.sql-tools.<parser> is required to ensure method's registration[metabase.sql-tools.macaw.core][metabase.sql-tools.metrics]diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.cljindex 904d8942126..91b09dd1502 100644--- a/src/metabase/sql_tools/sqlglot/core.clj+++ b/src/metabase/sql_tools/sqlglot/core.clj@@ -1,7 +1,7 @@(ns metabase.sql-tools.sqlglot.core(:require[clojure.string :as str]- [metabase.driver.sql :as driver.sql]+ [metabase.driver.sql.normalize :as sql.normalize][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.sql-parsing.core :as sql-parsing]@@ -54,7 +54,7 @@(let [db-tables (lib.metadata/tables query)db-transforms (lib.metadata/transforms query)sql (lib/raw-native-query query)- default-schema (driver.sql/default-schema driver)+ default-schema (sql.normalize/default-schema driver)query-tables (sql-parsing/referenced-tables (driver->dialect driver) sql)](into #{}(keep (fn [[_catalog table-schema table]]@@ -90,7 +90,7 @@(assoc m k(cond(and (normalizable-keys k) (string? v))- (driver.sql/normalize-name driver v)+ (sql.normalize/normalize-name driver v)(map? v)(normalize-field driver v)