The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Surface analysis errors with upstream dependencies (#68092)
Riley Thompson06d4da18c3d2
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj b/enterprise/backend/src/metabase_enterprise/dependencies/analysis.cljindex 23da2103fae..8e275e61821 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj@@ -18,13 +18,13 @@(lib/returned-columns query)));; Analyzing an entity in memory ================================================================-(mu/defn- check-query :- [:set [:ref ::lib.schema.validate/error]]- "Find any bad refs in a `query`."+(mu/defn- check-query :- [:set [:ref ::lib.schema.validate/error-with-source]]+ "Find any bad refs in a `query`. Returns errors with source entity information when possible."[driver :- :keywordquery :- ::lib.schema/query](if (lib/any-native-stage? query)(deps.native/validate-native-query driver query)- (lib/find-bad-refs query)))+ (lib/find-bad-refs-with-source query)))(defmulti check-entity"Given a metadata provider, an entity type and an entity id, find any bad refs in that entity."@@ -36,9 +36,9 @@[_metadata-provider _entity-type _entity-id]nil)-(mu/defmethod check-entity :card :- [:set [:ref ::lib.schema.validate/error]]+(mu/defmethod check-entity :card :- [:set [:ref ::lib.schema.validate/error-with-source]]"Given a `MetadataProvider` and a card ID, analyses the card's query to find any bad refs or other issues.- Returns any findings, and `nil` for a clean query."+ Returns any findings with source information, and `nil` for a clean query."[metadata-provider :- ::lib.schema.metadata/metadata-provider_entity-typecard-id :- ::lib.schema.id/card]@@ -46,10 +46,10 @@driver (:engine (lib.metadata/database query))](check-query driver query)))-(mu/defmethod check-entity :transform :- [:set [:ref ::lib.schema.validate/error]]+(mu/defmethod check-entity :transform :- [:set [:ref ::lib.schema.validate/error-with-source]]"Given a `MetadataProvider` and a transform ID, analyses the transform's query to find any bad refs or other issues.- Returns any findings, and `nil` for a clean transform."+ Returns any findings with source information, and `nil` for a clean transform."[metadata-provider :- ::lib.schema.metadata/metadata-provider_entity-typetransform-id :- ::lib.schema.id/transform]@@ -67,7 +67,7 @@(cond-> (check-query driver query)duplicated-fields (into duplicated-fields))))-(mu/defmethod check-entity :segment :- [:set [:ref ::lib.schema.validate/error]]+(mu/defmethod check-entity :segment :- [:set [:ref ::lib.schema.validate/error-with-source]][metadata-provider :- ::lib.schema.metadata/metadata-provider_entity-typesegment-id :- ::lib.schema.id/segment]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex 1344697a9a2..c2bb86dfb66 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -447,16 +447,42 @@(apply merge-with +)))children-map)))-(defn- node-errors [nodes-by-type]- (-> (into {}- (mapcat (fn [[type ids]]- (->> (t2/select [:model/AnalysisFinding :analyzed_entity_id :finding_details]- :analyzed_entity_type type- :analyzed_entity_id [:in ids])- (map (fn [{:keys [analyzed_entity_id finding_details]}]- [[type analyzed_entity_id] finding_details])))))- nodes-by-type)- not-empty))+(defn- node-errors+ "Fetches and normalizes AnalysisFindingErrors for the given entities.+ Returns {[entity-type entity-id] #{error-maps...}}, or nil if none."+ [nodes-by-type]+ (letfn [(normalize-finding-error+ [{:keys [error_type error_detail]}]+ ;; error_type is stored without namespace (e.g. :missing-column)+ ;; but API schema expects :validate/missing-column+ (case error_type+ ;; These error types use :name+ (:validate/missing-column+ :validate/missing-table-alias+ :validate/duplicate-column)+ {:type error_type :name error_detail}+ ;; validation-exception-error uses :message+ :validate/validation-exception-error+ {:type error_type :message error_detail}+ ;; syntax-error has no additional fields+ :validate/syntax-error+ {:type error_type}+ ;; Default: use :name if detail exists+ (cond-> {:type error_type}+ error_detail (assoc :name error_detail))))+ (normalize-entity-errors [[[entity-type entity-id] errors]]+ (let [normalized-errors (into #{} (map normalize-finding-error) errors)]+ [[entity-type entity-id] normalized-errors]))+ (errors-by-entity-type-and-id [[type ids]]+ (let [finding-errors (t2/select :model/AnalysisFindingError+ :analyzed_entity_type type+ :analyzed_entity_id [:in ids])]+ (->> finding-errors+ (group-by (juxt :analyzed_entity_type :analyzed_entity_id))+ (map normalize-entity-errors))))]+ (->> nodes-by-type+ (into {} (mapcat errors-by-entity-type-and-id))+ not-empty)))(defn- hydrate-entities [entity-type entities](case entity-typediff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.cljindex c7b0bcd151d..18b8edc0d88 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj@@ -1,7 +1,7 @@(ns metabase-enterprise.dependencies.models.analysis-finding(:require[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]- [metabase.lib.normalize :as lib.normalize]+ [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error][metabase.models.interface :as mi][methodical.core :as methodical][toucan2.core :as t2]))@@ -10,39 +10,52 @@(derive :model/AnalysisFinding :metabase/model)-(defn- analysis-finding-details-out [finding]- (some->> (mi/json-out-with-keywordization finding)- (into #{} (map #(lib.normalize/normalize :metabase.lib.schema.validate/error %)))))-(t2/deftransforms :model/AnalysisFinding- {:analyzed_entity_type mi/transform-keyword- :finding_details {:in mi/json-in- :out analysis-finding-details-out}})+ {:analyzed_entity_type mi/transform-keyword})(def ^:dynamic *current-analysis-finding-version*"Current version of the query validation logic.This should be incremented when the analysis logic changes.The background task will re-analyze anything with out-of-date analyses.- This generally shouldn't be rebound in real code, but making it dynamic is convenient for testing"- 4)+ This generally shouldn't be rebound in real code, but making it dynamic is convenient for testing.++ Version history:+ - 3: Initial version+ - 4: Disable naive sql validation+ - 5: Added source entity tracking in analysis_finding_error table"+ 5)++(defn- error->finding-error-row+ "Convert an error from lib/find-bad-refs-with-source to a row for analysis_finding_error table.+ We use `:message` for the error detail if there was an `validation-exception-error`."+ [error]+ {:error-type (:type error)+ :error-detail (or (:name error) (:message error))+ :source-entity-type (:source-entity-type error)+ :source-entity-id (:source-entity-id error)})(defn upsert-analysis!- "Given the details of an AnalysisFinding row, upsert the data into the actual db."+ "Given the details of an AnalysisFinding row, upsert the data into the actual db.+ Also writes individual errors to the analysis_finding_error table with source information."[type instance-id result finding-details]- (let [update {:analyzed_at (mi/now)- :analysis_version *current-analysis-finding-version*- :result result- :finding_details finding-details}- existing-id (t2/select-one-fn :id [:model/AnalysisFinding :id]- :analyzed_entity_type type- :analyzed_entity_id instance-id)]- (if existing-id- (t2/update! :model/AnalysisFinding existing-id update)- (t2/insert! :model/AnalysisFinding- (assoc update- :analyzed_entity_type type- :analyzed_entity_id instance-id)))))+ (t2/with-transaction [_conn]+ (let [update {:analyzed_at (mi/now)+ :analysis_version *current-analysis-finding-version*+ :result result}+ existing-id (t2/select-one-fn :id [:model/AnalysisFinding :id]+ :analyzed_entity_type type+ :analyzed_entity_id instance-id)]+ (if existing-id+ (t2/update! :model/AnalysisFinding existing-id update)+ (t2/insert! :model/AnalysisFinding+ (assoc update+ :analyzed_entity_type type+ :analyzed_entity_id instance-id)))+ (deps.analysis-finding-error/replace-errors-for-entity!+ type+ instance-id+ (map error->finding-error-row finding-details)))))(defn instances-for-analysis"Find a batch of instances with missing or outdated AnalysisFindings"diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_error.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_error.cljnew file mode 100644index 00000000000..e3cff70b6bf--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_error.clj@@ -0,0 +1,83 @@+(ns metabase-enterprise.dependencies.models.analysis-finding-error+ "Model for the `analysis_finding_error` table which stores individual validation errors+ with source entity information. This enables querying errors by the data source that+ causes them (e.g., which cards are broken because table X removed column Y?)."+ (:require+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase.lib.schema.validate :as lib.schema.validate]+ [metabase.models.interface :as mi]+ [metabase.util :as u]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/AnalysisFindingError [_model] :analysis_finding_error)++(derive :model/AnalysisFindingError :metabase/model)++(mr/def ::analysis-finding-error+ "Schema for an AnalysisFindingError record."+ [:map+ [:id ms/PositiveInt]+ [:analyzed_entity_type ::deps.dependency-types/dependency-types]+ [:analyzed_entity_id ms/PositiveInt]+ [:error_type ::lib.schema.validate/validate-error-type]+ [:error_detail {:optional true} [:maybe :string]]+ [:source_entity_type {:optional true} [:maybe ::lib.schema.validate/source-entity-type]]+ [:source_entity_id {:optional true} [:maybe ms/PositiveInt]]])++(t2/deftransforms :model/AnalysisFindingError+ {:analyzed_entity_type mi/transform-keyword+ :error_type mi/transform-keyword+ :source_entity_type mi/transform-keyword})++(mr/def ::error-input+ "Schema for error input maps passed to replace-errors-for-entity!"+ [:map+ [:error-type ::lib.schema.validate/validate-error-type]+ [:error-detail {:optional true} [:maybe :string]]+ [:source-entity-type {:optional true} [:maybe ::lib.schema.validate/source-entity-type]]+ [:source-entity-id {:optional true} [:maybe ms/PositiveInt]]])++(mu/defn replace-errors-for-entity!+ "Delete existing errors for an entity and insert new ones.+ `errors` is a sequence of maps with keys:+ - `:error-type` - keyword like `:validate/missing-column`+ - `:error-detail` - string (column name, alias, message, etc.) or nil+ - `:source-entity-type` - keyword like `:table`, `:card`, or nil+ - `:source-entity-id` - int or nil"+ [entity-type :- ::deps.dependency-types/dependency-types+ entity-id :- ms/PositiveInt+ errors :- [:sequential ::error-input]]+ (t2/with-transaction [_conn]+ (t2/delete! :model/AnalysisFindingError+ :analyzed_entity_type entity-type+ :analyzed_entity_id entity-id)+ (when (seq errors)+ (t2/insert! :model/AnalysisFindingError+ (mapv (fn [{:keys [error-type error-detail source-entity-type source-entity-id]}]+ {:analyzed_entity_type (name entity-type)+ :analyzed_entity_id entity-id+ :error_type (u/qualified-name error-type)+ :error_detail error-detail+ :source_entity_type (some-> source-entity-type name)+ :source_entity_id source-entity-id})+ errors)))))++(mu/defn errors-by-source :- [:sequential ::analysis-finding-error]+ "Get all errors caused by a specific source entity."+ [source-entity-type :- [:maybe ::lib.schema.validate/source-entity-type]+ source-entity-id :- ms/PositiveInt]+ (t2/select :model/AnalysisFindingError+ :source_entity_type source-entity-type+ :source_entity_id source-entity-id))++(mu/defn errors-for-entity :- [:sequential ::analysis-finding-error]+ "Get all errors for a specific analyzed entity."+ [entity-type :- ::deps.dependency-types/dependency-types+ entity-id :- ms/PositiveInt]+ (t2/select :model/AnalysisFindingError+ :analyzed_entity_type entity-type+ :analyzed_entity_id entity-id))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex 2f370655c2a..6289bc5385e 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -57,6 +57,7 @@(def excluded-models"List of models which are not going to be serialized ever."["AnalysisFinding"+ "AnalysisFindingError""ApiKey""ApplicationPermissionsRevision""AuditLog"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.cljindex aa1155e0f99..34bdf0ab942 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj@@ -145,8 +145,14 @@errors (dependencies/errors-from-proposed-edits {:card [card']}:base-provider provider:graph graph)]- (is (=? {:card {downstream-card-id #{(lib/missing-column-error "Tax Rate")}- transformed-card-id #{(lib/missing-column-error "Tax Rate")}}}+ (is (=? {:card {downstream-card-id #{{:type :validate/missing-column+ :name "Tax Rate"+ :source-entity-type :card+ :source-entity-id 101}}+ transformed-card-id #{{:type :validate/missing-column+ :name "Tax Rate"+ :source-entity-type :table+ :source-entity-id 1234567}}}}errors))(is (= [:card] (keys errors)))(is (= #{downstream-card-id transformed-card-id} (set (keys (:card errors)))))))@@ -176,7 +182,10 @@;; queries that consume the transform's table.(is (=? {:card {direct-sql-card-id #{(lib/missing-table-alias-error "NONEXISTENT_TABLE")}transformed-sql-card-id #{(lib/missing-table-alias-error "TRANSFORMED.OUTPUT_TF31")}- transformed-mbql-card-id #{(lib/missing-column-error "RATING")}}+ transformed-mbql-card-id #{{:type :validate/missing-column+ :name "RATING"+ :source-entity-type :table+ :source-entity-id 1234568}}}:transform {(:id sql-transform) #{(lib/missing-table-alias-error "NONEXISTENT_TABLE")}}}errors))(is (= #{:card :transform}diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_error_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_error_test.cljnew file mode 100644index 00000000000..648db911f65--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_error_test.clj@@ -0,0 +1,107 @@+(ns metabase-enterprise.dependencies.models.analysis-finding-error-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(deftest replace-errors-for-entity!-test+ (mt/with-premium-features #{:dependencies}+ (testing "Can insert errors for an entity"+ (mt/with-empty-h2-app-db!+ (let [errors [{:error-type :validate/missing-column+ :error-detail "CATEGORY"+ :source-entity-type :table+ :source-entity-id 100}+ {:error-type :validate/missing-column+ :error-detail "PRICE"+ :source-entity-type :table+ :source-entity-id 100}]]+ (deps.analysis-finding-error/replace-errors-for-entity! :card 1 errors)+ (let [stored (t2/select :model/AnalysisFindingError+ :analyzed_entity_type :card+ :analyzed_entity_id 1)]+ (is (= 2 (count stored)))+ (is (= #{:validate/missing-column} (set (map :error_type stored))))+ (is (= #{"CATEGORY" "PRICE"} (set (map :error_detail stored))))+ (is (= #{:table} (set (map :source_entity_type stored))))+ (is (= #{100} (set (map :source_entity_id stored))))))))+ (testing "Replaces existing errors when called again"+ (mt/with-empty-h2-app-db!+ (let [old-errors [{:error-type :validate/missing-column+ :error-detail "OLD_COLUMN"+ :source-entity-type :table+ :source-entity-id 100}]+ new-errors [{:error-type :validate/syntax-error+ :error-detail nil+ :source-entity-type nil+ :source-entity-id nil}]]+ (deps.analysis-finding-error/replace-errors-for-entity! :card 1 old-errors)+ (is (= 1 (t2/count :model/AnalysisFindingError+ :analyzed_entity_type :card+ :analyzed_entity_id 1)))+ (deps.analysis-finding-error/replace-errors-for-entity! :card 1 new-errors)+ (let [stored (t2/select-one :model/AnalysisFindingError+ :analyzed_entity_type :card+ :analyzed_entity_id 1)]+ (is (= :validate/syntax-error (:error_type stored)))+ (is (nil? (:error_detail stored)))+ (is (nil? (:source_entity_type stored)))+ (is (nil? (:source_entity_id stored)))))))+ (testing "Clears errors when passed empty list"+ (mt/with-empty-h2-app-db!+ (let [errors [{:error-type :validate/missing-column+ :error-detail "CATEGORY"+ :source-entity-type :table+ :source-entity-id 100}]]+ (deps.analysis-finding-error/replace-errors-for-entity! :card 1 errors)+ (is (= 1 (t2/count :model/AnalysisFindingError+ :analyzed_entity_type :card+ :analyzed_entity_id 1)))++ (deps.analysis-finding-error/replace-errors-for-entity! :card 1 [])+ (is (= 0 (t2/count :model/AnalysisFindingError+ :analyzed_entity_type :card+ :analyzed_entity_id 1))))))))++(deftest errors-by-source-test+ (mt/with-premium-features #{:dependencies}+ (mt/with-empty-h2-app-db!+ (testing "Can query errors by source entity"+ (let [errors-card-1 [{:error-type :validate/missing-column+ :error-detail "CATEGORY"+ :source-entity-type :table+ :source-entity-id 100}]+ errors-card-2 [{:error-type :validate/missing-column+ :error-detail "PRICE"+ :source-entity-type :table+ :source-entity-id 100}+ {:error-type :validate/missing-column+ :error-detail "NAME"+ :source-entity-type :table+ :source-entity-id 200}]]+ (deps.analysis-finding-error/replace-errors-for-entity! :card 1 errors-card-1)+ (deps.analysis-finding-error/replace-errors-for-entity! :card 2 errors-card-2)+ (let [errors-from-100 (deps.analysis-finding-error/errors-by-source :table 100)]+ (is (= 2 (count errors-from-100)))+ (is (= #{1 2} (set (map :analyzed_entity_id errors-from-100)))))+ (let [errors-from-200 (deps.analysis-finding-error/errors-by-source :table 200)]+ (is (= 1 (count errors-from-200)))+ (is (= #{2} (set (map :analyzed_entity_id errors-from-200))))))))))++(deftest errors-for-entity-test+ (mt/with-premium-features #{:dependencies}+ (mt/with-empty-h2-app-db!+ (testing "Can query all errors for a specific entity"+ (let [errors [{:error-type :validate/missing-column+ :error-detail "CATEGORY"+ :source-entity-type :table+ :source-entity-id 100}+ {:error-type :validate/missing-column+ :error-detail "PRICE"+ :source-entity-type :card+ :source-entity-id 50}]]+ (deps.analysis-finding-error/replace-errors-for-entity! :card 1 errors)+ (let [entity-errors (deps.analysis-finding-error/errors-for-entity :card 1)]+ (is (= 2 (count entity-errors)))+ (is (= #{"CATEGORY" "PRICE"} (set (map :error_detail entity-errors))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 5f9c8724f57..bc14274fb2c 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -33,6 +33,7 @@- exported as a child of something else (eg. timeline_event under timeline)so they don't need a generated entity_id."#{:model/AnalysisFinding+ :model/AnalysisFindingError:model/ApiKey:model/AuthIdentity:model/HTTPActiondiff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 71290cdbb56..a5f05a7fb2b 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -291,6 +291,133 @@ databaseChangeLog:baseTableName: report_cardconstraintName: fk_report_card_source_card_id_ref_report_card_id+ - changeSet:+ id: v59.2026-01-16T00:00:00+ author: rileythomp+ comment: Create analysis_finding_error table for storing individual errors with source info+ changes:+ - createTable:+ tableName: analysis_finding_error+ remarks: Individual validation errors from analysis with source entity information+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: analyzed_entity_type+ type: varchar(254)+ remarks: The type of the analyzed entity (card, transform, segment, etc.)+ constraints:+ nullable: false+ - column:+ name: analyzed_entity_id+ type: int+ remarks: The ID of the analyzed entity+ constraints:+ nullable: false+ - column:+ name: error_type+ type: varchar(254)+ remarks: The type of error (missing-column, missing-table-alias, duplicate-column, syntax-error, validation-exception-error)+ constraints:+ nullable: false+ - column:+ name: error_detail+ type: varchar(254)+ remarks: Additional error details (e.g., column name, alias name, error message)+ constraints:+ nullable: true+ - column:+ name: source_entity_type+ type: varchar(254)+ remarks: The type of the source entity causing the error (table, card, etc.)+ constraints:+ nullable: true+ - column:+ name: source_entity_id+ type: int+ remarks: The ID of the source entity causing the error+ constraints:+ nullable: true+ rollback:+ - dropTable:+ tableName: analysis_finding_error++ - changeSet:+ id: v59.2026-01-16T00:00:01+ author: rileythomp+ comment: Index for querying errors by source entity+ changes:+ - createIndex:+ tableName: analysis_finding_error+ indexName: idx_analysis_finding_error_source+ columns:+ - column:+ name: source_entity_type+ - column:+ name: source_entity_id+ rollback:+ - dropIndex:+ tableName: analysis_finding_error+ indexName: idx_analysis_finding_error_source++ - changeSet:+ id: v59.2026-01-16T00:00:02+ author: rileythomp+ comment: Index for querying/deleting errors by analyzed entity+ changes:+ - createIndex:+ tableName: analysis_finding_error+ indexName: idx_analysis_finding_error_analyzed+ columns:+ - column:+ name: analyzed_entity_type+ - column:+ name: analyzed_entity_id+ rollback:+ - dropIndex:+ tableName: analysis_finding_error+ indexName: idx_analysis_finding_error_analyzed++ - changeSet:+ id: v59.2026-01-16T00:00:03+ author: rileythomp+ comment: Index for filtering by error type+ changes:+ - createIndex:+ tableName: analysis_finding_error+ indexName: idx_analysis_finding_error_type+ columns:+ - column:+ name: error_type+ rollback:+ - dropIndex:+ tableName: analysis_finding_error+ indexName: idx_analysis_finding_error_type++ - changeSet:+ id: v59.2026-01-16T00:00:04+ author: rileythomp+ comment: Drop redundant finding_details column from analysis_finding table+ changes:+ - dropColumn:+ tableName: analysis_finding+ columnName: finding_details+ rollback:+ - addColumn:+ tableName: analysis_finding+ columns:+ - column:+ name: finding_details+ type: ${text.type}+ remarks: Arbitrary EDN text of the findings+ constraints:+ nullable: true+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex d9145b4a045..99864e2a79d 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -505,6 +505,7 @@[lib.validateduplicate-column-errorfind-bad-refs+ find-bad-refs-with-sourcemissing-column-errormissing-table-alias-errorsyntax-errordiff --git a/src/metabase/lib/schema/validate.cljc b/src/metabase/lib/schema/validate.cljcindex db8d3b67788..06e3c0e6d95 100644--- a/src/metabase/lib/schema/validate.cljc+++ b/src/metabase/lib/schema/validate.cljc@@ -41,3 +41,28 @@[:validate/duplicate-column [:ref ::duplicate-column-error]][:validate/syntax-error [:ref ::syntax-error]][:validate/validation-exception-error [:ref ::validation-exception-error]]]])++(mr/def ::source-entity-type+ "The type of the source entity causing an error."+ [:enum :table :card])++(mr/def ::source-entity+ "Optional source entity tracking fields for dependency analysis."+ [:map+ [:source-entity-type {:optional true} [:maybe ::source-entity-type]]+ [:source-entity-id {:optional true} [:maybe pos-int?]]])++(mr/def ::error-with-source+ "An error with optional source entity information.+ Extends the base error types with source tracking for dependency analysis."+ [:and+ [:map+ [:type {:decode/normalize common/normalize-keyword} ::validate-error-type]+ [:source-entity-type {:optional true} [:maybe ::source-entity-type]]+ [:source-entity-id {:optional true} [:maybe pos-int?]]]+ [:multi {:dispatch #(-> % :type keyword)}+ [:validate/missing-column [:merge [:ref ::missing-column-error] [:ref ::source-entity]]]+ [:validate/missing-table-alias [:merge [:ref ::missing-table-alias-error] [:ref ::source-entity]]]+ [:validate/duplicate-column [:merge [:ref ::duplicate-column-error] [:ref ::source-entity]]]+ [:validate/syntax-error [:merge [:ref ::syntax-error] [:ref ::source-entity]]]+ [:validate/validation-exception-error [:merge [:ref ::validation-exception-error] [:ref ::source-entity]]]]])diff --git a/src/metabase/lib/validate.cljc b/src/metabase/lib/validate.cljcindex fee621f83b9..c05b1b16802 100644--- a/src/metabase/lib/validate.cljc+++ b/src/metabase/lib/validate.cljc@@ -2,9 +2,12 @@"Checks and validation for queries."(:require[metabase.lib.field.resolution :as lib.field.resolution]+ [metabase.lib.join :as lib.join]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.schema :as lib.schema][metabase.lib.schema.validate :as lib.schema.validate]+ [metabase.lib.util :as lib.util][metabase.lib.walk :as lib.walk][metabase.util.malli :as mu]))@@ -37,6 +40,53 @@{:type :validate/validation-exception-error:message message})+(defn- extract-source-from-field-ref+ "Extract the source entity (table or card) for a field ref.+ Returns `{:source-entity-type :table/:card, :source-entity-id <id>}` or `nil`.++ Rules (from the spec):+ 1. If `:source-field` present → find the field's table+ 2. If `:join-alias` present → find the join's source table/card+ 3. If it's the first stage → use main source-table or source-card+ 4. Otherwise → references previous stage column, no source"+ [query stage-number field-ref]+ (let [[_tag {:keys [source-field join-alias]} _id-or-name] field-ref+ stage (lib.util/query-stage query stage-number)]+ (cond+ ;; Case 1: :source-field present - find the field's table via implicit join+ source-field+ (when-let [field-metadata (lib.metadata/field query source-field)]+ (when-let [table-id (:table-id field-metadata)]+ {:source-entity-type :table+ :source-entity-id table-id}))++ ;; Case 2: :join-alias present - find the join's source table/card+ join-alias+ (when-let [join (lib.join/maybe-resolve-join query stage-number join-alias)]+ (let [first-join-stage (first (:stages join))]+ (cond+ (:source-table first-join-stage)+ {:source-entity-type :table+ :source-entity-id (:source-table first-join-stage)}++ (:source-card first-join-stage)+ {:source-entity-type :card+ :source-entity-id (:source-card first-join-stage)})))++ ;; Case 3: First stage - use main source-table or source-card+ (zero? stage-number)+ (cond+ (:source-table stage)+ {:source-entity-type :table+ :source-entity-id (:source-table stage)}++ (:source-card stage)+ {:source-entity-type :card+ :source-entity-id (:source-card stage)})++ ;; Case 4: References previous stage column - no source+ :else nil)))+(mu/defn find-bad-refs :- [:set [:ref ::lib.schema.validate/error]]"Returns a set of validation errors in this query.@@ -60,6 +110,34 @@nil))@bad-fields))+(mu/defn find-bad-refs-with-source :- [:set ::lib.schema.validate/error-with-source]+ "Like [[find-bad-refs]] but includes source entity information for each error.+ Returns a set of error maps, each containing:+ - `:type` - the error type (e.g., `:validate/missing-column`)+ - `:name` - the column name (for missing-column errors)+ - `:source-entity-type` - optional, `:table` or `:card` if source could be determined+ - `:source-entity-id` - optional, the ID of the source entity"+ [query :- ::lib.schema/query]+ (let [bad-fields (volatile! #{})]+ (lib.walk/walk-clauses+ query+ (fn [query path-type path clause]+ (when (and (= path-type :lib.walk/stage)+ (vector? clause)+ (= (first clause) :field))+ (let [{:keys [query stage-number]} (lib.walk/query-for-path query path)+ column (lib.field.resolution/resolve-field-ref query stage-number clause)]+ (when (or (not column)+ (::lib.field.resolution/fallback-metadata? column)+ (not (:active column true)))+ (let [col-name (lib.metadata.calculation/column-name query stage-number column)+ source (extract-source-from-field-ref query stage-number clause)+ error (cond-> (missing-column-error col-name)+ source (merge source))]+ (vswap! bad-fields conj error)))))+ nil))+ @bad-fields))+(comment(require '[metabase.lib.core :as lib])(require '[metabase.lib.test-metadata :as meta])diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex 060686b1ea1..a08cbff78c0 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -14,6 +14,7 @@Tests will check to make sure new models get included in this map."'{:model/Action metabase.actions.models:model/AnalysisFinding metabase-enterprise.dependencies.models.analysis-finding+ :model/AnalysisFindingError metabase-enterprise.dependencies.models.analysis-finding-error:model/ApiKey metabase.api-keys.models.api-key:model/ApplicationPermissionsRevision metabase.permissions.models.application-permissions-revision:model/AuditLog metabase.audit-app.models.audit-logdiff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex 0b8f7caf0ce..b90f4734eac 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -26,6 +26,7 @@(def ^:private models-to-exclude"Models that should *not* be migrated in `load-from-h2`."#{:model/AnalysisFinding+ :model/AnalysisFindingError:model/ApiKey:model/CacheConfig:model/CardFavoritediff --git a/test/metabase/lib/validate_test.cljc b/test/metabase/lib/validate_test.cljcnew file mode 100644index 00000000000..7659b047b62--- /dev/null+++ b/test/metabase/lib/validate_test.cljc@@ -0,0 +1,179 @@+(ns metabase.lib.validate-test+ (:require+ [clojure.test :refer [deftest ^:parallel is testing]]+ [medley.core :as m]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]))++(deftest ^:parallel find-bad-refs-with-source-table-test+ (testing "invalidl query of a table returns error with table as source"+ (let [mp meta/metadata-provider+ query (-> (lib/query mp (meta/table-metadata :orders))+ (lib/filter (lib/= (meta/field-metadata :orders :user-id) 1)))+ bad-query (assoc-in query [:stages 0 :filters 0 2 2] "missingcol")+ errors (lib/find-bad-refs-with-source bad-query)]+ (is (= 1 (count errors)))+ (is (= {:type :validate/missing-column+ :source-entity-type :table+ :source-entity-id (meta/id :orders)+ :name "missingcol"}+ (first errors))))))++(deftest ^:parallel find-bad-refs-with-source-card-test+ (testing "invalid query of a card returns error with card as source"+ (let [mp meta/metadata-provider+ base-query (lib/query mp (meta/table-metadata :products))+ mp (lib.tu/metadata-provider-with-card-from-query mp 101 base-query)+ card (lib/query mp {:lib/type :metadata/card :id 101})+ query (-> card+ (lib/filter (lib/= (m/find-first #(= (:name %) "CATEGORY")+ (lib/returned-columns card))+ "Widget")))+ bad-query (assoc-in query [:stages 0 :filters 0 2 2] "missingcol")+ errors (lib/find-bad-refs-with-source bad-query)]+ (is (= 1 (count errors)))+ (is (= {:type :validate/missing-column+ :source-entity-type :card+ :source-entity-id 101+ :name "missingcol"}+ (first errors))))))++(deftest ^:parallel find-bad-refs-with-source-join-test+ (testing "invalid query with join returns error with joined table as source"+ (let [mp meta/metadata-provider+ orders (meta/table-metadata :orders)+ products (meta/table-metadata :products)+ base-query (lib/query mp orders)+ join-clause (-> (lib/join-clause products)+ (lib/with-join-conditions+ [(lib/= (meta/field-metadata :orders :product-id)+ (meta/field-metadata :products :id))])+ (lib/with-join-fields :all))+ query (lib/join base-query join-clause)+ joined-cols (filter #(= (:lib/source %) :source/joins)+ (lib/returned-columns query))+ joined-col (m/find-first #(= (:name %) "CATEGORY") joined-cols)+ filter-query (lib/filter query (lib/= joined-col "Widget"))+ bad-query (assoc-in filter-query [:stages 0 :filters 0 2 2] "missingcol")+ errors (lib/find-bad-refs-with-source bad-query)]+ (is (= 1 (count errors)))+ (is (= {:type :validate/missing-column+ :source-entity-type :table+ :source-entity-id (meta/id :products)+ :name "missingcol"}+ (first errors))))))++(deftest ^:parallel find-bad-refs-with-source-join-card-test+ (testing "invalid query with join to a card returns error with joined card as source"+ (let [mp meta/metadata-provider+ card-query (lib/query mp (meta/table-metadata :products))+ mp (lib.tu/metadata-provider-with-card-from-query mp 101 card-query)+ orders (meta/table-metadata :orders)+ base-query (lib/query mp orders)+ card-metadata (lib.metadata/card mp 101)+ join-clause (-> (lib/join-clause card-metadata)+ (lib/with-join-conditions+ [(lib/= (meta/field-metadata :orders :product-id)+ (m/find-first #(= (:name %) "ID")+ (lib/returned-columns (lib/query mp card-metadata))))])+ (lib/with-join-fields :all))+ query (lib/join base-query join-clause)+ joined-cols (filter #(= (:lib/source %) :source/joins)+ (lib/returned-columns query))+ joined-col (m/find-first #(= (:name %) "CATEGORY") joined-cols)+ filter-query (lib/filter query (lib/= joined-col "Widget"))+ bad-query (assoc-in filter-query [:stages 0 :filters 0 2 2] "missingcol")+ errors (lib/find-bad-refs-with-source bad-query)]+ (is (= 1 (count errors)))+ (is (= {:type :validate/missing-column+ :source-entity-type :card+ :source-entity-id 101+ :name "missingcol"}+ (first errors))))))++(deftest ^:parallel find-bad-refs-with-source-multiple-joins-test+ (testing "invalid query with joins to both table and card returns errors with both source types"+ (let [mp meta/metadata-provider+ ;; Create a card from people table+ card-query (lib/query mp (meta/table-metadata :people))+ mp (lib.tu/metadata-provider-with-card-from-query mp 101 card-query)+ ;; Base query on orders+ orders (meta/table-metadata :orders)+ base-query (lib/query mp orders)+ ;; Join 1: products table+ products (meta/table-metadata :products)+ products-join (-> (lib/join-clause products)+ (lib/with-join-conditions+ [(lib/= (meta/field-metadata :orders :product-id)+ (meta/field-metadata :products :id))])+ (lib/with-join-fields :all))+ ;; Join 2: people card+ card-metadata (lib.metadata/card mp 101)+ card-join (-> (lib/join-clause card-metadata)+ (lib/with-join-conditions+ [(lib/= (meta/field-metadata :orders :user-id)+ (m/find-first #(= (:name %) "ID")+ (lib/returned-columns (lib/query mp card-metadata))))])+ (lib/with-join-fields :all))+ query (-> base-query+ (lib/join products-join)+ (lib/join card-join))+ joined-cols (filter #(= (:lib/source %) :source/joins)+ (lib/returned-columns query))+ ;; Get a column from each join+ products-col (m/find-first #(= (:name %) "CATEGORY") joined-cols)+ people-col (m/find-first #(= (:name %) "EMAIL") joined-cols)+ ;; Add filters using both joined columns+ filter-query (-> query+ (lib/filter (lib/= products-col "Widget"))+ (lib/filter (lib/= people-col "[email protected]")))+ ;; Corrupt both field refs+ bad-query (-> filter-query+ (assoc-in [:stages 0 :filters 0 2 2] "missingcol1")+ (assoc-in [:stages 0 :filters 1 2 2] "missingcol2"))+ errors (lib/find-bad-refs-with-source bad-query)]+ (is (= 2 (count errors)))+ (is (= #{{:type :validate/missing-column+ :source-entity-type :table+ :source-entity-id (meta/id :products)+ :name "missingcol1"}+ {:type :validate/missing-column+ :source-entity-type :card+ :source-entity-id 101+ :name "missingcol2"}}+ (set errors))))))++(deftest ^:parallel find-bad-refs-with-source-multi-stage-test+ (testing "invalid query referencing previous stage has no source"+ (let [mp meta/metadata-provider+ base-query (-> (lib/query mp (meta/table-metadata :orders))+ (lib/breakout (meta/field-metadata :orders :quantity))+ (lib/aggregate (lib/count))+ lib/append-stage)+ prev-cols (lib/returned-columns base-query)+ count-col (m/find-first #(= (:name %) "count") prev-cols)+ filter-query (lib/filter base-query (lib/> count-col 100))+ bad-query (assoc-in filter-query [:stages 1 :filters 0 2 2] "missingcol")+ errors (lib/find-bad-refs-with-source bad-query)]+ (is (= 1 (count errors)))+ (is (= {:type :validate/missing-column+ :name "missingcol"}+ (first errors))))))++(deftest ^:parallel find-bad-refs-with-source-implicit-join-test+ (testing "invalid query with implicit join via :source-field returns error with FK field's table as source"+ (let [mp meta/metadata-provider+ query (lib/query mp (meta/table-metadata :orders))+ cols (lib/filterable-columns query)+ ij-col (m/find-first #(= (:lib/source %) :source/implicitly-joinable) cols)+ filter-query (lib/filter query (lib/= ij-col "test"))+ bad-query (assoc-in filter-query [:stages 0 :filters 0 2 2] "missingcol")+ errors (lib/find-bad-refs-with-source bad-query)]+ (is (= 1 (count errors)))+ (is (= {:type :validate/missing-column+ :source-entity-type :table+ :source-entity-id (meta/id :orders)+ :name "missingcol"}+ (first errors))))))