Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Enable native entity validation (#69784)

lbrdnk639c8debc91c
diff --git a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
index 0f6b6b0c3e1..e9b5f57bb30 100644
--- a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
+++ b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
@@ -161,9 +161,12 @@ describe("scenarios > dependencies > dependency checks", () => {
cy.get<number>("@transformId").then(H.visitTransform);
goToEditorAndType('SELECT name FROM "Schema A"."Animals"');
- cy.log("no confirmation is shown");
+ cy.log("confirmation is shown");
H.DataStudio.Transforms.saveChangesButton().click();
- cy.wait("@updateTransform");
+ cy.contains(
+ "h2",
+ "These changes will break some other things. Save anyway?",
+ );
});
it("should not show a confirmation if there are no breaking changes when updating a SQL transform after it was run", () => {
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 95d1f71841b..6cd10364d80 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -90,7 +90,9 @@
(cond-> #_card
(:result_metadata body) (assoc :result-metadata (:result_metadata body))))
edits {:card [card]}
- breakages (dependencies/errors-from-proposed-edits edits :base-provider base-provider)]
+ breakages (dependencies/errors-from-proposed-edits edits
+ :base-provider base-provider
+ :include-native? true)]
(broken-cards-response breakages)))
(mr/def ::transform-body
@@ -121,7 +123,9 @@
(cond-> #_transform source (assoc :source source))
(cond-> #_transform target (assoc :target target)))
edits {:transform [transform]}
- breakages (dependencies/errors-from-proposed-edits edits :base-provider base-provider)]
+ breakages (dependencies/errors-from-proposed-edits edits
+ :base-provider base-provider
+ :include-native? true)]
(broken-cards-response breakages))
;; if this isn't a sql query, just claim it works
{:success true}))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
index 6a0d981dff7..4e96c4ca626 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
@@ -285,8 +285,7 @@
(methodical/defmethod events/publish-event! ::check-card-dependents
[_ {:keys [object]}]
- (when (and (premium-features/has-feature? :dependencies)
- (not (models.dependency/is-native-entity? :card object)))
+ (when (premium-features/has-feature? :dependencies)
(lib-be/with-metadata-provider-cache
(let [has-stale-dependents? (t2/with-transaction [_conn]
(deps.findings/upsert-analysis! object)
@@ -301,8 +300,7 @@
(methodical/defmethod events/publish-event! ::check-transform
[_ {:keys [object]}]
- (when (and (premium-features/has-feature? :dependencies)
- (not (models.dependency/is-native-entity? :transform object)))
+ (when (premium-features/has-feature? :dependencies)
(lib-be/with-metadata-provider-cache
(deps.findings/upsert-analysis! object))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
index cb5f9fe73d9..116d79b2f10 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
@@ -40,13 +40,10 @@
(when-let [db-id (instance-db-id toucan-instance)]
(let [mp (lib-be/application-database-metadata-provider db-id)
model (t2/model toucan-instance)
- results (if (models.dependency/is-native-entity? (deps.dependency-types/model->dependency-type model)
- toucan-instance)
- []
- (try (deps.analysis/check-entity mp (deps.dependency-types/model->dependency-type model) (:id toucan-instance))
- (catch Exception e
- (log/error e "Error analyzing entity")
- [(lib/validation-exception-error (.getMessage e))])))
+ results (try (deps.analysis/check-entity mp (deps.dependency-types/model->dependency-type model) (:id toucan-instance))
+ (catch Exception e
+ (log/error e "Error analyzing entity")
+ [(lib/validation-exception-error (.getMessage e))]))
success (empty? results)]
(deps.analysis-finding/upsert-analysis! (deps.dependency-types/model->dependency-type model) (:id toucan-instance) success results))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index 117023c2b19..7595423cd4c 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -14,7 +14,7 @@
(def current-dependency-analysis-version
"Current version of the dependency analysis logic.
This should be incremented when the dependency analysis logic changes."
- 5)
+ 6)
(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index a9b5a2cfb65..abe4818359e 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -301,9 +301,9 @@
dependent-query (lib/native-query mp (str "select * from {{#"
(:id base-card)
"}} orders where total > 100"))
- _dependent-card (card/create-card!
- (card-with-query "Dependent Card filtering on Total" dependent-query)
- user)
+ dependent-card (card/create-card!
+ (card-with-query "Dependent Card filtering on Total" dependent-query)
+ user)
;; Propose changing to products table (doesn't have TOTAL column, breaks downstream)
proposed-query (lib/query mp (lib.metadata/table mp (mt/id :products)))
proposed-card {:id (:id base-card)
@@ -311,8 +311,8 @@
:dataset_query proposed-query
:result_metadata nil}
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check_card" proposed-card)]
- (is (=? {:success true
- :bad_cards []
+ (is (=? {:success false
+ :bad_cards [{:id (:id dependent-card)}]
:bad_transforms []}
response)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index 8f92eeecf51..bcd554ddc15 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -902,20 +902,39 @@
:analyzed_entity_type :card
:analyzed_entity_id card-id))))))))))
-(deftest ^:sequential card-update-ignores-native-cards-test
+(deftest ^:sequential card-update-triggers-native-cards-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Native card updates should not trigger analysis"
- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/native-query mp "select * from products")}
- :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
+ (let [old-version models.analysis-finding/*current-analysis-finding-version*
+ new-version (inc old-version)]
+ (mt/with-temp [:model/Card
+ {parent-id :id :as parent}
+ {:dataset_query (lib/native-query mp "select * from products")}
+
+ :model/Card
+ {child-id :id :as child}
+ {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
+
:model/Dependency _ {:from_entity_type :card
- :from_entity_id child-card-id
+ :from_entity_id child-id
:to_entity_type :card
- :to_entity_id parent-card-id}]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
- (assert-has-analyses
- {:card {parent-card-id nil
- child-card-id nil}}))))))
+ :to_entity_id parent-id}]
+ (deps.findings/upsert-analysis! parent)
+ (deps.findings/upsert-analysis! child)
+ (t2/with-transaction [_conn]
+ (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
+ (events/publish-event! :event/card-update {:object parent
+ :previous-object parent
+ :user-id api/*current-user-id*})
+ (testing "Native card update should trigger analysis"
+ (assert-has-analyses
+ {:card {parent-id new-version}}))
+ (testing "Its dependents should be marked stale"
+ (let [child-finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id child-id)]
+ (is (= old-version (:analysis_version child-finding)))
+ (is (true? (:stale child-finding))))))))))))
(deftest ^:sequential card-update-stops-on-transforms-test
(run-with-dependencies-setup!
@@ -1033,10 +1052,10 @@
other-card-id old-version}
:transform {transform-id new-version}})))))))))
-(deftest ^:sequential transform-update-ignores-native-transforms-test
+(deftest ^:sequential transform-update-triggers-native-transforms-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Native transform updates should not trigger analysis"
+ (testing "Native transform updates should trigger analysis"
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query
:query (lib/native-query mp "select * from products")}
:name "transform_sample"
@@ -1045,7 +1064,7 @@
:type :table}}]
(events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
(assert-has-analyses
- {:transform {transform-id nil}}))))))
+ {:transform {transform-id models.analysis-finding/*current-analysis-finding-version*}}))))))
(deftest ^:sequential transform-run-works-with-no-analyses-test
(run-with-dependencies-setup!
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
index 02adb23e782..42ab84b973b 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
@@ -59,14 +59,14 @@
(binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
(is (= 2 (deps.findings/analyze-batch! :card 2))))))))
-(deftest ^:sequential does-not-analyze-native-entities-test
- (testing "assumes native queries are always fine"
+(deftest ^:sequential does-analyze-native-entities-test
+ (testing "failed analysis result is appropriately stored in the appdb"
(backfill-all-entity-analyses!)
(let [mp (mt/metadata-provider)]
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query mp "utter nonsense")}]
(is (= 1 (deps.findings/analyze-batch! :card 1)))
- (is (= [models.analysis-finding/*current-analysis-finding-version* true]
+ (is (= [models.analysis-finding/*current-analysis-finding-version* false]
(t2/select-one-fn (juxt :analysis_version :result)
:model/AnalysisFinding
:analyzed_entity_id card-id