Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Disable native sql validation (#68148)

Williameacff63768a1
diff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.ts
index 5d9757b1d32..a364520f52f 100644
--- a/e2e/support/helpers/e2e-transform-helpers.ts
+++ b/e2e/support/helpers/e2e-transform-helpers.ts
@@ -31,6 +31,10 @@ export function visitTransform(transformId: TransformId) {
cy.visit(`/data-studio/transforms/${transformId}`);
}
+export function runTransform(transformId: TransformId) {
+ cy.request("POST", `/api/ee/transform/${transformId}/run`);
+}
+
const WAIT_TIMEOUT = 10000;
const WAIT_INTERVAL = 100;
diff --git a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
index 2e94cfc56b2..3126c6cb9b4 100644
--- a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
+++ b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
@@ -104,43 +104,6 @@ describe("scenarios > dependencies > dependency checks", () => {
});
});
- describe("models", () => {
- it("should be able to confirm or cancel breaking changes to a SQL model", () => {
- createSqlModelWithDependentSqlQuestions();
-
- cy.log("make breaking changes");
- cy.get<number>("@modelId").then(H.visitModel);
- H.openQuestionActions("Edit query definition");
- H.NativeEditor.clear().type("SELECT ID, TITLE FROM PRODUCTS");
- H.runNativeQuery();
-
- cy.log("cancel breaking changes");
- H.datasetEditBar().button("Save changes").click();
- H.modal().within(() => {
- cy.findByText("Question with fields").should("be.visible");
- cy.findByText("Question without fields").should("not.exist");
- cy.findByText("Base model").should("not.exist");
- cy.button("Cancel").click();
- });
- cy.get("@updateCard.all").should("have.length", 0);
-
- cy.log("confirm breaking changes");
- H.datasetEditBar().button("Save changes").click();
- H.modal().button("Save anyway").click();
- cy.wait("@updateCard");
- });
-
- it("should not show a confirmation if there are no breaking changes when updating a SQL model", () => {
- createSqlModelWithDependentSqlQuestions();
- cy.get<number>("@modelId").then(H.visitModel);
- H.openQuestionActions("Edit query definition");
- H.NativeEditor.clear().type("SELECT ID, CATEGORY FROM PRODUCTS");
- H.runNativeQuery();
- H.datasetEditBar().button("Save changes").click();
- cy.wait("@updateCard");
- });
- });
-
describe("metrics", () => {
it("should be able to confirm or cancel breaking changes to a metric", () => {
createMetricWithDependentMbqlQuestionsAndTransforms();
@@ -191,33 +154,15 @@ describe("scenarios > dependencies > dependency checks", () => {
H.NativeEditor.clear().type(queryString);
};
- it("should be able to confirm or cancel breaking changes to a SQL transform after it was run", () => {
+ it("should ignore breaking changes to a SQL transform after it was run", () => {
createSqlTransformWithDependentMbqlQuestions();
cy.log("make breaking changes");
cy.get<number>("@transformId").then(H.visitTransform);
goToEditorAndType('SELECT name FROM "Schema A"."Animals"');
- cy.log("cancel breaking changes");
+ cy.log("no confirmation is shown");
H.DataStudio.Transforms.saveChangesButton().click();
- H.modal().within(() => {
- cy.findByText("Question with fields").should("be.visible");
- cy.findByText("Question without fields").should("not.exist");
- cy.findByText("Base transform").should("not.exist");
- cy.button("Cancel").click();
- });
- cy.get("@updateTransform.all").should("have.length", 0);
-
- goToEditorAndType('SELECT name FROM "Schema A"."Animals"');
-
- cy.log("confirm breaking changes");
- H.DataStudio.Transforms.saveChangesButton().click();
- H.modal().within(() => {
- cy.findByText("Question with fields").should("be.visible");
- cy.findByText("Question without fields").should("not.exist");
- cy.findByText("Base transform").should("not.exist");
- cy.button("Save anyway").click();
- });
cy.wait("@updateTransform");
});
@@ -230,7 +175,7 @@ describe("scenarios > dependencies > dependency checks", () => {
});
it("should be able to confirm or cancel breaking changes to a MBQL transform before it was run", () => {
- createMbqlTransformWithDependentSqlTransforms();
+ createMbqlTransformWithDependentMbqlTransforms();
cy.log("make breaking changes");
cy.get<number>("@transformId").then(H.visitTransform);
@@ -263,7 +208,7 @@ describe("scenarios > dependencies > dependency checks", () => {
});
it("should not show a confirmation if there are no breaking changes when updating a MBQL transform before it was run", () => {
- createMbqlTransformWithDependentSqlTransforms();
+ createMbqlTransformWithDependentMbqlTransforms();
cy.get<number>("@transformId").then(H.visitTransform);
H.DataStudio.Transforms.editDefinition().click();
H.getNotebookStep("data").button("Sort").click();
@@ -272,51 +217,6 @@ describe("scenarios > dependencies > dependency checks", () => {
cy.wait("@updateTransform");
});
});
-
- describe("snippets", () => {
- it("should be able to confirm or cancel breaking changes to a snippet", () => {
- createSnippetWithDependentQuestionsAndTransforms();
-
- cy.log("make breaking changes");
- startEditingSnippet();
- H.modal()
- .findByLabelText("Enter some SQL here so you can reuse it later")
- .clear()
- .type("price = 1");
-
- cy.log("cancel breaking changes");
- H.modal().within(() => {
- cy.button("Save").click();
- cy.findByText("Question with snippet").should("be.visible");
- cy.findByText("Transform with snippet").should("be.visible");
- cy.findByText("ScoreSnippet").should("not.exist");
- cy.button("Cancel").click();
- });
- cy.get("@updateSnippet.all").should("have.length", 0);
-
- cy.log("confirm breaking changes");
- H.modal().within(() => {
- cy.button("Save").click();
- cy.findByText("Question with snippet").should("be.visible");
- cy.findByText("Transform with snippet").should("be.visible");
- cy.findByText("ScoreSnippet").should("not.exist");
- cy.button("Save anyway").click();
- });
- cy.wait("@updateSnippet");
- });
-
- it("should not warn if changes to a snippet are backward compatible", () => {
- createSnippetWithDependentQuestionsAndTransforms();
- startEditingSnippet();
- H.modal().within(() => {
- cy.findByLabelText("Enter some SQL here so you can reuse it later")
- .clear()
- .type("score = 2");
- cy.button("Save").click();
- });
- cy.wait("@updateSnippet");
- });
- });
});
function createMbqlQuestionWithDependentMbqlQuestions() {
@@ -349,53 +249,6 @@ function createMbqlQuestionWithDependentMbqlQuestions() {
});
}
-function createSqlModelWithDependentSqlQuestions() {
- H.createNativeQuestion({
- name: "Base model",
- type: "model",
- native: {
- query: "SELECT ID, TITLE, CATEGORY FROM PRODUCTS",
- "template-tags": {},
- },
- }).then(({ body: card }) => {
- cy.wrap(card.id).as("modelId");
- cy.request("POST", `/api/card/${card.id}/query`);
-
- const tagName = `#${card.id}`;
- H.createNativeQuestion({
- name: "Question with fields",
- native: {
- query: `SELECT ID, CATEGORY FROM {{#${card.id}}}`,
- "template-tags": {
- [tagName]: {
- id: "10422a0f-292d-10a3-fd90-407cc9e3e20e",
- name: tagName,
- "display-name": tagName,
- type: "card",
- "card-id": card.id,
- },
- },
- },
- });
-
- H.createNativeQuestion({
- name: "Question without fields",
- native: {
- query: `SELECT COUNT(*) FROM {{#${card.id}}}`,
- "template-tags": {
- [tagName]: {
- id: "10422a0f-292d-10a3-fd90-407cc9e3e20f",
- name: tagName,
- "display-name": tagName,
- type: "card",
- "card-id": card.id,
- },
- },
- },
- });
- });
-}
-
function createMetricWithDependentMbqlQuestionsAndTransforms() {
H.getTableId({ databaseId: WRITABLE_DB_ID, name: "Animals" }).then(
(tableId) => {
@@ -542,7 +395,7 @@ function createSqlTransformWithDependentMbqlQuestions() {
});
}
-function createMbqlTransformWithDependentSqlTransforms() {
+function createMbqlTransformWithDependentMbqlTransforms() {
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
(tableId) => {
H.createTransform(
@@ -567,120 +420,64 @@ function createMbqlTransformWithDependentSqlTransforms() {
},
{ wrapId: true },
);
+ cy.get<number>("@transformId").then((transformId) =>
+ H.runTransform(transformId),
+ );
+ H.waitForSucceededTransformRuns();
+
+ H.getTableId({ databaseId: WRITABLE_DB_ID, name: "base_transform" }).then(
+ (tableId) => {
+ H.getFieldId({ tableId, name: "name" }).then((fieldId) => {
+ H.createTransform({
+ name: "Name transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "query",
+ query: {
+ "source-table": tableId,
+ fields: [["field", fieldId, null]],
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "name_transform",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ });
+ });
- H.createTransform({
- name: "Name transform",
- source: {
- type: "query",
- query: {
- database: WRITABLE_DB_ID,
- type: "native",
- native: {
- query: "SELECT name FROM base_transform",
- "template-tags": {},
- },
- },
- },
- target: {
- type: "table",
- name: "name_transform",
- schema: "public",
- database: WRITABLE_DB_ID,
- },
- });
-
- H.createTransform({
- name: "Score transform",
- source: {
- type: "query",
- query: {
- database: WRITABLE_DB_ID,
- type: "native",
- native: {
- query: "SELECT score FROM base_transform",
- "template-tags": {},
- },
- },
- },
- target: {
- type: "table",
- name: "score_transform",
- schema: "public",
- database: WRITABLE_DB_ID,
+ H.getFieldId({ tableId, name: "score" }).then((fieldId) => {
+ H.createTransform({
+ name: "Score transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "query",
+ query: {
+ "source-table": tableId,
+ fields: [["field", fieldId, null]],
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "score_transform",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ });
+ });
},
- });
+ );
},
);
}
-function createSnippetWithDependentQuestionsAndTransforms() {
- H.createSnippet({
- name: "ScoreSnippet",
- content: "score = 1",
- }).then(({ body: snippet }) => {
- cy.wrap(snippet.id).as("snippetId");
-
- H.createNativeQuestion({
- name: "Question with snippet",
- database: WRITABLE_DB_ID,
- native: {
- query:
- 'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',
- "template-tags": {
- "snippet:ScoreSnippet": {
- id: "4b77cc1f-ea70-4ef6-84db-58432fce6928",
- name: "snippet:ScoreSnippet",
- "display-name": "snippet:ScoreSnippet",
- type: "snippet",
- "snippet-id": snippet.id,
- "snippet-name": snippet.name,
- },
- },
- },
- });
-
- H.createTransform({
- name: "Transform with snippet",
- source: {
- type: "query",
- query: {
- database: WRITABLE_DB_ID,
- type: "native",
- native: {
- query:
- 'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',
- "template-tags": {
- "snippet:ScoreSnippet": {
- id: "4b77cc1f-ea70-4ef6-84db-58432fce6929",
- name: "snippet:ScoreSnippet",
- "display-name": "snippet:ScoreSnippet",
- type: "snippet",
- "snippet-id": snippet.id,
- "snippet-name": snippet.name,
- },
- },
- },
- },
- },
- target: {
- type: "table",
- name: "transform_with_snippet",
- schema: "public",
- database: WRITABLE_DB_ID,
- },
- });
- });
-}
-
-function startEditingSnippet() {
- H.startNewNativeQuestion();
- cy.findByTestId("native-query-editor-action-buttons").icon("snippet").click();
- cy.findByTestId("sidebar-content").within(() => {
- cy.icon("chevrondown").click({ force: true });
- cy.button(/Edit/).click();
- });
-}
-
function confirmDiscardChanges() {
H.modal().should("have.length", 2).last().button("Discard changes").click();
}
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 445a966f406..1344697a9a2 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -88,7 +88,7 @@
(cond-> #_card
(:result_metadata body) (assoc :result-metadata (:result_metadata body))))
edits {:card [card]}
- breakages (dependencies/errors-from-proposed-edits base-provider edits)]
+ breakages (dependencies/errors-from-proposed-edits edits :base-provider base-provider)]
(broken-cards-response breakages)))
(mr/def ::transform-body
@@ -119,7 +119,7 @@
(cond-> #_transform source (assoc :source source))
(cond-> #_transform target (assoc :target target)))
edits {:transform [transform]}
- breakages (dependencies/errors-from-proposed-edits base-provider edits)]
+ breakages (dependencies/errors-from-proposed-edits edits :base-provider base-provider)]
(broken-cards-response breakages))
;; if this isn't a sql query, just claim it works
{:success true}))
@@ -131,7 +131,7 @@
"Check a proposed edit to a native snippet, and return the cards, etc. which will be broken."
[_route-params
_query-params
- {:keys [id content], snippet-name :name}
+ {:keys [id], snippet-name :name}
:- [:map
[:id {:optional false} ::lib.schema.id/snippet]
[:name {:optional true} native-query-snippets/NativeQuerySnippetName]
@@ -142,14 +142,8 @@
(not= snippet-name (:name original))
(t2/exists? :model/NativeQuerySnippet :name snippet-name))
(throw (ex-info (tru "A snippet with that name already exists. Please pick a different name.")
- {:status-code 400})))
- snippet (cond-> (m/assoc-some original
- :lib/type :metadata/native-query-snippet
- :name snippet-name
- :content content)
- content native-query-snippets/add-template-tags)
- breakages (dependencies/errors-from-proposed-edits {:snippet [snippet]})]
- (broken-cards-response breakages)))
+ {:status-code 400})))]
+ (broken-cards-response {})))
(def ^:private entity-keys
{:table [:name :description :display_name :db_id :db :schema :fields]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
index b47ba045d15..f0eedacd527 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
@@ -6,7 +6,6 @@
[metabase-enterprise.dependencies.analysis :as deps.analysis]
[metabase-enterprise.dependencies.metadata-provider :as deps.provider]
[metabase-enterprise.dependencies.models.dependency :as deps.graph]
- [metabase.graph.core :as graph]
[metabase.lib-be.core :as lib-be]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
@@ -25,6 +24,12 @@
;; TODO: Make this more specific.
[:map-of ::entity-type [:sequential [:map [:id {:optional true} :int]]]])
+(defn- transitive-dependents
+ [& {:keys [graph updated-entities include-native?]}]
+ (if include-native?
+ (deps.graph/transitive-dependents graph updated-entities)
+ (deps.graph/transitive-mbql-dependents graph updated-entities)))
+
(mu/defn- metadata-provider :- ::lib.schema.metadata/metadata-provider
"Constructs a `MetadataProvider` with some pending edits applied.
@@ -33,13 +38,19 @@
Note that if `:result-metadata` is present on an updated Card, it **will be used!** The only case where that is
actually useful is with a native query which has been executed, so the caller has driver metadata to give us. Any
- old, pre-update `:result-metadata` should be dropped from any other cards in `updated-entities`."
+ old, pre-update `:result-metadata` should be dropped from any other cards in `updated-entities`.
+
+ Also note that `include-native?` is only relevant if you don't pass in `dependents`. If you pass in `dependents`,
+ those will be included in the metadata provider whether or not they are native. If you don't pass in `dependents`,
+ `include-native` will determine whether native dependents are included in the metadata provider."
([base-provider :- ::lib.schema.metadata/metadata-provider
updated-entities :- ::updates-map
- & {:keys [graph dependents]}]
+ & {:keys [graph dependents include-native?]}]
;; Reusing the cache with different overrides breaks the caching of [[lib.metadata/card]] calls.
(lib.metadata.protocols/clear-cache! base-provider)
- (let [dependents (or dependents (deps.graph/transitive-dependents graph updated-entities))]
+ (let [dependents (or dependents (transitive-dependents :graph graph
+ :updated-entities updated-entities
+ :include-native? include-native?))]
(deps.provider/override-metadata-provider
{:base-provider base-provider
:updated-entities updated-entities
@@ -96,27 +107,36 @@
The 1-arity groups all the dependents by which Database they are part of, and runs the analysis for each of them.
The output is a map: `{entity-type {id [errors...]}}`; an empty map is returned when there are no errors
- detected."
- ([edits :- ::updates-map]
- (let [all-deps (deps.graph/transitive-dependents edits)
- by-db (group-by-db all-deps)]
- (reduce (fn [errors [db-id deps]]
- (-> (lib-be/application-database-metadata-provider db-id)
- (metadata-provider edits :dependents deps)
- check-query-soundness
- (merge errors)))
- {} by-db)))
-
- ([base-provider :- ::lib.schema.metadata/metadata-provider
- edits :- ::updates-map]
- (errors-from-proposed-edits base-provider nil edits))
-
- ([base-provider :- ::lib.schema.metadata/metadata-provider
- graph :- [:maybe ::graph/graph]
- edits :- ::updates-map]
- (-> base-provider
- (metadata-provider edits :graph graph)
- check-query-soundness)))
+ detected.
+
+ When `include-native?` is false, this function will ignore any entities using native sql and their children."
+ ([edits :- ::updates-map
+ & {:keys [base-provider graph include-native?]}]
+ (let [valid-edits (if include-native?
+ edits
+ (into {}
+ (map (fn [[entity-type instances]]
+ [entity-type (remove #(deps.graph/is-native-entity? entity-type %)
+ instances)]))
+ edits))]
+ (cond
+ (not (some seq (vals valid-edits)))
+ {}
+
+ base-provider
+ (-> base-provider
+ (metadata-provider valid-edits :graph graph :include-native? include-native?)
+ check-query-soundness)
+
+ :else
+ (let [all-deps (transitive-dependents :updated-entities valid-edits :include-native? include-native?)
+ by-db (group-by-db all-deps)]
+ (reduce (fn [errors [db-id deps]]
+ (-> (lib-be/application-database-metadata-provider db-id)
+ (metadata-provider valid-edits :dependents deps)
+ check-query-soundness
+ (merge errors)))
+ {} by-db))))))
#_{:clj-kondo/ignore [:unresolved-namespace]}
(comment
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
index de951a7ab61..4165000a7ac 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
@@ -252,20 +252,6 @@
[_ {:keys [object]}]
(t2/delete! :model/Dependency :from_entity_type :segment :from_entity_id (:id object)))
-(defn- check-dependents! [type object recur-through-transforms?]
- (let [graph (if recur-through-transforms?
- (models.dependency/graph-dependents)
- (models.dependency/filtered-graph-dependents
- nil
- (fn [type-field _id-field]
- [:not= type-field "transform"])))
- children-map (models.dependency/transitive-dependents graph {type [object]})]
- (doseq [[type children] children-map
- :when (deps.findings/supported-entities type)
- instances (partition 50 50 nil children)]
- (-> (t2/select (deps.dependency-types/dependency-type->model type) :id [:in instances])
- deps.findings/analyze-instances!))))
-
;; ### Measures
(derive ::measure-deps :metabase/event)
(derive :event/measure-create ::measure-deps)
@@ -290,6 +276,20 @@
(when (premium-features/has-feature? :dependencies)
(t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))))
+(defn- check-dependents! [type object recur-through-transforms?]
+ (let [graph (if recur-through-transforms?
+ (models.dependency/graph-dependents)
+ (models.dependency/filtered-graph-dependents
+ nil
+ (fn [type-field _id-field]
+ [:not= type-field "transform"])))
+ children-map (models.dependency/transitive-mbql-dependents graph {type [object]})]
+ (doseq [[type children] children-map
+ :when (deps.findings/supported-entities type)
+ instances (partition 50 50 nil children)]
+ (-> (t2/select (deps.dependency-types/dependency-type->model type) :id [:in instances])
+ deps.findings/analyze-instances!))))
+
(derive ::check-card-dependents :metabase/event)
(derive :event/card-create ::check-card-dependents)
(derive :event/card-update ::check-card-dependents)
@@ -297,7 +297,8 @@
(methodical/defmethod events/publish-event! ::check-card-dependents
[_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
+ (when (and (premium-features/has-feature? :dependencies)
+ (not (models.dependency/is-native-entity? :card object)))
(lib-be/with-metadata-provider-cache
(deps.findings/upsert-analysis! object)
(check-dependents! :card object false))))
@@ -309,7 +310,8 @@
(methodical/defmethod events/publish-event! ::check-transform
[_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
+ (when (and (premium-features/has-feature? :dependencies)
+ (not (models.dependency/is-native-entity? :transform object)))
(lib-be/with-metadata-provider-cache
(deps.findings/upsert-analysis! object))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
index 86cdf733801..8e6096ce873 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
@@ -3,6 +3,7 @@
[metabase-enterprise.dependencies.analysis :as deps.analysis]
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
[metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.util.log :as log]
@@ -39,10 +40,13 @@
(when-let [db-id (instance-db-id toucan-instance)]
(let [mp (lib-be/application-database-metadata-provider db-id)
model (t2/model toucan-instance)
- results (try (deps.analysis/check-entity mp (deps.dependency-types/model->dependency-type model) (:id toucan-instance))
- (catch Exception e
- (log/error e "Error analyzing entity")
- [(lib/validation-exception-error (.getMessage e))]))
+ results (if (models.dependency/is-native-entity? (deps.dependency-types/model->dependency-type model)
+ toucan-instance)
+ []
+ (try (deps.analysis/check-entity mp (deps.dependency-types/model->dependency-type model) (:id toucan-instance))
+ (catch Exception e
+ (log/error e "Error analyzing entity")
+ [(lib/validation-exception-error (.getMessage e))])))
success (empty? results)]
(deps.analysis-finding/upsert-analysis! (deps.dependency-types/model->dependency-type model) (:id toucan-instance) success results))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj
index 9e811d371ed..2d9fafcab6c 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj
@@ -35,11 +35,9 @@
[:= type-field "segment"]
[:= type-field "measure"]]))
(graph/filtered-graph (fn [[type id]]
- (or (not= type :card)
- (-> (lib.metadata/card mp id)
- :dataset-query
- lib/any-native-stage?
- not))))
+ (not (and (= type :card)
+ (->> (lib.metadata/card mp id)
+ (models.dependency/is-native-entity? :card))))))
graph/cached-graph))
(mu/defn- dependent-mbql-cards :- [:sequential ::lib.schema.id/card]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
index 0300b256bec..c7b0bcd151d 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
@@ -25,7 +25,7 @@
The background task will re-analyze anything with out-of-date analyses.
This generally shouldn't be rebound in real code, but making it dynamic is convenient for testing"
- 3)
+ 4)
(defn upsert-analysis!
"Given the details of an AnalysisFinding row, upsert the data into the actual db."
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index 0e6c43eff6e..9531b0f82a5 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -1,9 +1,12 @@
(ns metabase-enterprise.dependencies.models.dependency
(:require
[clojure.set :as set]
+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
[metabase.graph.core :as graph]
+ [metabase.lib.core :as lib]
[metabase.models.interface :as mi]
[metabase.util :as u]
+ [metabase.util.malli :as mu]
[methodical.core :as methodical]
[potemkin :as p]
[toucan2.core :as t2]))
@@ -153,8 +156,21 @@
([destination-filter-fn source-filter-fn]
(filtered-graph key-dependents destination-filter-fn source-filter-fn)))
+(defn entities->nodes
+ "Converts a map of entities `{entity-type [{:id 1, ...} ...]}` into a list of nodes `[[entity-type entity-id]]`."
+ [entities-map]
+ (for [[entity-type entities] entities-map
+ entity entities
+ :when (:id entity)]
+ [entity-type (:id entity)]))
+
+(defn group-nodes
+ "Groups a list of nodes `[[entity-type entity-id]]` by their type."
+ [nodes]
+ (u/group-by first second conj #{} nodes))
+
(defn transitive-dependents
- "Given a map of updated entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependents
+ "Given a map of entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependents
as `{entity-type #{4 5 6}}` - that is, a map from downstream entity type to a set of IDs.
Uses the provided `graph`, or defaults to the `:model/Dependency` table in AppDB.
@@ -163,15 +179,61 @@
`MetadataProvider` entities, user input, etc.
**Excludes** the input entities from the list of dependents!"
- ([updated-entities] (transitive-dependents nil updated-entities))
- ([graph updated-entities]
+ ([entities-map] (transitive-dependents nil entities-map))
+ ([graph entities-map]
(let [graph (or graph (graph-dependents))
- starters (for [[entity-type updates] updated-entities
- entity updates
- :when (:id entity)]
- [entity-type (:id entity)])]
+ starters (entities->nodes entities-map)]
(->> (graph/transitive graph starters) ; This returns a flat list.
- (u/group-by first second conj #{})))))
+ group-nodes))))
+
+(mu/defn is-native-entity? :- :boolean
+ "Checks whether an entity involves native sql. `entity` can either be a toucan object or a metadata object."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ entity]
+ (boolean
+ (case entity-type
+ :card (some-> entity
+ ((some-fn :dataset-query :dataset_query))
+ lib/any-native-stage?)
+ :transform (some-> entity
+ :source
+ :query
+ lib/any-native-stage?)
+ :snippet true
+ false)))
+
+(defn- native-lookup-map [children]
+ (let [grouped (-> (graph/all-map-nodes children)
+ group-nodes)]
+ (into {}
+ (mapcat (fn [[node-type ids]]
+ (let [model (deps.dependency-types/dependency-type->model node-type)]
+ (t2/select-fn-vec (fn [entity]
+ [[node-type (:id entity)]
+ (is-native-entity? node-type entity)])
+ model :id [:in ids]))))
+ grouped)))
+
+(defn transitive-mbql-dependents
+ "Equivalent to `transitive-dependents`, except it excludes any native cards/transforms/segments and their children.
+
+ Also, the order is more flexible (though consistent between runs).
+
+ Note that this does not check the passed in entities for native-ness -- the filter is only applied to their
+ transitive children."
+ ([entities-map]
+ (transitive-mbql-dependents nil entities-map))
+ ([graph entities-map]
+ (let [start-nodes (set (entities->nodes entities-map))
+ children (graph/transitive-children-of (or graph (graph-dependents)) (seq start-nodes))
+ native-lookup (native-lookup-map children)]
+ (group-nodes
+ (graph/keep-children (fn [node]
+ (cond
+ (start-nodes node) nil
+ (native-lookup node) ::graph/stop
+ :else node))
+ children)))))
(defn replace-dependencies!
"Replace the dependencies of the entity of type `entity-type` with id `entity-id` with
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/dependencies.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/dependencies.clj
index 57e7d13544d..aa109969b16 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/dependencies.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/dependencies.clj
@@ -57,7 +57,9 @@
updated-metadata (cond-> metadata
source (assoc :source source))
edits {:transform [updated-metadata]}
- breakages (dependencies/errors-from-proposed-edits base-provider edits)]
+ breakages (dependencies/errors-from-proposed-edits edits
+ :base-provider base-provider
+ :include-native? true)]
(format-broken-transforms id breakages))
;; If this is a non-SQL query, we don't do any checks yet, so just return success
{:success true :bad_transforms []})]
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index cc8009daf0f..aa3974c4a31 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -236,6 +236,34 @@
:bad_transforms []}
(update response :bad_cards #(into #{} (map :id) %)))))))))))
+(deftest check-card-skips-native-cards-test
+ (testing "POST /api/ee/dependencies/check_card does not validate native cards"
+ (mt/dataset test-data
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (let [mp (mt/metadata-provider)
+ ;; Create base card querying real orders table
+ base-card (card/create-card! (basic-card) user)
+ ;; Create dependent card that filters on TOTAL
+ dependent-query (lib/native-query mp (str "select * from {{#"
+ (:id base-card)
+ "}} orders where total > 100"))
+ _dependent-card (card/create-card!
+ (card-with-query "Dependent Card filtering on Total" dependent-query)
+ user)
+ ;; Propose changing to products table (doesn't have TOTAL column, breaks downstream)
+ proposed-query (lib/query mp (lib.metadata/table mp (mt/id :products)))
+ proposed-card {:id (:id base-card)
+ :type :question
+ :dataset_query proposed-query
+ :result_metadata nil}
+ response (mt/user-http-request :rasta :post 200 "ee/dependencies/check_card" proposed-card)]
+ (is (=? {:success true
+ :bad_cards []
+ :bad_transforms []}
+ response)))))))))
+
(deftest check-transform-test
(testing "POST /api/ee/dependencies/check_transform"
(mt/with-premium-features #{:dependencies}
@@ -253,8 +281,8 @@
(is (= {:bad_cards [], :bad_transforms [], :success true}
response)))))))
-(deftest check-snippet-content-change-breaks-cards-test
- (testing "POST /api/ee/dependencies/check_snippet detects when snippet content changes break dependent cards"
+(deftest check-snippet-content-change-doest-not-break-cards-test
+ (testing "POST /api/ee/dependencies/check_snippet doesn't catch when a change would break a card, because native query validation is disabled"
(mt/dataset test-data
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}
@@ -269,17 +297,17 @@
:type :snippet
:snippet-name snippet-name
:snippet-id snippet-id}}))
- card (card/create-card! {:name "Card using snippet"
- :dataset_query native-query
- :display :table
- :visualization_settings {}}
- user)
+ _card (card/create-card! {:name "Card using snippet"
+ :dataset_query native-query
+ :display :table
+ :visualization_settings {}}
+ user)
proposed-content "WHERE NONEXISTENT_COLUMN > 100"
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check_snippet"
{:id snippet-id
:content proposed-content})]
- (is (=? {:success false
- :bad_cards [{:id (:id card)}]
+ (is (=? {:success true
+ :bad_cards []
:bad_transforms []}
response)))))))))
@@ -1398,7 +1426,7 @@
:dataset_query (lib/query mp products)}
:model/Card {broken-card-id :id} {:name "Broken Card - brokentest"
:type :question
- :dataset_query (lib/native-query mp "not a query")}]
+ :dataset_query (broken-mbql-query)}]
(while (> (dependencies.findings/analyze-batch! :card 50) 0))
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=brokentest")]
(is (=? {:data [{:id broken-card-id
@@ -1413,7 +1441,7 @@
products (lib.metadata/table mp (mt/id :products))]
(mt/with-temp [:model/Transform {broken-transform-id :id} {:name "Broken Transform - brokentest"
:source {:type :query
- :query (lib/native-query mp "not a query")}
+ :query (broken-mbql-query)}
:target {:schema "PUBLIC"
:name "broken_transform_table"}}
:model/Transform _ {:name "Good Transform - brokentest"
@@ -1431,62 +1459,60 @@
(deftest ^:sequential broken-card-types-test
(testing "GET /api/ee/dependencies/broken - broken models and metrics are filtered by card_types and pagination"
(mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/Card {broken-model-id :id} {:name "A - Broken Model - cardtype"
- :type :model
- :dataset_query (lib/native-query mp "not a query")}
- :model/Card {broken-metric-id :id} {:name "B - Broken Metric - cardtype"
- :type :metric
- :dataset_query (lib/native-query mp "not a query")}]
- (while (> (dependencies.findings/analyze-batch! :card 50) 0))
- (testing "filtering by model only"
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=model&query=cardtype")]
- (is (=? {:data [{:id broken-model-id
- :type "card"
- :data {:name "A - Broken Model - cardtype"
- :type "model"}}]}
- response))))
- (testing "filtering by metric only"
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=metric&query=cardtype")]
- (is (=? {:data [{:id broken-metric-id
- :type "card"
- :data {:name "B - Broken Metric - cardtype"
- :type "metric"}}]}
- response))))
- (testing "filtering by model and metric as the default card types"
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&query=cardtype")]
- (is (=? {:data [{:id broken-model-id
- :type "card"
- :data {:name "A - Broken Model - cardtype"
- :type "model"}}
- {:id broken-metric-id
- :type "card"
- :data {:name "B - Broken Metric - cardtype"
- :type "metric"}}]}
- response)))))))))
+ (mt/with-temp [:model/Card {broken-model-id :id} {:name "A - Broken Model - cardtype"
+ :type :model
+ :dataset_query (broken-mbql-query)}
+ :model/Card {broken-metric-id :id} {:name "B - Broken Metric - cardtype"
+ :type :metric
+ :dataset_query (broken-mbql-query)}]
+ (while (> (dependencies.findings/analyze-batch! :card 50) 0))
+ (testing "filtering by model only"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=model&query=cardtype")]
+ (is (=? {:data [{:id broken-model-id
+ :type "card"
+ :data {:name "A - Broken Model - cardtype"
+ :type "model"}}]}
+ response))))
+ (testing "filtering by metric only"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=metric&query=cardtype")]
+ (is (=? {:data [{:id broken-metric-id
+ :type "card"
+ :data {:name "B - Broken Metric - cardtype"
+ :type "metric"}}]}
+ response))))
+ (testing "filtering by model and metric as the default card types"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&query=cardtype")]
+ (is (=? {:data [{:id broken-model-id
+ :type "card"
+ :data {:name "A - Broken Model - cardtype"
+ :type "model"}}
+ {:id broken-metric-id
+ :type "card"
+ :data {:name "B - Broken Metric - cardtype"
+ :type "metric"}}]}
+ response))))))))
(deftest ^:sequential broken-archived-card-test
(testing "GET /api/ee/dependencies/graph/broken with archived parameter"
(mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/Card {broken-card-id :id} {:name "Broken Card - archivedbrokentestcard"
- :type :question
- :dataset_query (lib/native-query mp "not a query")}
- :model/Card {archived-broken-card-id :id} {:name "Archived Broken Card - archivedbrokentestcard"
- :type :question
- :archived true
- :dataset_query (lib/native-query mp "not a query")}]
- (while (> (dependencies.findings/analyze-batch! :card 50) 0))
- (testing "archived=false (default) excludes archived broken card"
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard")
- card-ids (set (map :id (:data response)))]
- (is (contains? card-ids broken-card-id))
- (is (not (contains? card-ids archived-broken-card-id)))))
- (testing "archived=true includes archived broken card"
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard&archived=true")
- card-ids (set (map :id (:data response)))]
- (is (contains? card-ids broken-card-id))
- (is (contains? card-ids archived-broken-card-id)))))))))
+ (mt/with-temp [:model/Card {broken-card-id :id} {:name "Broken Card - archivedbrokentestcard"
+ :type :question
+ :dataset_query (broken-mbql-query)}
+ :model/Card {archived-broken-card-id :id} {:name "Archived Broken Card - archivedbrokentestcard"
+ :type :question
+ :archived true
+ :dataset_query (broken-mbql-query)}]
+ (while (> (dependencies.findings/analyze-batch! :card 50) 0))
+ (testing "archived=false (default) excludes archived broken card"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard")
+ card-ids (set (map :id (:data response)))]
+ (is (contains? card-ids broken-card-id))
+ (is (not (contains? card-ids archived-broken-card-id)))))
+ (testing "archived=true includes archived broken card"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard&archived=true")
+ card-ids (set (map :id (:data response)))]
+ (is (contains? card-ids broken-card-id))
+ (is (contains? card-ids archived-broken-card-id))))))))
(deftest ^:sequential broken-archived-segment-test
(testing "GET /api/ee/dependencies/graph/broken with archived parameter for segments"
@@ -1603,50 +1629,48 @@
(testing "GET /api/ee/dependencies/graph/broken with include_personal_collections parameter"
(mt/with-premium-features #{:dependencies}
(binding [collection/*allow-deleting-personal-collections* true]
- (let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/User {user-id :id} {}
- :model/Collection {personal-coll-id :id} {:personal_owner_id user-id
- :name "Test Personal Collection"}
- :model/Card {broken-in-personal :id} {:name "Broken Card in Personal - personalcollbrokentest"
- :type :question
- :collection_id personal-coll-id
- :dataset_query (lib/native-query mp "not a query")}
- :model/Card {broken-regular :id} {:name "Broken Card Regular - personalcollbrokentest"
- :type :question
- :dataset_query (lib/native-query mp "not a query")}]
- (while (> (dependencies.findings/analyze-batch! :card 50) 0))
- (testing "include_personal_collections=false (default) excludes broken cards in personal collections"
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=personalcollbrokentest")
- card-ids (set (map :id (:data response)))]
- (is (not (contains? card-ids broken-in-personal)))
- (is (contains? card-ids broken-regular))))
- (testing "include_personal_collections=true includes broken cards in personal collections"
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=personalcollbrokentest&include_personal_collections=true")
- card-ids (set (map :id (:data response)))]
- (is (contains? card-ids broken-in-personal))
- (is (contains? card-ids broken-regular))))))))))
+ (mt/with-temp [:model/User {user-id :id} {}
+ :model/Collection {personal-coll-id :id} {:personal_owner_id user-id
+ :name "Test Personal Collection"}
+ :model/Card {broken-in-personal :id} {:name "Broken Card in Personal - personalcollbrokentest"
+ :type :question
+ :collection_id personal-coll-id
+ :dataset_query (broken-mbql-query)}
+ :model/Card {broken-regular :id} {:name "Broken Card Regular - personalcollbrokentest"
+ :type :question
+ :dataset_query (broken-mbql-query)}]
+ (while (> (dependencies.findings/analyze-batch! :card 50) 0))
+ (testing "include_personal_collections=false (default) excludes broken cards in personal collections"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=personalcollbrokentest")
+ card-ids (set (map :id (:data response)))]
+ (is (not (contains? card-ids broken-in-personal)))
+ (is (contains? card-ids broken-regular))))
+ (testing "include_personal_collections=true includes broken cards in personal collections"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=personalcollbrokentest&include_personal_collections=true")
+ card-ids (set (map :id (:data response)))]
+ (is (contains? card-ids broken-in-personal))
+ (is (contains? card-ids broken-regular)))))))))
(deftest ^:sequential broken-pagination-test
(testing "GET /api/ee/dependencies/broken - should paginate results"
(mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/Card {card1-id :id} {:name "Card 1 - brokentest"
- :type :question
- :dataset_query (lib/native-query mp "not a query")}
- :model/Card {card2-id :id} {:name "Card 2 - brokentest"
- :type :question
- :dataset_query (lib/native-query mp "not a query")}]
- (while (> (dependencies.findings/analyze-batch! :card 50) 0))
- (is (=? {:data [{:id card1-id}]
- :total 2
- :offset 0
- :limit 1}
- (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=brokentest&offset=0&limit=1")))
- (is (=? {:data [{:id card2-id}]
- :total 2
- :offset 1
- :limit 1}
- (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=brokentest&offset=1&limit=1"))))))))
+ (mt/with-temp [:model/Card {card1-id :id} {:name "Card 1 - brokentest"
+ :type :question
+ :dataset_query (broken-mbql-query)}
+ :model/Card {card2-id :id} {:name "Card 2 - brokentest"
+ :type :question
+ :dataset_query (broken-mbql-query)}]
+ (while (> (dependencies.findings/analyze-batch! :card 50) 0))
+ (is (=? {:data [{:id card1-id}]
+ :total 2
+ :offset 0
+ :limit 1}
+ (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=brokentest&offset=0&limit=1")))
+ (is (=? {:data [{:id card2-id}]
+ :total 2
+ :offset 1
+ :limit 1}
+ (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=brokentest&offset=1&limit=1")))))))
(deftest ^:sequential unreferenced-sort-by-name-test
(testing "GET /api/ee/dependencies/graph/unreferenced - sorting by name"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
index d08c7d5b835..aa1155e0f99 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
@@ -136,12 +136,15 @@
{transformed-card-id :id} :mbql-transform-consumer} (testbed)]
(testing "a column that no longer exists will cause errors when referenced"
(let [card' (-> mbql-base
- ;; ridiculous
- (update :dataset-query lib/update-query-stage -1
- update-in [:expressions 0]
- lib/update-options assoc :lib/expression-name "Sales Taxes")
+ (assoc :dataset-query
+ (-> (lib/query provider (meta/table-metadata :orders))
+ ;; ridiculous
+ (lib/expression "Sales Taxes" (lib// (meta/field-metadata :orders :tax)
+ (meta/field-metadata :orders :subtotal)))))
(dissoc :result-metadata))
- errors (dependencies/errors-from-proposed-edits provider graph {:card [card']})]
+ errors (dependencies/errors-from-proposed-edits {:card [card']}
+ :base-provider provider
+ :graph graph)]
(is (=? {:card {downstream-card-id #{(lib/missing-column-error "Tax Rate")}
transformed-card-id #{(lib/missing-column-error "Tax Rate")}}}
errors))
@@ -149,13 +152,14 @@
(is (= #{downstream-card-id transformed-card-id} (set (keys (:card errors)))))))
(testing "changing something unrelated will cause no errors"
(let [card' (-> mbql-base
- (assoc-in [:dataset-query :query :filter]
- [:> [:field (meta/id :orders :quantity) nil] 100])
+ (update :dataset-query lib/filter (lib/> (meta/field-metadata :orders :quantity)
+ 100))
(dissoc :result-metadata))]
- (is (= {} (dependencies/errors-from-proposed-edits provider graph {:card [card']}))))))))
-
+ (is (= {} (dependencies/errors-from-proposed-edits {:card [card']}
+ :base-provider provider
+ :graph graph))))))))
(deftest ^:parallel sql-snippet->card->transform->cards-test
- (testing "changing a snippet correctly finds downstream errors"
+ (testing "changing a snippet correctly finds downstream errors when asked"
(let [{:keys [provider graph sql-transform snippet-inner]
{direct-sql-card-id :id} :sql-base
{transformed-sql-card-id :id} :sql-transform-sql-consumer
@@ -164,7 +168,10 @@
(let [snippet' (assoc snippet-inner
:content "nonexistent_table"
:template-tags {})
- errors (dependencies/errors-from-proposed-edits provider graph {:snippet [snippet']})]
+ errors (dependencies/errors-from-proposed-edits {:snippet [snippet']}
+ :base-provider provider
+ :graph graph
+ :include-native? true)]
;; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL
;; queries that consume the transform's table.
(is (=? {:card {direct-sql-card-id #{(lib/missing-table-alias-error "NONEXISTENT_TABLE")}
@@ -178,3 +185,16 @@
(set (keys (:card errors)))))
(is (= #{(:id sql-transform)}
(set (keys (:transform errors))))))))))
+
+(deftest ^:parallel sql-snippet-without-including-native-test
+ (testing "changing a snippet correctly ignores downstream errors"
+ (let [{:keys [provider graph snippet-inner]} (testbed)]
+ (testing "when breaking the inner snippet with a nonexistent table"
+ (let [snippet' (assoc snippet-inner
+ :content "nonexistent_table"
+ :template-tags {})
+ errors (dependencies/errors-from-proposed-edits {:snippet [snippet']}
+ :base-provider provider
+ :graph graph)]
+ ;; Because we are changing a snippet, don't check anything downstream
+ (is (= {} errors)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index 23807e0df5c..93a4f365713 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -601,7 +601,9 @@
Format is `{entity-type {entity-id analysis-version}}`.
- Use -1 for analysis-version if you want to check that an analysis exists without asserting a specific version."
+ Use -1 for analysis-version if you want to check that an analysis exists without asserting a specific version.
+
+ Use nil for analysis-version if you want to check that no analysis exists."
[spec]
(doseq [[entity-type ids-and-versions] spec]
(let [analyses-map (->> (t2/select [:model/AnalysisFinding :analyzed_entity_id :analysis_version])
@@ -662,6 +664,21 @@
child-card-id new-version
other-card-id old-version}}))))))))
+(deftest ^:sequential card-update-ignores-native-cards-test
+ (run-with-dependencies-setup
+ (fn [mp]
+ (testing "Native card updates should not trigger analysis"
+ (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/native-query mp "select * from products")}
+ :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id child-card-id
+ :to_entity_type :card
+ :to_entity_id parent-card-id}]
+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ (assert-has-analyses
+ {:card {parent-card-id nil
+ child-card-id nil}}))))))
+
(deftest ^:sequential card-update-stops-on-transforms-test
(run-with-dependencies-setup
(fn [mp]
@@ -767,6 +784,20 @@
other-card-id old-version}
:transform {transform-id new-version}})))))))))
+(deftest ^:sequential transform-update-ignores-native-transforms-test
+ (run-with-dependencies-setup
+ (fn [mp]
+ (testing "Native transform updates should not trigger analysis"
+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query
+ :query (lib/native-query mp "select * from products")}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
+ (assert-has-analyses
+ {:transform {transform-id nil}}))))))
+
(deftest ^:sequential transform-run-works-with-no-analyses-test
(run-with-dependencies-setup
(fn [mp]
@@ -836,6 +867,32 @@
other-card-id old-version}
:transform {transform-id old-version}}))))))))
+(deftest ^:sequential transform-run-triggers-analysis-for-native-transforms-test
+ (run-with-dependencies-setup
+ (fn [mp]
+ (testing "Native transform runs should still trigger analysis"
+ (let [products-id (mt/id :products)
+ products (lib.metadata/table mp products-id)]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Transform {transform-id :id} {:source {:type :query
+ :query (lib/native-query mp "select * from products")}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id card-id
+ :to_entity_type :transform
+ :to_entity_id transform-id}]
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "public"
+ :output-table "sample"
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (assert-has-analyses
+ {:card {card-id -1}})))))))
+
(deftest ^:sequential segment-update-works-with-no-analyses-test
(run-with-dependencies-setup
(fn [mp]
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
index 042e1bcad1a..314c3971bb6 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
@@ -49,3 +49,16 @@
(binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
;; can't check the actual cards analyzed here because this could be any starting card in the data
(is (= 2 (deps.findings/analyze-batch! :card 2))))))
+
+(deftest ^:sequential does-not-analyze-native-entities-test
+ (testing "assumes native queries are always fine"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query mp "utter nonsense")}]
+ (is (= 1 (deps.findings/analyze-batch! :card 1)))
+ (is (= [models.analysis-finding/*current-analysis-finding-version* true]
+ (t2/select-one-fn (juxt :analysis_version :result)
+ :model/AnalysisFinding
+ :analyzed_entity_id card-id
+ :analyzed_entity_type :card))))))))
diff --git a/src/metabase/graph/core.cljc b/src/metabase/graph/core.cljc
index 17dec2f2be8..d1bbd08fa4b 100644
--- a/src/metabase/graph/core.cljc
+++ b/src/metabase/graph/core.cljc
@@ -123,6 +123,14 @@
(recur new-traverse new-child-map)
new-child-map))))
+(mu/defn all-map-nodes :- [:sequential :any]
+ "Returns every node mentioned in `children`."
+ [children :- ::child-map]
+ (-> (into (set (keys children))
+ cat
+ (vals children))
+ sort))
+
(mu/defn keep-children :- [:sequential :any]
"Iterates through a child map, calls `f` for each node, and returns the non-nil results as a list.
@@ -132,10 +140,7 @@
recurse down the current node's children."
[f :- [:-> ::node :any]
children :- ::child-map]
- (let [all-nodes (-> (into (set (keys children))
- cat
- (vals children))
- sort)
+ (let [all-nodes (all-map-nodes children)
full-parent-map (->> children
(mapcat (fn [[parent current-children]]
(map (fn [child]