Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Dependencies tracking (#63248)

William85fc7cb3fd99
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index d320ccc7291..5438d7b3c31 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -428,6 +428,11 @@
:api #{metabase.database-routing.core}
:uses #{premium-features}}
+ dependencies
+ {:team "Querying"
+ :api #{metabase.premium-features.core}
+ :uses #{premium-features}}
+
driver
{:team "Drivers"
:api #{metabase.driver
@@ -556,6 +561,11 @@
models
util}}
+ graph
+ {:team "Querying"
+ :api #{metabase.graph.core}
+ :uses #{util}}
+
indexed-entities
{:team "UX West"
:api #{metabase.indexed-entities.api
@@ -725,6 +735,7 @@
metabase.native-query-snippets.core}
:uses #{api
collections
+ events
lib
models
permissions
@@ -1716,6 +1727,29 @@
warehouse-schema
warehouses}}
+ enterprise/dependencies
+ {:team "Querying"
+ :api #{metabase-enterprise.dependencies.api
+ metabase-enterprise.dependencies.core
+ metabase-enterprise.dependencies.init}
+ :uses #{analyze
+ api
+ app-db
+ collections
+ config
+ driver
+ events
+ graph
+ lib
+ lib-be
+ models
+ native-query-snippets
+ premium-features
+ queries
+ query-processor
+ task
+ util}}
+
enterprise/documents
{:team "UX West"
:api #{metabase-enterprise.documents.api
diff --git a/deps.edn b/deps.edn
index b82eabaf744..6577a696963 100644
--- a/deps.edn
+++ b/deps.edn
@@ -90,7 +90,7 @@
:git/sha "4998a1fdd9d3713d7034d04509e9212204773bf2"
:git/url "https://github.com/eerohele/pp"}
;; TODO: replace back to normal artifact
- io.github.metabase/macaw {:mvn/version "0.2.27"} ; Parse native SQL queries
+ io.github.metabase/macaw {:mvn/version "0.2.28"} ; Parse native SQL queries
io.github.resilience4j/resilience4j-retry ^:antq/exclude ; 2.x do not support Java 8
{:mvn/version "1.7.1"} ; Support for retrying operations
io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collector
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 36bd741b38a..cc78d390ed8 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -29,6 +29,8 @@ title: Driver interface changelog
- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase
type (from the type hierarchy) as a HoneySQL spec. This method handles general Metabase base types.
+- Added driver multimethods driver/native-result-metadata, driver/validate-native-query-fields, driver.sql/resolve-field, driver.sql.normalize-unquoted-name, driver.sql.normalize/reserved-literal, driver.sql.references/find-used-fields, driver.sql.references/find-returned-fields, and driver.sql.references/field-references-impl for use with the :dependencies/native feature.
+
- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources
into existing tables. This multimethod dispatches on both the driver and the data source type
(`:rows` or `:jsonl-file`). It allows drivers to optimize based on the data source type and returns the number
diff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.ts
new file mode 100644
index 00000000000..ee0beb942c5
--- /dev/null
+++ b/e2e/support/helpers/e2e-transform-helpers.ts
@@ -0,0 +1,37 @@
+import type {
+ ListTransformRunsResponse,
+ TransformId,
+ TransformRun,
+} from "metabase-types/api";
+
+export function visitTransform(transformId: TransformId) {
+ cy.visit(`/admin/transforms/${transformId}`);
+}
+
+const WAIT_TIMEOUT = 10000;
+const WAIT_INTERVAL = 100;
+
+export function waitForTransformRuns(
+ filter: (runs: TransformRun[]) => boolean,
+ timeout = WAIT_TIMEOUT,
+): Cypress.Chainable {
+ return cy
+ .request<ListTransformRunsResponse>("GET", "/api/ee/transform/run")
+ .then((response) => {
+ if (filter(response.body.data)) {
+ return cy.wrap(response);
+ } else if (timeout > 0) {
+ cy.wait(WAIT_INTERVAL);
+ return waitForTransformRuns(filter, timeout - WAIT_INTERVAL);
+ } else {
+ throw new Error("Run retry timeout");
+ }
+ });
+}
+
+export function waitForSucceededTransformRuns() {
+ waitForTransformRuns(
+ (runs) =>
+ runs.length > 0 && runs.every((run) => run.status === "succeeded"),
+ );
+}
diff --git a/e2e/support/helpers/index.ts b/e2e/support/helpers/index.ts
index 36e00b8db54..6bea5ad97a7 100644
--- a/e2e/support/helpers/index.ts
+++ b/e2e/support/helpers/index.ts
@@ -47,6 +47,7 @@ export * from "./e2e-slack-helpers";
export * from "./e2e-snowplow-helpers";
export * from "./e2e-table-metadata-helpers";
export * from "./e2e-token-helpers";
+export * from "./e2e-transform-helpers";
export * from "./e2e-ui-elements-helpers";
export * from "./e2e-ui-elements-overflow-helpers";
export * from "./e2e-upload-helpers";
diff --git a/e2e/test/scenarios/admin/transforms.cy.spec.ts b/e2e/test/scenarios/admin/transforms.cy.spec.ts
index 60ef560d58f..1c3fbe438b8 100644
--- a/e2e/test/scenarios/admin/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/admin/transforms.cy.spec.ts
@@ -6,7 +6,6 @@ import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
import type {
CardType,
- ListTransformRunsResponse,
PythonTransformTableAliases,
TransformTagId,
} from "metabase-types/api";
@@ -1944,7 +1943,7 @@ describe("scenarios > admin > transforms > jobs", () => {
tag_ids: [tag.id],
});
});
- waitForRuns();
+ H.waitForSucceededTransformRuns();
visitRunListPage();
getContentTable().within(() => {
cy.findAllByText("MBQL transform").should("have.length.gte", 1);
@@ -2961,24 +2960,6 @@ function visitTableQuestion({
);
}
-const WAIT_TIMEOUT = 10000;
-const WAIT_INTERVAL = 100;
-
-function waitForRuns(timeout = WAIT_TIMEOUT): Cypress.Chainable {
- return cy
- .request<ListTransformRunsResponse>("GET", "/api/ee/transform/run")
- .then((response) => {
- if (response.body.data.length > 0) {
- return cy.wrap(response);
- } else if (timeout > 0) {
- cy.wait(WAIT_INTERVAL);
- return waitForRuns(timeout - WAIT_INTERVAL);
- } else {
- throw new Error("Run retry timeout");
- }
- });
-}
-
function assertTableDoesNotExistError({
targetTable = TARGET_TABLE,
targetSchema = TARGET_SCHEMA,
diff --git a/e2e/test/scenarios/dependencies/dependencies.cy.spec.ts b/e2e/test/scenarios/dependencies/dependencies.cy.spec.ts
new file mode 100644
index 00000000000..8bc92b89c54
--- /dev/null
+++ b/e2e/test/scenarios/dependencies/dependencies.cy.spec.ts
@@ -0,0 +1,678 @@
+const { H } = cy;
+
+import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
+import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
+import { ORDERS_DASHBOARD_ID } from "e2e/support/cypress_sample_instance_data";
+
+const { PRODUCTS_ID } = SAMPLE_DATABASE;
+
+H.describeWithSnowplowEE("documents", () => {
+ beforeEach(() => {
+ H.restore("postgres-writable");
+ H.resetTestTable({ type: "postgres", table: "many_schemas" });
+ cy.signInAsAdmin();
+ H.activateToken("bleeding-edge");
+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: "Animals" });
+ H.resetSnowplow();
+
+ cy.intercept("PUT", "/api/card/*").as("updateCard");
+ cy.intercept("PUT", "/api/native-query-snippet/*").as("updateSnippet");
+ cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");
+ });
+
+ afterEach(() => {
+ H.expectNoBadSnowplowEvents();
+ });
+
+ describe("questions", () => {
+ it("should be able to confirm or cancel breaking changes to a MBQL question", () => {
+ createMbqlQuestionWithDependentMbqlQuestions();
+
+ cy.log("make breaking changes");
+ H.visitQuestion("@questionId");
+ H.openNotebook();
+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+
+ cy.log("cancel breaking changes");
+ H.queryBuilderHeader().button("Save").click();
+ H.modal().button("Save").click();
+ H.modal().within(() => {
+ cy.findByText("Question with fields").should("be.visible");
+ cy.findByText("Question without fields").should("not.exist");
+ cy.findByText("Base question").should("not.exist");
+ cy.button("Cancel").click();
+ });
+ cy.get("@updateCard.all").should("have.length", 0);
+
+ cy.log("confirm breaking changes");
+ H.queryBuilderHeader().button("Save").click();
+ H.modal().button("Save").click();
+ H.modal().button("Save anyway").click();
+ cy.wait("@updateCard");
+ });
+
+ it("should be able to navigate to affected questions or their collection", () => {
+ createMbqlQuestionWithDependentMbqlQuestions();
+
+ cy.log("check that we can navigate to the broken question");
+ H.visitQuestion("@questionId");
+ H.openNotebook();
+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+ H.queryBuilderHeader().button("Save").click();
+ H.modal().button("Save").click();
+ H.modal().findByText("Question with fields").click();
+ confirmDiscardChanges();
+ H.queryBuilderHeader()
+ .findByDisplayValue("Question with fields")
+ .should("be.visible");
+
+ cy.log("check that we can navigate to the collection of that question");
+ cy.go("back");
+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+ H.queryBuilderHeader().button("Save").click();
+ H.modal().button("Save").click();
+ H.modal().within(() => {
+ cy.findByText("Question with fields").should("be.visible");
+ cy.findByText("Our analytics").click();
+ });
+ confirmDiscardChanges();
+ H.collectionTable().should("be.visible");
+
+ cy.log("check that we can navigate to the dashboard of that question");
+ cy.go("back");
+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+ H.queryBuilderHeader().button("Save").click();
+ H.modal().button("Save").click();
+ H.modal().within(() => {
+ cy.findByText("Question with fields").should("be.visible");
+ cy.findByText("Orders in a dashboard").click();
+ });
+ confirmDiscardChanges();
+ H.dashboardHeader().should("be.visible");
+ });
+
+ it("should not show a confirmation if there are no breaking changes when updating a MBQL question", () => {
+ createMbqlQuestionWithDependentMbqlQuestions();
+ H.visitQuestion("@questionId");
+ H.openNotebook();
+ H.getNotebookStep("expression").findByText("Expr").click();
+ H.CustomExpressionEditor.clear().type("2 + 2");
+ H.popover().button("Update").click();
+ H.queryBuilderHeader().button("Save").click();
+ H.modal().button("Save").click();
+ cy.wait("@updateCard");
+ });
+ });
+
+ describe("models", () => {
+ it("should be able to confirm or cancel breaking changes to a SQL model", () => {
+ createSqlModelWithDependentSqlQuestions();
+
+ cy.log("make breaking changes");
+ cy.get<number>("@modelId").then(H.visitModel);
+ H.openQuestionActions("Edit query definition");
+ H.NativeEditor.clear().type("SELECT ID, TITLE FROM PRODUCTS");
+ H.runNativeQuery();
+
+ cy.log("cancel breaking changes");
+ H.datasetEditBar().button("Save changes").click();
+ H.modal().within(() => {
+ cy.findByText("Question with fields").should("be.visible");
+ cy.findByText("Question without fields").should("not.exist");
+ cy.findByText("Base model").should("not.exist");
+ cy.button("Cancel").click();
+ });
+ cy.get("@updateCard.all").should("have.length", 0);
+
+ cy.log("confirm breaking changes");
+ H.datasetEditBar().button("Save changes").click();
+ H.modal().button("Save anyway").click();
+ cy.wait("@updateCard");
+ });
+
+ it("should not show a confirmation if there are no breaking changes when updating a SQL model", () => {
+ createSqlModelWithDependentSqlQuestions();
+ cy.get<number>("@modelId").then(H.visitModel);
+ H.openQuestionActions("Edit query definition");
+ H.NativeEditor.clear().type("SELECT ID, CATEGORY FROM PRODUCTS");
+ H.runNativeQuery();
+ H.datasetEditBar().button("Save changes").click();
+ cy.wait("@updateCard");
+ });
+ });
+
+ describe("metrics", () => {
+ it("should be able to confirm or cancel breaking changes to a metric", () => {
+ createMetricWithDependentMbqlQuestionsAndTransforms();
+
+ cy.log("make breaking changes");
+ cy.get<number>("@metricId").then(H.visitMetric);
+ H.openQuestionActions("Edit metric definition");
+ H.getNotebookStep("summarize").findByText("Min of Score").click();
+ H.popover().within(() => {
+ cy.icon("chevronleft").click();
+ cy.findByText("Maximum of ...").click();
+ cy.findByText("Score").click();
+ });
+
+ cy.log("cancel breaking changes");
+ cy.findByTestId("edit-bar").button("Save changes").click();
+ H.modal().within(() => {
+ cy.findByText("Question with 2 stages").should("be.visible");
+ cy.findByText("Transform with 2 stages").should("be.visible");
+ cy.findByText("Question with 1 stage").should("not.exist");
+ cy.findByText("Transform with 1 stage").should("not.exist");
+ cy.findByText("Base metric").should("not.exist");
+ cy.button("Cancel").click();
+ });
+ cy.get("@updateCard.all").should("have.length", 0);
+
+ cy.log("confirm breaking changes");
+ cy.findByTestId("edit-bar").button("Save changes").click();
+ H.modal().button("Save anyway").click();
+ cy.wait("@updateCard");
+ });
+
+ it("should not show a warning when a change to a metric is backward-compatible with existing content", () => {
+ createMetricWithDependentMbqlQuestionsAndTransforms();
+ cy.get<number>("@metricId").then(H.visitMetric);
+ H.openQuestionActions("Edit metric definition");
+ H.getNotebookStep("summarize").findByText("Min of Score").click();
+ H.popover().findByText("Name").click();
+ cy.findByTestId("edit-bar").button("Save changes").click();
+ cy.wait("@updateCard");
+ });
+ });
+
+ describe("transforms", () => {
+ it("should be able to confirm or cancel breaking changes to a SQL transform after it was run", () => {
+ createSqlTransformWithDependentMbqlQuestions();
+
+ cy.log("make breaking changes");
+ cy.get<number>("@transformId").then(H.visitTransform);
+ cy.findByTestId("transform-page").findByText("Edit query").click();
+ H.NativeEditor.clear().type('SELECT name FROM "Schema A"."Animals"');
+
+ cy.log("cancel breaking changes");
+ cy.findByTestId("transform-query-editor").button("Save changes").click();
+ H.modal().within(() => {
+ cy.findByText("Question with fields").should("be.visible");
+ cy.findByText("Question without fields").should("not.exist");
+ cy.findByText("Base transform").should("not.exist");
+ cy.button("Cancel").click();
+ });
+ cy.get("@updateTransform.all").should("have.length", 0);
+
+ cy.log("confirm breaking changes");
+ cy.findByTestId("transform-query-editor").button("Save changes").click();
+ H.modal().within(() => {
+ cy.findByText("Question with fields").should("be.visible");
+ cy.findByText("Question without fields").should("not.exist");
+ cy.findByText("Base transform").should("not.exist");
+ cy.button("Save anyway").click();
+ });
+ cy.wait("@updateTransform");
+ });
+
+ it("should not show a confirmation if there are no breaking changes when updating a SQL transform after it was run", () => {
+ createSqlTransformWithDependentMbqlQuestions();
+ cy.get<number>("@transformId").then(H.visitTransform);
+ cy.findByTestId("transform-page").findByText("Edit query").click();
+ H.NativeEditor.clear().type(
+ 'SELECT score, name FROM "Schema A"."Animals"',
+ );
+ cy.findByTestId("transform-query-editor").button("Save changes").click();
+ cy.wait("@updateTransform");
+ });
+
+ it("should be able to confirm or cancel breaking changes to a MBQL transform before it was run", () => {
+ createMbqlTransformWithDependentSqlTransforms();
+
+ cy.log("make breaking changes");
+ cy.get<number>("@transformId").then(H.visitTransform);
+ cy.findByTestId("transform-page").findByText("Edit query").click();
+ H.getNotebookStep("data").findByLabelText("Pick columns").click();
+ H.popover().findByLabelText("Score").click();
+
+ cy.log("cancel breaking changes");
+ cy.findByTestId("transform-query-editor").button("Save changes").click();
+ H.modal().within(() => {
+ cy.findByText("Score transform").should("be.visible");
+ cy.findByText("Name transform").should("not.exist");
+ cy.findByText("Base transform").should("not.exist");
+ cy.button("Cancel").click();
+ });
+ cy.get("@updateTransform.all").should("have.length", 0);
+
+ cy.log("confirm breaking changes");
+ cy.findByTestId("transform-query-editor").button("Save changes").click();
+ H.modal().within(() => {
+ cy.findByText("Score transform").should("be.visible");
+ cy.findByText("Name transform").should("not.exist");
+ cy.button("Save anyway").click();
+ });
+ cy.wait("@updateTransform");
+ });
+
+ it("should not show a confirmation if there are no breaking changes when updating a MBQL transform before it was run", () => {
+ createMbqlTransformWithDependentSqlTransforms();
+ cy.get<number>("@transformId").then(H.visitTransform);
+ cy.findByTestId("transform-page").findByText("Edit query").click();
+ H.getNotebookStep("data").button("Sort").click();
+ H.popover().findByText("Score").click();
+ cy.findByTestId("transform-query-editor").button("Save changes").click();
+ cy.wait("@updateTransform");
+ });
+ });
+
+ describe("snippets", () => {
+ it("should be able to confirm or cancel breaking changes to a snippet", () => {
+ createSnippetWithDependentQuestionsAndTransforms();
+
+ cy.log("make breaking changes");
+ startEditingSnippet();
+ H.modal()
+ .findByLabelText("Enter some SQL here so you can reuse it later")
+ .clear()
+ .type("price = 1");
+
+ cy.log("cancel breaking changes");
+ H.modal().within(() => {
+ cy.button("Save").click();
+ cy.findByText("Question with snippet").should("be.visible");
+ cy.findByText("Transform with snippet").should("be.visible");
+ cy.findByText("ScoreSnippet").should("not.exist");
+ cy.button("Cancel").click();
+ });
+ cy.get("@updateSnippet.all").should("have.length", 0);
+
+ cy.log("confirm breaking changes");
+ H.modal().within(() => {
+ cy.button("Save").click();
+ cy.findByText("Question with snippet").should("be.visible");
+ cy.findByText("Transform with snippet").should("be.visible");
+ cy.findByText("ScoreSnippet").should("not.exist");
+ cy.button("Save anyway").click();
+ });
+ cy.wait("@updateSnippet");
+ });
+
+ it("should not warn if changes to a snippet are backward compatible", () => {
+ createSnippetWithDependentQuestionsAndTransforms();
+ startEditingSnippet();
+ H.modal().within(() => {
+ cy.findByLabelText("Enter some SQL here so you can reuse it later")
+ .clear()
+ .type("score = 2");
+ cy.button("Save").click();
+ });
+ cy.wait("@updateSnippet");
+ });
+ });
+});
+
+function createMbqlQuestionWithDependentMbqlQuestions() {
+ H.createQuestion({
+ name: "Base question",
+ query: {
+ "source-table": PRODUCTS_ID,
+ expressions: {
+ Expr: ["+", 1, 1],
+ },
+ },
+ }).then(({ body: card }) => {
+ cy.wrap(card.id).as("questionId");
+
+ H.createQuestion({
+ name: "Question with fields",
+ query: {
+ "source-table": `card__${card.id}`,
+ filter: [">", ["field", "Expr", { "base-type": "type/Integer" }], 1],
+ },
+ dashboard_id: ORDERS_DASHBOARD_ID,
+ });
+
+ H.createQuestion({
+ name: "Question without fields",
+ query: {
+ "source-table": `card__${card.id}`,
+ },
+ });
+ });
+}
+
+function createSqlModelWithDependentSqlQuestions() {
+ H.createNativeQuestion({
+ name: "Base model",
+ type: "model",
+ native: {
+ query: "SELECT ID, TITLE, CATEGORY FROM PRODUCTS",
+ "template-tags": {},
+ },
+ }).then(({ body: card }) => {
+ cy.wrap(card.id).as("modelId");
+ cy.request("POST", `/api/card/${card.id}/query`);
+
+ const tagName = `#${card.id}`;
+ H.createNativeQuestion({
+ name: "Question with fields",
+ native: {
+ query: `SELECT ID, CATEGORY FROM {{#${card.id}}}`,
+ "template-tags": {
+ [tagName]: {
+ id: "10422a0f-292d-10a3-fd90-407cc9e3e20e",
+ name: tagName,
+ "display-name": tagName,
+ type: "card",
+ "card-id": card.id,
+ },
+ },
+ },
+ });
+
+ H.createNativeQuestion({
+ name: "Question without fields",
+ native: {
+ query: `SELECT COUNT(*) FROM {{#${card.id}}}`,
+ "template-tags": {
+ [tagName]: {
+ id: "10422a0f-292d-10a3-fd90-407cc9e3e20f",
+ name: tagName,
+ "display-name": tagName,
+ type: "card",
+ "card-id": card.id,
+ },
+ },
+ },
+ });
+ });
+}
+
+function createMetricWithDependentMbqlQuestionsAndTransforms() {
+ H.getTableId({ databaseId: WRITABLE_DB_ID, name: "Animals" }).then(
+ (tableId) => {
+ H.getFieldId({ tableId, name: "score" }).then((fieldId) => {
+ H.createQuestion({
+ name: "Base metric",
+ type: "metric",
+ database: WRITABLE_DB_ID,
+ query: {
+ "source-table": tableId,
+ aggregation: [["min", ["field", fieldId, null]]],
+ },
+ }).then(({ body: metric }) => {
+ cy.wrap(metric.id).as("metricId");
+
+ H.createQuestion({
+ name: "Question with 1 stage",
+ database: WRITABLE_DB_ID,
+ query: {
+ "source-table": tableId,
+ aggregation: [["metric", metric.id]],
+ },
+ });
+
+ H.createQuestion({
+ name: "Question with 2 stages",
+ database: WRITABLE_DB_ID,
+ query: {
+ "source-query": {
+ "source-table": tableId,
+ aggregation: [["metric", metric.id]],
+ breakout: [["field", fieldId, null]],
+ },
+ aggregation: [
+ ["avg", ["field", "min", { "base-type": "type/Integer" }]],
+ ],
+ },
+ });
+
+ H.createTransform({
+ name: "Transform with 1 stage",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "query",
+ query: {
+ "source-table": tableId,
+ aggregation: [["metric", metric.id]],
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "transform_1_stage",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ });
+
+ H.createTransform({
+ name: "Transform with 2 stages",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "query",
+ query: {
+ "source-query": {
+ "source-table": tableId,
+ aggregation: [["metric", metric.id]],
+ breakout: [["field", fieldId, null]],
+ },
+ aggregation: [
+ ["avg", ["field", "min", { "base-type": "type/Integer" }]],
+ ],
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "transform_2_stages",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ });
+ });
+ });
+ },
+ );
+}
+
+function createSqlTransformWithDependentMbqlQuestions() {
+ const transformTableName = "base_transform";
+
+ H.createTransform({
+ name: "Base transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "native",
+ native: {
+ query: 'SELECT name, score FROM "Schema A"."Animals"',
+ "template-tags": {},
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: transformTableName,
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ }).then(({ body: transform }) => {
+ cy.wrap(transform.id).as("transformId");
+ cy.request("POST", `/api/ee/transform/${transform.id}/run`);
+ H.waitForSucceededTransformRuns();
+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: transformTableName });
+
+ H.getTableId({ databaseId: WRITABLE_DB_ID, name: transformTableName }).then(
+ (tableId) => {
+ H.createQuestion({
+ name: "Question with fields",
+ database: WRITABLE_DB_ID,
+ query: {
+ "source-table": tableId,
+ filter: [
+ ">",
+ ["field", "score", { "base-type": "type/Integer" }],
+ 1,
+ ],
+ },
+ });
+ H.createQuestion({
+ name: "Question without fields",
+ database: WRITABLE_DB_ID,
+ query: {
+ "source-table": tableId,
+ },
+ });
+ },
+ );
+ });
+}
+
+function createMbqlTransformWithDependentSqlTransforms() {
+ H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
+ (tableId) => {
+ H.createTransform(
+ {
+ name: "Base transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "query",
+ query: {
+ "source-table": tableId,
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "base_transform",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ },
+ { wrapId: true },
+ );
+
+ H.createTransform({
+ name: "Name transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "native",
+ native: {
+ query: "SELECT name FROM base_transform",
+ "template-tags": {},
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "name_transform",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ });
+
+ H.createTransform({
+ name: "Score transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "native",
+ native: {
+ query: "SELECT score FROM base_transform",
+ "template-tags": {},
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "score_transform",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ });
+ },
+ );
+}
+
+function createSnippetWithDependentQuestionsAndTransforms() {
+ H.createSnippet({
+ name: "ScoreSnippet",
+ content: "score = 1",
+ }).then(({ body: snippet }) => {
+ cy.wrap(snippet.id).as("snippetId");
+
+ H.createNativeQuestion({
+ name: "Question with snippet",
+ database: WRITABLE_DB_ID,
+ native: {
+ query:
+ 'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',
+ "template-tags": {
+ "snippet:ScoreSnippet": {
+ id: "4b77cc1f-ea70-4ef6-84db-58432fce6928",
+ name: "snippet:ScoreSnippet",
+ "display-name": "snippet:ScoreSnippet",
+ type: "snippet",
+ "snippet-id": snippet.id,
+ "snippet-name": snippet.name,
+ },
+ },
+ },
+ });
+
+ H.createTransform({
+ name: "Transform with snippet",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "native",
+ native: {
+ query:
+ 'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',
+ "template-tags": {
+ "snippet:ScoreSnippet": {
+ id: "4b77cc1f-ea70-4ef6-84db-58432fce6929",
+ name: "snippet:ScoreSnippet",
+ "display-name": "snippet:ScoreSnippet",
+ type: "snippet",
+ "snippet-id": snippet.id,
+ "snippet-name": snippet.name,
+ },
+ },
+ },
+ },
+ },
+ target: {
+ type: "table",
+ name: "transform_with_snippet",
+ schema: "public",
+ database: WRITABLE_DB_ID,
+ },
+ });
+ });
+}
+
+function startEditingSnippet() {
+ H.startNewNativeQuestion();
+ cy.findByTestId("native-query-editor-action-buttons").icon("snippet").click();
+ cy.findByTestId("sidebar-content").within(() => {
+ cy.icon("chevrondown").click({ force: true });
+ cy.button(/Edit/).click();
+ });
+}
+
+function confirmDiscardChanges() {
+ H.modal().should("have.length", 2).last().button("Discard changes").click();
+}
diff --git a/enterprise/backend/src/metabase_enterprise/api/routes.clj b/enterprise/backend/src/metabase_enterprise/api/routes.clj
index 18a010c7264..b5a056623f0 100644
--- a/enterprise/backend/src/metabase_enterprise/api/routes.clj
+++ b/enterprise/backend/src/metabase_enterprise/api/routes.clj
@@ -20,6 +20,7 @@
[metabase-enterprise.content-verification.api.routes]
[metabase-enterprise.database-replication.api :as database-replication.api]
[metabase-enterprise.database-routing.api]
+ [metabase-enterprise.dependencies.api]
[metabase-enterprise.documents.api]
[metabase-enterprise.email.api]
[metabase-enterprise.embedding-hub.api]
@@ -52,6 +53,7 @@
:ai-entity-analysis (deferred-tru "AI Entity Analysis")
:collection-cleanup (deferred-tru "Collection Cleanup")
:content-translation (deferred-tru "Content translation")
+ :dependencies (deferred-tru "Dependency Tracking")
:documents (deferred-tru "Documents")
:embedding (deferred-tru "Embedding")
:etl-connections (deferred-tru "ETL Connections")
@@ -104,6 +106,7 @@
(premium-handler :etl-connections)
(premium-handler :etl-connections-pg))
"/database-routing" (premium-handler metabase-enterprise.database-routing.api/routes :database-routing)
+ "/dependencies" (premium-handler metabase-enterprise.dependencies.api/routes :dependencies)
"/document" (premium-handler metabase-enterprise.documents.api/routes :documents)
"/email" (premium-handler metabase-enterprise.email.api/routes :cloud-custom-smtp)
"/embedding-hub" (premium-handler metabase-enterprise.embedding-hub.api/routes :embedding)
diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj
index ac1b618e720..db80c8986fc 100644
--- a/enterprise/backend/src/metabase_enterprise/core/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj
@@ -12,6 +12,7 @@
[metabase-enterprise.cache.init]
[metabase-enterprise.comments.init]
[metabase-enterprise.database-replication.init]
+ [metabase-enterprise.dependencies.init]
[metabase-enterprise.documents.init]
[metabase-enterprise.gsheets.init]
[metabase-enterprise.llm.init]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
new file mode 100644
index 00000000000..4ddf2874033
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -0,0 +1,124 @@
+(ns metabase-enterprise.dependencies.api
+ (:require
+ [medley.core :as m]
+ [metabase-enterprise.dependencies.core :as dependencies]
+ [metabase.analyze.core :as analyze]
+ [metabase.api.macros :as api.macros]
+ [metabase.api.routes.common :refer [+auth]]
+ [metabase.api.util.handlers :as handlers]
+ [metabase.collections.models.collection.root :as collection.root]
+ [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.models.interface :as mi]
+ [metabase.native-query-snippets.core :as native-query-snippets]
+ [metabase.queries.schema :as queries.schema]
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.log :as log]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
+ [toucan2.core :as t2]))
+
+(mr/def ::card-body
+ [:map
+ [:id {:optional false} ms/PositiveInt]
+ [:dataset_query {:optional true} [:maybe ms/Map]]
+ [:type {:optional true} [:maybe ::queries.schema/card-type]]
+ [:result_metadata {:optional true} [:maybe analyze/ResultsMetadata]]])
+
+(defn- broken-cards-response
+ [{:keys [card transform]}]
+ (let [broken-card-ids (keys card)
+ broken-cards (when (seq broken-card-ids)
+ (-> (t2/select :model/Card :id [:in broken-card-ids])
+ (t2/hydrate [:collection :effective_ancestors] :dashboard)))
+ broken-transform-ids (keys transform)
+ broken-transforms (when (seq broken-transform-ids)
+ (t2/select :model/Transform :id [:in broken-transform-ids]))]
+ {:success (and (empty? broken-card-ids)
+ (empty? broken-transform-ids))
+ :bad_cards (into [] (comp (filter (fn [card]
+ (if (mi/can-read? card)
+ card
+ (do (log/warnf "Eliding broken card %d - not readable by the user" (:id card))
+ nil))))
+ (map (fn [card]
+ (-> card
+ collection.root/hydrate-root-collection
+ (update :dashboard #(some-> % (select-keys [:id :name])))))))
+ broken-cards)
+ :bad_transforms (into [] broken-transforms)}))
+
+(api.macros/defendpoint :post "/check_card"
+ "Check a proposed edit to a card, and return the card IDs for those cards this edit will break."
+ [_route-params
+ _query-params
+ body :- ::card-body]
+ (let [database-id (-> body :dataset_query :database)
+ base-provider (lib-be.metadata.jvm/application-database-metadata-provider database-id)
+ original (lib.metadata/card base-provider (:id body))
+ card (-> original
+ (assoc :dataset-query (:dataset_query body)
+ :type (:type body (:type original)))
+ ;; Remove the old `:result-metadata` from the card, it's likely wrong now.
+ (dissoc :result-metadata)
+ ;; But if the request includes `:result_metadata`, use that. It may be from a native card
+ ;; that's been run before saving the card.
+ (cond-> #_card
+ (:result_metadata body) (assoc :result-metadata (:result_metadata body))))
+ edits {:card [card]}
+ breakages (dependencies/errors-from-proposed-edits base-provider edits)]
+ (broken-cards-response breakages)))
+
+(mr/def ::transform-body
+ [:map
+ [:id {:optional false} ms/PositiveInt]
+ [:name {:optional true} :string]
+ [:source {:optional true} [:maybe ms/Map]]
+ [:target {:optional true} [:maybe ms/Map]]])
+
+(api.macros/defendpoint :post "/check_transform"
+ "Check a proposed edit to a transform, and return the card, transform, etc. IDs for things that will break."
+ [_route-params
+ _query-params
+ {:keys [id source target] :as _body} :- ::transform-body]
+ (if (= (keyword (:type source))
+ :query)
+ (let [database-id (-> source :query :database)
+ base-provider (lib-be.metadata.jvm/application-database-metadata-provider database-id)
+ original (lib.metadata/transform base-provider id)
+ transform (-> original
+ (cond-> #_transform source (assoc :source source))
+ (cond-> #_transform target (assoc :target target)))
+ edits {:transform [transform]}
+ breakages (dependencies/errors-from-proposed-edits base-provider edits)]
+ (broken-cards-response breakages))
+ ;; if this isn't a sql query, just claim it works
+ {:success true}))
+
+(api.macros/defendpoint :post "/check_snippet"
+ "Check a proposed edit to a native snippet, and return the cards, etc. which will be broken."
+ [_route-params
+ _query-params
+ {:keys [id content], snippet-name :name}
+ :- [:map
+ [:id {:optional false} ms/PositiveInt]
+ [:name {:optional true} native-query-snippets/NativeQuerySnippetName]
+ [:content {:optional true} :string]]]
+ (let [original (t2/select-one :model/NativeQuerySnippet id)
+ _ (when (and snippet-name
+ (not= snippet-name (:name original))
+ (t2/exists? :model/NativeQuerySnippet :name snippet-name))
+ (throw (ex-info (tru "A snippet with that name already exists. Please pick a different name.")
+ {:status-code 400})))
+ snippet (cond-> (m/assoc-some original
+ :lib/type :metadata/native-query-snippet
+ :name snippet-name
+ :content content)
+ content native-query-snippets/add-template-tags)
+ breakages (dependencies/errors-from-proposed-edits {:snippet [snippet]})]
+ (broken-cards-response breakages)))
+
+(def ^{:arglists '([request respond raise])} routes
+ "`/api/ee/dependencies` routes."
+ (handlers/routes
+ (api.macros/ns-handler *ns* +auth)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
new file mode 100644
index 00000000000..085f9936c03
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -0,0 +1,66 @@
+(ns metabase-enterprise.dependencies.calculation
+ (:require
+ [metabase-enterprise.dependencies.native-validation :as deps.native]
+ [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.lib.util :as lib.util]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]))
+
+(defn- upstream-deps:mbql-query [legacy-query]
+ (lib.util/source-tables-and-cards [legacy-query]))
+
+(defn- upstream-deps:native-query [metadata-provider query]
+ (let [engine (:engine (lib.metadata/database metadata-provider))
+ deps (deps.native/native-query-deps engine metadata-provider query)]
+ ;; The deps are in #{{:table 7} ...} form and need conversion to ::upstream-deps form.
+ (u/group-by ffirst (comp second first) conj #{} deps)))
+
+(mr/def ::upstream-deps
+ [:map
+ [:card {:optional true} [:set ::lib.schema.id/card]]
+ [:table {:optional true} [:set ::lib.schema.id/table]]
+ [:snippet {:optional true} [:set ::lib.schema.id/snippet]]
+ [:transform {:optional true} [:set ::lib.schema.id/transform]]])
+
+(defn- upstream-deps:query [metadata-provider legacy-query]
+ (case (:type legacy-query)
+ (:query "query") (upstream-deps:mbql-query legacy-query)
+ (:native "native") (upstream-deps:native-query metadata-provider legacy-query)
+ (throw (ex-info "Unhandled kind of query" {:query legacy-query}))))
+
+(mu/defn upstream-deps:card :- ::upstream-deps
+ "Given a Toucan `:model/Card`, return its upstream dependencies as a map from the kind to a set of IDs."
+ ([card] (upstream-deps:card (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))
+ card))
+ ([metadata-provider :- ::lib.schema.metadata/metadata-provider
+ {query :dataset_query :as _toucan-card}]
+ (upstream-deps:query metadata-provider query)))
+
+(mu/defn upstream-deps:transform :- ::upstream-deps
+ "Given a Transform (in Toucan form), return its upstream dependencies."
+ ([transform] (upstream-deps:transform nil transform))
+ ([metadata-provider :- [:maybe ::lib.schema.metadata/metadata-provider]
+ {{:keys [query type]} :source :as transform}]
+ (if (#{"query" :query} type)
+ (upstream-deps:query (or metadata-provider
+ (lib-be.metadata.jvm/application-database-metadata-provider (:database query)))
+ query)
+ (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) type)
+ {}))))
+
+(mu/defn upstream-deps:snippet :- ::upstream-deps
+ "Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."
+ [{:keys [template_tags] :as _snippet}]
+ (let [type->id-key {:card :card-id, :snippet :snippet-id}
+ dependencies (keep (fn [tag]
+ (let [entity-type (:type tag)]
+ (when-let [id-key (type->id-key entity-type)]
+ (when-let [entity-id (id-key tag)]
+ [entity-type entity-id]))))
+ (vals template_tags))]
+ (u/group-by first second conj #{} dependencies)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
new file mode 100644
index 00000000000..de034ba06fd
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
@@ -0,0 +1,205 @@
+(ns metabase-enterprise.dependencies.core
+ "API namespace for the `metabase-enterprise.dependencies` module.
+
+ Call [[errors-from-proposed-edits]] to find out what things will break downstream of a set of new/updated entities."
+ (:require
+ [metabase-enterprise.dependencies.metadata-provider :as deps.provider]
+ [metabase-enterprise.dependencies.models.dependency :as deps.graph]
+ [metabase-enterprise.dependencies.native-validation :as deps.native]
+ [metabase.graph.core :as graph]
+ [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [toucan2.core :as t2]))
+
+(mr/def ::entity-type
+ [:enum :card :transform :snippet :table])
+
+(mr/def ::updates-map
+ ;; TODO: Make this more specific.
+ [:map-of ::entity-type [:sequential [:map [:id {:optional true} :int]]]])
+
+(mu/defn- metadata-provider :- ::lib.schema.metadata/metadata-provider
+ "Constructs a `MetadataProvider` with some pending edits applied.
+
+ The edits are *transitive*! Since a card's output columns can depend on its (possibly updated) inputs, all dependents
+ of editing things are considered edited too.
+
+ Note that if `:result-metadata` is present on an updated Card, it **will be used!** The only case where that is
+ actually useful is with a native query which has been executed, so the caller has driver metadata to give us. Any
+ old, pre-update `:result-metadata` should be dropped from any other cards in `updated-entities`."
+ ([base-provider :- ::lib.schema.metadata/metadata-provider
+ updated-entities :- ::updates-map
+ & {:keys [graph dependents]}]
+ (let [dependents (or dependents (deps.graph/transitive-dependents graph updated-entities))]
+ (deps.provider/override-metadata-provider base-provider updated-entities dependents))))
+
+(defmulti check-query-inner
+ "Given a `MetadataProvider` and a lib query, find any bad refs in that query."
+ {:arglists '([metadata-provider driver query])}
+ (fn [_mp _driver query]
+ (-> (lib/query-stage query 0)
+ :lib/type)))
+
+(defmethod check-query-inner :mbql.stage/mbql
+ [_mp _driver query]
+ (lib/find-bad-refs query))
+
+(defmethod check-query-inner :mbql.stage/native
+ [mp driver query]
+ (deps.native/validate-native-query driver mp query))
+
+(defn- check-query
+ "Given a `MetadataProvider` and a query, find any bad refs in that query."
+ [mp driver query]
+ (check-query-inner mp driver (lib/query mp query)))
+
+(defmulti check-entity
+ "Given a `MetadataProvider`, and entity type, and an entity id, find any bad refs in that entity."
+ {:arglists '([metadata-provider entity-type entity-id])}
+ (fn [_mp entity-type _entity-id]
+ entity-type))
+
+(defmethod check-entity :default
+ [_mp _entity-type _entity-id]
+ nil)
+
+(defmethod check-entity :card
+ [mp _entity-type entity-id]
+ (let [query (:dataset-query (lib.metadata/card mp entity-id))
+ driver (:engine (lib.metadata/database mp))]
+ (check-query mp driver query)))
+
+(defmethod check-entity :transform
+ [mp _entity-type entity-id]
+ (let [{{target-schema :schema target-name :name} :target
+ {:keys [query]} :source
+ :as _transform} (lib.metadata/transform mp entity-id)
+ driver (:engine (lib.metadata/database mp))
+ output-table (some #(when (and (= (:schema %) target-schema)
+ (= (:name %) target-name))
+ %)
+ (lib.metadata/tables mp))
+ output-fields (lib.metadata/active-fields mp (:id output-table))
+ {:keys [duplicate-fields]} (reduce (fn [{:keys [seen duplicate-fields]}
+ {name :name :as field}]
+ (if (seen name)
+ {:seen seen
+ :duplicate-fields (conj duplicate-fields field)}
+ {:seen (conj seen name)
+ :duplicate-fields duplicate-fields}))
+ {:seen #{}}
+ output-fields)]
+ (into duplicate-fields (check-query mp driver query))))
+
+(mu/defn- check-query-soundness ;; :- [:map-of ::lib.schema.id/card [:sequential ::lib.schema.mbql-clause/clause]]
+ "Given a `MetadataProvider` as returned by [[metadata-provider]], scan all its updated entities and their dependents
+ to check that everything is still sound.
+
+ In particular, we're looking for bad clauses with refs to columns which no longer exist, since those queries fail
+ to compile.
+
+ May return false positives: if a card already has a bad ref in it, then it will be returned even if that bad ref has
+ nothing to do with the updates in the `provider`.
+
+ Returns a map `{:card {card-id [bad-ref ...]}, :transform {...}}`. It will be empty, if there are no bad refs
+ detected."
+ [provider :- ::lib.schema.metadata/metadata-provider]
+ (let [overrides (deps.provider/all-overrides provider)
+ errors (volatile! {})]
+ (doseq [[entity-type ids] overrides
+ id ids
+ :let [bad-refs (check-entity provider entity-type id)]]
+ (when (seq bad-refs)
+ (vswap! errors assoc-in [entity-type id] bad-refs)))
+ @errors))
+
+(defn- group-by-db [deps]
+ (let [by-db (volatile! {})]
+ (when (seq (:card deps))
+ (doseq [[db-id card-ids] (->> (t2/select [:model/Card :id :database_id :card_schema] :id [:in (:card deps)])
+ (u/group-by :database_id :id conj #{}))]
+ (vswap! by-db assoc-in [db-id :card] card-ids)))
+ (when (seq (:table deps))
+ (doseq [[db-id table-ids] (->> (t2/select [:model/Table :id :db_id] :id [:in (:table deps)])
+ (u/group-by :db_id :id conj #{}))]
+ (vswap! by-db assoc-in [db-id :table] table-ids)))
+ (when (seq (:transform deps))
+ (doseq [[db-id transform-ids] (->> (t2/select [:model/Transform :id :source] :id [:in (:transform deps)])
+ (u/group-by #(get-in % [:source :query :database]) :id conj #{}))]
+ (if db-id
+ (vswap! by-db assoc-in [db-id :transform] transform-ids)
+ (log/warnf "Unable to infer database from transforms %s" transform-ids))))
+ (when (seq (:snippet deps))
+ ;; Copy any snippet deps to each database, since they span them all.
+ (vswap! by-db update-vals #(assoc % :snippet (:snippet deps))))
+
+ @by-db))
+
+;; TODO: (Braden 09/22/2025) More precise schemas for the errors this function returns. Currently they're pretty
+;; opaque, since the consumers of this function really care about "working/broken" and not the details of what's wrong
+;; with any particular card.
+(mu/defn errors-from-proposed-edits :- [:map-of ::entity-type [:map-of :int [:or :boolean [:sequential :any]]]]
+ "Given a regular `MetadataProvider`, and a map of entity types (`:card`, `:transform`, `:snippet`) to lists of
+ updated entities, this returns a map of `{entity-type {entity-id [bad-ref ...]}}`.
+
+ The 1-arity groups all the dependents by which Database they are part of, and runs the analysis for each of them.
+
+ The output is a map: `{entity-type {id [errors...]}}`; an empty map is returned when there are no errors
+ detected."
+ ([edits :- ::updates-map]
+ (let [all-deps (deps.graph/transitive-dependents edits)
+ by-db (group-by-db all-deps)]
+ (reduce (fn [errors [db-id deps]]
+ (-> (lib-be.metadata.jvm/application-database-metadata-provider db-id)
+ (metadata-provider edits :dependents deps)
+ check-query-soundness
+ (merge errors)))
+ {} by-db)))
+
+ ([base-provider :- ::lib.schema.metadata/metadata-provider
+ edits :- ::updates-map]
+ (errors-from-proposed-edits base-provider nil edits))
+
+ ([base-provider :- ::lib.schema.metadata/metadata-provider
+ graph :- [:maybe ::graph/graph]
+ edits :- ::updates-map]
+ (-> base-provider
+ (metadata-provider edits :graph graph)
+ check-query-soundness)))
+
+#_{:clj-kondo/ignore [:unresolved-namespace]}
+(comment
+ ;; This should work on any fresh-ish Metabase instance; these are the built-in example questions.
+ (let [base-mp (lib-be.metadata.jvm/application-database-metadata-provider 39)
+ transform (lib.metadata/transform base-mp 1)
+ transform (-> transform
+ (update-in [:source :query :query :expressions]
+ ;; Replacing the expression Age with Duration - this breaks downstream uses!
+ update-keys (constantly "Fair Die"))
+ #_(dissoc :result-metadata))
+ mp (metadata-provider base-mp {:transform [transform]})]
+ #_(deps.provider/all-overrides mp)
+ (check-query-soundness mp))
+
+ ;; Checking that we can properly group everything by database. Surprisingly fast even fetching everything.
+ (let [deps {:card (t2/select-fn-set :id :model/Card)
+ :transform (t2/select-fn-set :id :model/Transform)
+ :table (t2/select-fn-set :id :model/Table)}]
+ (group-by-db deps))
+
+ (t2/select-fn-vec (juxt :id :name :active :table_id) :model/Field :id [:in [1065 1066 1067 1068 1069]])
+
+ (t2/select-one :model/Card :id 124)
+ (deps.graph/transitive-dependents {:transform [{:id 1}]})
+ (deps.graph/transitive-dependents {:table [{:id 155}]})
+ (metabase.premium-features.core/token-features)
+
+ (let [card (t2/select-one :model/Card :id 121)
+ mp (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))]
+ (upstream-deps:card mp card)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
new file mode 100644
index 00000000000..51eb03b6482
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
@@ -0,0 +1,4 @@
+(ns metabase-enterprise.dependencies.init
+ (:require
+ [metabase-enterprise.dependencies.models.dependency]
+ [metabase-enterprise.dependencies.task.backfill]))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
new file mode 100644
index 00000000000..4454ade9055
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
@@ -0,0 +1,271 @@
+(ns metabase-enterprise.dependencies.metadata-provider
+ "A `MetadataProvider` wrapper that understands how to update entities, and to lazily compute the new metadata for
+ updated entities on demand. This is useful when analyzing a tree of dependents, which might depend on each other in
+ complex ways.
+
+ Just to illustrate, imagine I change a card A which is used by a transform B whose output table C is used by a card
+ D. If I remove a column from card A, and transform B selects all columns, that field needs to disappear from C
+ before we can correctly analyze card D.
+
+ By representing the updated entities as lazy `delay`s, this `MetadataProvider` can support analysis of the
+ downstream entities in any order."
+ (:require
+ [medley.core :as m]
+ [metabase-enterprise.dependencies.native-validation :as deps.native]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [pretty.core :as pretty]))
+
+(set! *warn-on-reflection* true)
+
+(defn- metadatas [delegate overrides {metadata-type :lib/type
+ search-name :name
+ :keys [id card-id table-id]
+ :as metadata-spec}]
+ (let [type-overrides (get overrides metadata-type)
+ metadata-keys (set (or id search-name))]
+ (log/tracef "OverridingMetadataProvider request for %s" metadata-spec)
+ (cond
+ ;; For tables, cards, snippets, transforms, and columns, return overrides if present and looking for specific
+ ;; keys and delegate if not.
+ (and (#{:metadata/table :metadata/card :metadata/transform :metadata/native-query-snippet :metadata/column} metadata-type)
+ (seq metadata-keys))
+ (let [overrides (if (seq id)
+ (select-keys type-overrides id)
+
+ (into {} (keep #(when-let [x (deref %)]
+ (when (search-name (:name x))
+ [(:name x) %])))
+ (vals type-overrides)))
+ override-key (if (seq id) :id :name)
+ overridden-keys (set (keys overrides))
+ missing-keys (remove overridden-keys metadata-keys)]
+ (log/tracef "Overridden keys %s; missing keys %s" (sort overridden-keys) (sort missing-keys))
+ (cond-> (map deref (vals overrides))
+ (seq missing-keys)
+ (concat (lib.metadata.protocols/metadatas delegate {:lib/type metadata-type
+ override-key (set missing-keys)}))))
+
+ ;; if fetching all tables or transforms, get every override and everything from the delegate and then dedupe
+ (and (empty? metadata-keys)
+ (#{:metadata/table :metadata/transform} metadata-type))
+ (as-> (vals type-overrides) entities
+ (into [] (map force) entities)
+ (into entities (lib.metadata.protocols/metadatas delegate metadata-spec))
+ (m/distinct-by :id entities))
+
+ ;; For columns, return overrides if they're present.
+ (= metadata-type :metadata/column)
+ (if-let [overridden-columns (cond
+ card-id (get-in overrides [::card-columns card-id])
+ table-id (get-in overrides [::table-columns table-id]))]
+ @overridden-columns
+ (lib.metadata.protocols/metadatas delegate metadata-spec))
+
+ :else (throw (ex-info "Unknown :lib/metadata type for OverridingMetadataProvider" metadata-spec)))))
+
+(declare all-overrides setup-transform! setup-transforms!)
+
+(deftype OverridingMetadataProvider [delegate *overrides]
+ lib.metadata.protocols/MetadataProvider
+ (database [_this] (lib.metadata.protocols/database delegate))
+ (metadatas [this metadata-spec]
+ (setup-transforms! this)
+ (metadatas delegate @*overrides metadata-spec))
+ (setting [_this setting-key]
+ (lib.metadata.protocols/setting delegate setting-key))
+
+ ;; Cached things are direct delegation.
+ lib.metadata.protocols/CachedMetadataProvider
+ (cached-metadatas [_this metadata-type metadata-ids]
+ (lib.metadata.protocols/cached-metadatas delegate metadata-type metadata-ids))
+ (store-metadata! [_this a-metadata]
+ (lib.metadata.protocols/store-metadata! delegate a-metadata))
+ (cached-value [_this k not-found]
+ (lib.metadata.protocols/cached-value delegate k not-found))
+ (cache-value! [_this k v]
+ (lib.metadata.protocols/cache-value! delegate k v))
+ (has-cache? [_this]
+ (lib.metadata.protocols/has-cache? delegate))
+
+ pretty/PrettyPrintable
+ (pretty [this]
+ (list `overriding-metadata-provider (all-overrides this) delegate)))
+
+(defmulti add-override
+ "Given an `OverridingMetadataProvider`, an `entity-type` and that entity, add this entity as an override."
+ {:arglists '([metadata-provider entity-type id updates-or-nil])}
+ (fn [_mp entity-type _id _updates]
+ entity-type))
+
+(defn- inner-mp [^OverridingMetadataProvider mp]
+ (.delegate mp))
+
+(defn- with-overrides [^OverridingMetadataProvider mp kvs]
+ (swap! (.*overrides mp)
+ #(reduce (fn [m [ks v]]
+ (assoc-in m ks v))
+ % kvs)))
+
+(defn- get-returned-columns [mp queryable]
+ (let [query (lib/query mp queryable)]
+ (if (= (:lib/type (lib/query-stage query 0))
+ :mbql.stage/mbql)
+ (lib/returned-columns query)
+ (deps.native/native-result-metadata (:engine (lib.metadata/database mp))
+ mp query))))
+
+(defmethod add-override :card [^OverridingMetadataProvider mp _entity-type id updates]
+ (with-overrides mp
+ ;; If the `updates` contain `:result-metadata`, we want to use that. However, any `:result-metadata` from the
+ ;; inner-mp should be ignored.
+ {[:metadata/card id] (delay (merge (when id
+ (-> (inner-mp mp)
+ (lib.metadata/card id)
+ (dissoc :result-metadata)))
+ updates))
+ ;; This uses the outer OMP and so the overrides are visible!
+ [::card-columns id] (delay (get-returned-columns mp (lib.metadata/card mp id)))}))
+
+(defonce ^:private last-fake-id (atom 2000000000))
+
+(defn- fake-id []
+ (swap! last-fake-id inc))
+
+(defn- setup-transform!
+ [^OverridingMetadataProvider mp s+n->table {:keys [source target] :as _transform}]
+ ;; See if the output table exists in the underlying MetadataProvider.
+ ;; - If the table doesn't exist, invent an ID for it.
+ ;; - Always add an override for the table, even if it already existed.
+ ;; - Compare the transform's output columns to the fields that exist in the metadata.
+ ;; - For any newly added fields, invent an ID and add an override for the field.
+ ;; - For any removed fields that used to exist, add an override for their ID that is nil!
+ ;; - Add an internal [::table-columns table-id] with the list of output columns.
+ ;; Note that this assumes that all other overrides are already loaded!
+ (let [existing-table (get s+n->table [(:schema target) (:name target)])
+ output-table (or existing-table
+ (merge (select-keys target [:schema :name])
+ {:lib/type :metadata/table
+ :id (fake-id)
+ :db-id (:id (lib.metadata/database (inner-mp mp)))
+ :display-name (:name target)
+ :visibility-type nil}))
+ existing-cols (when existing-table
+ (lib/returned-columns (lib/query (inner-mp mp) existing-table)))
+ output-cols (delay
+ ;; Note that this will analyze the query with any upstream changes included!
+ (let [new-cols (get-returned-columns mp (:query source))
+ by-name (m/index-by :lib/desired-column-alias existing-cols)]
+ (into [] (for [col new-cols
+ :let [old-col (by-name (:lib/desired-column-alias col))]]
+ (merge (select-keys col [:display-name
+ :base-type :effective-type :semantic-type])
+ {:name (:lib/desired-column-alias col)
+ :lib/type :metadata/column
+ :id (or (:id old-col) (fake-id))
+ :table-id (:id output-table)
+ :active true})))))
+ outputs-by-id (delay (m/index-by :id @output-cols))]
+ ;; NOTE: Any newly added output columns can't be looked up directly by ID, with this version.
+ ;; The `output-cols` code above can be adapted to add them to the `*overrides` atom, but YAGNI for now.
+ (with-overrides mp
+ (into {[:metadata/table (:id output-table)] (delay output-table)
+ [::table-columns (:id output-table)] output-cols}
+ ;; For each pre-existing output column, we need to add an override for a direct read by ID.
+ ;; If the column does not exist in the output, return the original with :active false
+ (map (fn [existing-col]
+ [[:metadata/column (:id existing-col)]
+ (delay (or (get @outputs-by-id (:id existing-col))
+ (assoc existing-col :active false)))]))
+ existing-cols))))
+
+(defn- setup-transforms! [^OverridingMetadataProvider mp]
+ (locking mp
+ (when-not (-> (.*overrides mp) deref ::setup-complete)
+ (swap! (.*overrides mp) assoc ::setup-complete true)
+ (let [;; Fetching all the pre-existing tables.
+ ;; TODO: Once Cam's new by-name logic in the `MetadataProvider` lands, we can use that here.
+ tables (lib.metadata/tables (inner-mp mp))
+ s+n->table (m/index-by (juxt :schema :name) tables)]
+ (doseq [transform (-> (.*overrides mp) deref :metadata/transform vals)]
+ (setup-transform! mp s+n->table (deref transform)))))))
+
+(defmethod add-override :transform [^OverridingMetadataProvider mp _entity-type id updates]
+ ;; Transforms are complicated:
+ ;; 1. Treated as a card on the input side.
+ ;; 2. The returned-columns of that card become the columns of the output table.
+ ;; 3. The output table is constructed from the transform.
+ ;; 4. Its fields are adapted from the transform's input card.
+ ;; All of these can be wrapped in delays and therefore computed lazily on demand.
+ ;;
+ ;; The trouble is that a transform does not know the output table's ID, nor its field IDs.
+ ;; A modified transform might produce a new table, or have new fields or removed fields.
+ ;; The OverridingMetadataProvider has to account for all of this when it returns tables or fields by ID, or gets
+ ;; the column list for a table.
+ ;;
+ ;; This is achieved by putting `[:metadata/transform id]` into overrides for all transforms, and then one-time
+ ;; logic that runs before the first use of the OverridingMetadataProvider as a MetadataProvider will walk all the
+ ;; transforms and make sure their tables etc. are overridden properly.
+ (let [transform (merge (when id
+ (lib.metadata/transform (inner-mp mp) id))
+ updates)]
+ (with-overrides mp
+ {;; The transform itself, merged with the incoming updates.
+ [:metadata/transform id] (delay transform)
+ #_#_[::transform-columns id] (delay (->> (lib.metadata/transform mp id)
+ :source
+ :query
+ (lib/query mp)
+ lib/returned-columns))})))
+
+(defmethod add-override :table [^OverridingMetadataProvider mp _entity-type id updates]
+ (when (seq updates)
+ (throw (ex-info "Updating a table directly is not supported" {:table-id id, :edited updates})))
+ (with-overrides mp
+ {[:metadata/table id] (delay (log/warnf "Can't happen: Table %d dep should be replaced by its transform" id)
+ (lib.metadata/table (inner-mp mp) id))
+ [::table-columns id] (delay
+ (log/warnf "Can't happen: Table %d ::table-columns should be replaced by its transform" id)
+ (lib.metadata/fields (inner-mp mp) id))}))
+
+(defmethod add-override :snippet [^OverridingMetadataProvider mp _entity-type id updates]
+ (with-overrides mp
+ {[:metadata/native-query-snippet id] (delay (merge (when id
+ (lib.metadata/native-query-snippet (inner-mp mp) id))
+ (u/normalize-map updates)))}))
+
+(defn all-overrides
+ "Returns all the overrides by ID, in the same form as the map input to [[with-deps]]:
+ `{:card [1 2 3], :transform [45 99]}."
+ [^OverridingMetadataProvider override-metadata-provider]
+ (-> (.*overrides override-metadata-provider)
+ deref
+ (dissoc ::setup-complete)
+ (update-vals keys)
+ (update-keys {:metadata/card :card
+ :metadata/transform :transform
+ :metadata/native-query-snippet :snippet})
+ (select-keys [:card :transform :snippet])))
+
+(defn override-metadata-provider
+ "Given an underlying `MetadataProvider`, wraps it to support in-memory overrides of cards, fields, etc.
+
+ Important note: all overrides must be added first, before this is used as a `MetadataProvider`.
+ The easiest way to do that is to call the 3-arity, which provides the overrides up front."
+ ([metadata-provider]
+ (->OverridingMetadataProvider metadata-provider (atom {})))
+
+ ([metadata-provider updated-entities dependent-ids]
+ (let [^OverridingMetadataProvider omp (override-metadata-provider metadata-provider)]
+ (doseq [[entity-type updates] updated-entities
+ updated-entity updates]
+ (add-override omp entity-type (:id updated-entity) updated-entity))
+ (doseq [[entity-type dependents] dependent-ids
+ :let [updated (into #{} (map :id) (get updated-entities entity-type))]
+ id dependents
+ :when (not (updated id))]
+ (add-override omp entity-type id nil))
+ omp)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
new file mode 100644
index 00000000000..d8320e6b6eb
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -0,0 +1,212 @@
+(ns metabase-enterprise.dependencies.models.dependency
+ (:require
+ [clojure.set :as set]
+ [metabase-enterprise.dependencies.calculation :as deps.calculation]
+ [metabase.events.core :as events]
+ [metabase.graph.core :as graph]
+ [metabase.models.interface :as mi]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.util :as u]
+ [methodical.core :as methodical]
+ [potemkin :as p]
+ [toucan2.core :as t2]))
+
+(def current-dependency-analysis-version
+ "Current version of the dependency analysis logic.
+ This should be incremented when the dependency analysis logic changes."
+ 1)
+
+(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)
+
+(derive :model/Dependency :metabase/model)
+
+(t2/deftransforms :model/Dependency
+ {:from_entity_type mi/transform-keyword
+ :to_entity_type mi/transform-keyword})
+
+(defn- deps-children [src-type src-id dst-type dst-id key-seq]
+ ;; Group all keys with the same type together, so we make O(types) indexed [[t2/select]] calls, not O(n).
+ (transduce (map (fn [[entity-type entity-keys]]
+ (let [deps (t2/select :model/Dependency
+ src-type entity-type
+ src-id [:in entity-keys])]
+ (u/group-by (juxt src-type src-id)
+ (juxt dst-type dst-id)
+ conj #{}
+ deps))))
+ merge {}
+ (u/group-by first second key-seq)))
+
+(defn- key-dependents
+ "Get the dependent entity keys for the entity keys in `entity-keys`.
+ Entity keys are [entity-type, entity-id] pairs. See [[entity-type->model]]."
+ [key-seq]
+ (deps-children :to_entity_type :to_entity_id :from_entity_type :from_entity_id key-seq))
+
+(p/deftype+ DependencyGraph [children-fn]
+ graph/Graph
+ (children-of [_this key-seq]
+ (children-fn key-seq)))
+
+;; NOTE: We can easily construct a graph of upstream dependencies too, if it's useful.
+(defn- graph-dependents []
+ (->DependencyGraph key-dependents))
+
+(defn transitive-dependents
+ "Given a map of updated entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependents
+ as `{entity-type #{4 5 6}}` - that is, a map from downstream entity type to a set of IDs.
+
+ Uses the provided `graph`, or defaults to the `:model/Dependency` table in AppDB.
+
+ The inputs must be maps containing `:id`; anything without an `:id` is skipped. They could be Toucan entities,
+ `MetadataProvider` entities, user input, etc.
+
+ **Excludes** the input entities from the list of dependents!"
+ ([updated-entities] (transitive-dependents nil updated-entities))
+ ([graph updated-entities]
+ (let [graph (or graph (graph-dependents))
+ starters (for [[entity-type updates] updated-entities
+ entity updates
+ :when (:id entity)]
+ [entity-type (:id entity)])]
+ (->> (graph/transitive graph starters) ; This returns a flat list.
+ (u/group-by first second conj #{})))))
+
+(defn replace-dependencies
+ "Replace the dependencies of the entity of type `entity-type` with id `entity-id` with
+ the ones specified in `dependencies-by-type`. "
+ [entity-type entity-id dependencies-by-type]
+ (let [current-dependencies (t2/select [:model/Dependency :id :to_entity_type :to_entity_id]
+ :from_entity_type entity-type
+ :from_entity_id entity-id)
+ to-remove (keep (fn [{:keys [id to_entity_type to_entity_id]}]
+ (when-not (get-in dependencies-by-type [to_entity_type to_entity_id])
+ id))
+ current-dependencies)
+ current-by-type (-> (group-by :to_entity_type current-dependencies)
+ (update-vals #(into #{} (map :to_entity_id) %)))
+ to-add (for [[to-entity-type ids] dependencies-by-type
+ to-entity-id (set/difference ids (current-by-type to-entity-type))]
+ {:from_entity_type entity-type
+ :from_entity_id entity-id
+ :to_entity_type to-entity-type
+ :to_entity_id to-entity-id})]
+ (t2/with-transaction [_conn]
+ (when (seq to-remove)
+ (t2/delete! :model/Dependency :id [:in to-remove]))
+ (when (seq to-add)
+ (t2/insert! :model/Dependency to-add)))))
+
+;; ## Maintaining the dependency graph
+;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the
+;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created
+;; or changed at that point.
+
+;; ### Cards
+(derive ::card-deps :metabase/event)
+(derive :event/card-create ::card-deps)
+(derive :event/card-update ::card-deps)
+
+(methodical/defmethod events/publish-event! ::card-deps
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (t2/with-transaction [_conn]
+ (replace-dependencies :card (:id object) (deps.calculation/upstream-deps:card object))
+ (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)
+ (t2/update! :model/Card (:id object)
+ {:dependency_analysis_version current-dependency-analysis-version})))))
+
+(derive ::card-delete :metabase/event)
+(derive :event/card-delete ::card-delete)
+
+(methodical/defmethod events/publish-event! ::card-delete
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))))
+
+;; ### Snippets
+(derive ::snippet-deps :metabase/event)
+(derive :event/snippet-create ::snippet-deps)
+(derive :event/snippet-update ::snippet-deps)
+
+(methodical/defmethod events/publish-event! ::snippet-deps
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (t2/with-transaction [_conn]
+ (replace-dependencies :snippet (:id object) (deps.calculation/upstream-deps:snippet object))
+ (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)
+ (t2/update! :model/NativeQuerySnippet (:id object)
+ {:dependency_analysis_version current-dependency-analysis-version})))))
+
+(derive ::snippet-delete :metabase/event)
+(derive :event/snippet-delete ::snippet-delete)
+
+(methodical/defmethod events/publish-event! ::snippet-delete
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))))
+
+;; ### Transforms
+(derive ::transform-deps :metabase/event)
+(derive :event/create-transform ::transform-deps)
+(derive :event/update-transform ::transform-deps)
+
+;; On *saving* a transform, the upstream deps of its query are computed and saved.
+(defn- drop-outdated-target-dep! [{:keys [id source target] :as _transform}]
+ (let [db-id (some-> source :query :database)
+ downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency
+ :from_entity_type :table
+ :to_entity_type :transform
+ :to_entity_id id)
+ downstream-tables (when (seq downstream-table-ids)
+ (t2/select :model/Table :id [:in downstream-table-ids]))
+ outdated-tables (remove (fn [table]
+ (and (= (:schema table) (:schema target))
+ (= (:name table) (:name target))
+ (or (not db-id)
+ (= db-id (:db_id table)))))
+ downstream-tables)
+ not-found-table-ids (remove (into #{} (map :id) downstream-tables)
+ downstream-table-ids)]
+ (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)
+ (map :id) outdated-tables))]
+ (t2/delete! :model/Dependency
+ :from_entity_type :table
+ :from_entity_id [:in outdated-downstream-table-ids]
+ :to_entity_type :transform
+ :to_entity_id id))))
+
+(methodical/defmethod events/publish-event! ::transform-deps
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (t2/with-transaction [_conn]
+ (replace-dependencies :transform (:id object) (deps.calculation/upstream-deps:transform object))
+ (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)
+ (t2/update! :model/Transform (:id object) {:dependency_analysis_version current-dependency-analysis-version}))
+ (drop-outdated-target-dep! object))))
+
+(derive ::transform-delete :metabase/event)
+(derive :event/delete-transform ::transform-delete)
+
+(methodical/defmethod events/publish-event! ::transform-delete
+ [_ {:keys [id]}]
+ (when (premium-features/has-feature? :dependencies)
+ ;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?
+ (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)))
+
+;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.
+;; (And if the target has changed, the old table's dep on the transform is dropped.)
+;; The upstream deps of the transform are not touched - those change only when the transform is edited.
+(derive ::transform-run :metabase/event)
+(derive :event/transform-run-complete ::transform-run)
+
+(defn- transform-table-deps! [{:keys [db-id output-schema output-table transform-id] :as _details}]
+ (let [;; output-table is a keyword like :my_schema/my_table
+ table-name (name output-table)]
+ (when-let [table-id (t2/select-one-fn :id :model/Table :db_id db-id :schema output-schema :name table-name)]
+ (replace-dependencies :table table-id {:transform #{transform-id}}))))
+
+(methodical/defmethod events/publish-event! ::transform-run
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (transform-table-deps! object)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
new file mode 100644
index 00000000000..51291a74390
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
@@ -0,0 +1,38 @@
+(ns metabase-enterprise.dependencies.native-validation
+ (:require
+ [metabase.driver :as driver]
+ [metabase.lib.core :as lib]
+ [metabase.query-processor :as qp]))
+
+(defn- compile-query [metadata-provider query]
+ (->> (lib/query metadata-provider query)
+ lib/add-parameters-for-template-tags
+ (qp/compile-query-with-metadata-provider metadata-provider)
+ :query))
+
+(defn validate-native-query
+ "Compiles a (native) query and validates that the fields and tables it refers to really exist.
+
+ Returns either nil or a list of errors."
+ [driver metadata-provider query]
+ (->> (compile-query metadata-provider query)
+ (driver/validate-native-query-fields driver metadata-provider)))
+
+(defn native-result-metadata
+ "Compiles a (native) query and calculates its result metadata"
+ [driver metadata-provider query]
+ (->> (compile-query metadata-provider query)
+ (driver/native-result-metadata driver metadata-provider)))
+
+(defn native-query-deps
+ "Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs."
+ [driver metadata-provider query]
+ (let [compiled (compile-query metadata-provider query)]
+ (into (driver/native-query-deps driver compiled metadata-provider)
+ (keep #(case (:type %)
+ "snippet" {:snippet (:snippet-id %)}
+ :snippet {:snippet (:snippet-id %)}
+ "card" {:card (:card-id %)}
+ :card {:card (:card-id %)}
+ nil))
+ (-> query :native :template-tags vals))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
new file mode 100644
index 00000000000..09a19f3c038
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
@@ -0,0 +1,191 @@
+(ns metabase-enterprise.dependencies.task.backfill
+ "Implements a task that brings all entities with dependencies tracked in the dependency table
+ (see [[metabase-enterprise.dependencies.models.dependency]]) up to date, that is, makes sure
+ the dependency table contains fresh entries.
+
+ This is done by finding entities whose dependency_analysis_version is less than
+ [[metabase-enterprise.dependencies.models.dependency/current-dependency-analysis-version]] and
+ updates this field. In most cases this triggers an event handler that calculates the dependencies
+ and populates the dependency table. For cards, the event is emitted by the job handler itself,
+ because the update of the record doesn't trigger the event handler. "
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.scheduler :as qs]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [environ.core :as env]
+ [java-time.api :as t]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase.config.core :as config]
+ [metabase.events.core :as events]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2])
+ (:import
+ (java.util Map Set)
+ (java.util.concurrent ConcurrentHashMap)))
+
+(set! *warn-on-reflection* true)
+
+(defn- current-millis
+ "Returns the current epoch millis. Uses java-time so that clock settings can be used in tests."
+ []
+ (t/to-millis-from-epoch (t/instant)))
+
+(def ^:private entities [:model/Card :model/Transform :model/NativeQuerySnippet])
+
+;; In-memory state for tracking failed entities
+;; Stores {:model/Type {id {:fail-count N :next-retry-timestamp M}}}
+(def ^:private retry-state
+ (zipmap entities (repeatedly #(ConcurrentHashMap.))))
+
+;; Stores {:model/Type #{id1 id2 ...}} for entities that have exceeded MAX_RETRIES
+(def ^:private terminally-broken
+ (zipmap entities (repeatedly #(ConcurrentHashMap/newKeySet))))
+
+(def ^:private MAX_RETRIES 5)
+
+(defn- long-env-value
+ [env-var-kw default]
+ (or (parse-long (env/env env-var-kw "")) default))
+
+(defn- get-batch-size []
+ (long-env-value :mb-dependency-backfill-batch-size 20))
+
+(defn- get-delay-minutes []
+ (long-env-value :mb-dependency-backfill-delay-minutes 60))
+
+(defn- get-variance-minutes []
+ (long-env-value :mb-dependency-backfill-variance-minutes 10))
+
+(defn- get-processable-ids [model-kw batch-size]
+ (let [target-version models.dependency/current-dependency-analysis-version
+ terminally-broken-ids ^Set (terminally-broken model-kw)
+ retry-state-map ^Map (retry-state model-kw)
+ current-time (current-millis)]
+ (into []
+ (comp
+ (map :id)
+ (filter (fn [id]
+ (and (not (.contains terminally-broken-ids id))
+ (let [entity-retry-info (.get retry-state-map id)]
+ (or (nil? entity-retry-info)
+ (>= current-time (:next-retry-timestamp entity-retry-info)))))))
+ (take batch-size))
+ (t2/reducible-select [model-kw :id] :dependency_analysis_version [:< target-version]))))
+
+(defn- backfill-card!
+ [id target-version]
+ ;; We don't want to change the card at all, we just want to update the dependency data and
+ ;; mark the card as processed for this dependency analysis version.
+ (let [update-count (t2/update! :model/Card id :dependency_analysis_version [:< target-version]
+ {:dependency_analysis_version target-version})]
+ (when-let [card (and (pos? update-count)
+ (t2/select-one :model/Card id))]
+ (events/publish-event! :event/card-update {:object card :user-id config/internal-mb-user-id}))
+ update-count))
+
+(defn- backfill-entity!
+ [model-kw id target-version]
+ (t2/with-transaction [_]
+ (case model-kw
+ :model/Card (backfill-card! id target-version)
+ (t2/update! model-kw id :dependency_analysis_version [:< target-version]
+ {:dependency_analysis_version target-version}))))
+
+(defn- backfill-entity-batch!
+ [model-kw batch-size]
+ (let [model-name (name model-kw)
+ target-version models.dependency/current-dependency-analysis-version
+ retry-state-map ^Map (retry-state model-kw)
+ terminally-broken-set ^Set (terminally-broken model-kw)
+ ids (get-processable-ids model-kw batch-size)] ; Use the new get-processable-ids
+ (when (seq ids)
+ (log/infof "Processing a batch of %s %s(s)..." (count ids) model-name))
+ (reduce (fn [total id]
+ (+ total
+ (try
+ ;; this should update the dependency table via a toucan2 update hook
+ (let [update-count (backfill-entity! model-kw id target-version)]
+ (.remove retry-state-map id)
+ update-count)
+ (catch Exception e
+ (let [current-time (current-millis)
+ entity-retry-info (.get retry-state-map id)
+ failure-count (inc (:fail-count entity-retry-info 0))
+ retry-minutes (* failure-count (get-delay-minutes))
+ new-next-retry-timestamp (+ current-time (* retry-minutes 60 1000))]
+ (if (> failure-count MAX_RETRIES)
+ (do (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."
+ model-name id failure-count)
+ (.add terminally-broken-set id)
+ (.remove retry-state-map id)) ; Remove from retry map
+ (do (log/warnf e "Entity %s %s failed, failure count: %d, next retry no sooner than %d minutes."
+ model-name id failure-count retry-minutes)
+ (.put retry-state-map id {:fail-count failure-count
+ :next-retry-timestamp new-next-retry-timestamp}))))
+ 0))))
+ 0
+ ids)))
+
+(defn- backfill-dependencies
+ "Job to backfill dependencies for all entities.
+ Returns true if a full batch has been selected, nil or false otherwise."
+ []
+ (when (premium-features/has-feature? :dependencies)
+ (-> (reduce (fn [batch-size model-kw]
+ (if (< batch-size 1)
+ (reduced 0)
+ (let [processed (backfill-entity-batch! model-kw batch-size)]
+ (when (pos? processed)
+ (log/info "Updated" processed "entities."))
+ (- batch-size processed))))
+ (get-batch-size)
+ entities)
+ (< 1))))
+
+(defn- has-pending-retries? []
+ (some (fn [^Map model-retry-state]
+ (not (.isEmpty model-retry-state)))
+ (vals retry-state)))
+
+(declare schedule-next-run!)
+
+(task/defjob
+ ^{:doc "Backfill the dependency table."
+ org.quartz.DisallowConcurrentExecution true}
+ BackfillDependencies [ctx]
+ (log/info "Executing BackfillDependencies job...")
+ (let [full-batch-selected? (backfill-dependencies)
+ retries? (has-pending-retries?)]
+ (if (or full-batch-selected?
+ retries?)
+ (let [delay-seconds (* (get-delay-minutes) 60)
+ variance-seconds (* (get-variance-minutes) 60)
+ delay-in-seconds (max 0 (+ (- delay-seconds variance-seconds) (rand-int (* 2 variance-seconds))))]
+ (schedule-next-run! delay-in-seconds (.getScheduler ctx)))
+ (log/info "No more entities to backfill for, stopping."))))
+
+(def ^:private job-key "metabase.task.dependency-backfill.job")
+(def ^:private trigger-key "metabase.task.dependency-backfill.trigger")
+
+(defn- schedule-next-run!
+ ([delay-in-seconds] (schedule-next-run! delay-in-seconds nil))
+ ([delay-in-seconds scheduler]
+ (let [start-at (java.util.Date. (long (+ (current-millis) (* delay-in-seconds 1000))))
+ trigger (triggers/build
+ (triggers/with-identity (triggers/key (str trigger-key \. (random-uuid))))
+ (triggers/for-job job-key)
+ (triggers/start-at start-at))]
+ (log/info "Scheduling next run at" start-at)
+ (if scheduler
+ ;; re-scheduling from the job
+ (qs/add-trigger scheduler trigger)
+ ;; first schedule
+ (let [job (jobs/build (jobs/of-type BackfillDependencies) (jobs/with-identity job-key))]
+ (task/schedule-task! job trigger))))))
+
+(defmethod task/init! ::DependencyBackfill [_]
+ (if (pos? (get-batch-size))
+ (schedule-next-run! (rand-int (* (get-variance-minutes) 60)))
+ (log/info "Not starting dependency backfill job because the batch size is not positive")))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index d7572128a94..802eb335af7 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -65,6 +65,7 @@
"DashboardBookmark"
"DataPermissions"
"DatabaseRouter"
+ "Dependency"
"DocumentBookmark"
"HTTPAction"
"ImplicitAction"
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
index bfc527b5349..fa920c6bbae 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
@@ -4,6 +4,7 @@
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
+ [metabase.events.core :as events]
[metabase.lib.schema.common :as schema.common]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
@@ -35,6 +36,7 @@
{driver :engine :as database} (t2/select-one :model/Database db)
feature (transforms.util/required-database-feature transform)
transform-details {:db-id db
+ :transform-id id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver database)
:query (transforms.util/compile-source source)
@@ -56,7 +58,9 @@
run-id driver transform-details
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
(transforms.instrumentation/with-stage-timing [run-id :table-sync]
- (transforms.util/sync-target! target database run-id))))
+ (transforms.util/sync-target! target database run-id)
+ ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
+ (events/publish-event! :event/transform-run-complete {:object transform-details}))))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
index 8b5934873c8..7308dceed70 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
@@ -3,6 +3,7 @@
[clojure.set :as set]
[medley.core :as m]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
+ [metabase.events.core :as events]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.util :as u]
@@ -17,7 +18,7 @@
(derive :model/Transform trait))
(t2/deftransforms :model/Transform
- {:source mi/transform-json
+ {:source mi/transform-transform-source
:target mi/transform-json
:run_trigger mi/transform-keyword})
@@ -61,6 +62,18 @@
(for [transform transforms]
(assoc transform :tag_ids (vec (get transform-id->tag-ids (:id transform) [])))))))
+(t2/define-after-insert :model/Transform [transform]
+ (events/publish-event! :event/create-transform {:object transform})
+ transform)
+
+(t2/define-after-update :model/Transform [transform]
+ (events/publish-event! :event/update-transform {:object transform})
+ transform)
+
+(t2/define-before-delete :model/Transform [transform]
+ (events/publish-event! :event/delete-transform {:id (:id transform)})
+ transform)
+
(defn update-transform-tags!
"Update the tags associated with a transform using smart diff logic.
Only modifies what has changed: deletes removed tags, updates positions for moved tags,
@@ -134,7 +147,7 @@
(defmethod serdes/make-spec "Transform"
[_model-name opts]
{:copy [:name :description :entity_id]
- :skip []
+ :skip [:dependency_analysis_version]
:transform {:created_at (serdes/date)
:updated_at (serdes/date)
:source {:export serdes/export-mbql :import serdes/import-mbql}
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index 8b616b41756..301813f16da 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -176,8 +176,8 @@
(defn compile-source
"Compile the source query of a transform."
[{query-type :type :as source}]
- (case query-type
- "query" (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))
+ (case (keyword query-type)
+ :query (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))
(defn required-database-feature
"Returns the database feature necessary to execute `transform`."
diff --git a/enterprise/backend/test/metabase_enterprise/api/session_test.clj b/enterprise/backend/test/metabase_enterprise/api/session_test.clj
index e33028e8e31..344dfd0de24 100644
--- a/enterprise/backend/test/metabase_enterprise/api/session_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/api/session_test.clj
@@ -99,5 +99,6 @@
:database_routing true
:cloud_custom_smtp true
:etl_connections false
- :etl_connections_pg false}
+ :etl_connections_pg false
+ :dependencies false}
(:token-features (mt/user-http-request :crowberto :get 200 "session/properties"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
new file mode 100644
index 00000000000..d761133355a
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj
@@ -0,0 +1,189 @@
+(ns metabase-enterprise.dependencies.core-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [medley.core :as m]
+ [metabase-enterprise.dependencies.core :as dependencies]
+ [metabase-enterprise.dependencies.test-util :as deps.tu]
+ [metabase.graph.core :as graph]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]))
+
+(defn- testbed
+ "A `MetadataProvider` with a chain of MBQL cards and transforms for testing."
+ []
+ (let [mp (deps.tu/default-metadata-provider)
+ query1 (-> (lib/query mp (meta/table-metadata :orders))
+ (lib/expression "Tax Rate" (lib// (meta/field-metadata :orders :tax)
+ (meta/field-metadata :orders :subtotal))))
+ mp (lib.tu/metadata-provider-with-card-from-query mp 101 query1)
+ card1 (lib.metadata/card mp 101)
+ card1-query (lib/query mp card1)
+ card1-cols (lib/returned-columns card1-query)
+ tax-rate (m/find-first #(= (:lib/desired-column-alias %) "Tax Rate") card1-cols)
+ query2 (-> card1-query
+ (lib/filter (lib/> tax-rate 0.06)))
+ mp (lib.tu/metadata-provider-with-card-from-query mp 102 query2)
+
+ ;; A transform that depends on card1
+ tf1-query (-> (lib/query mp card1)
+ lib/->legacy-MBQL)
+ transform1 {:id 30
+ :name "MBQL Transform"
+ :source {:query tf1-query}
+ :target {:schema "Transformed"
+ :name "output_tf30"}}
+ tf1-output (-> (meta/table-metadata :orders)
+ (assoc :id 1234567
+ :schema "Transformed"
+ :name "output_tf30"
+ :display-name "Transform 30 Output"))
+ tf1-cols (map-indexed (fn [i col]
+ (assoc col
+ :id (+ 123456700 i)
+ :table-id 1234567
+ :lib/source :source/table-defaults))
+ card1-cols)
+ mp (lib.tu/mock-metadata-provider mp {:tables [tf1-output]
+ :fields tf1-cols
+ :transforms [transform1]})
+
+ ;; An MBQL card that consumes the output table of transform1.
+ ;; References some fields but does not change the output columns.
+ tf1-consumer (-> (lib/query mp (lib.metadata/table mp 1234567))
+ (lib/filter (lib/= (m/find-first #(= (:name %) "Tax Rate") tf1-cols)
+ 101)))
+ mp (lib.tu/metadata-provider-with-card-from-query mp 301 tf1-consumer)
+
+ ;; SQL cards and snippets
+ ;; Card 7 from [[deps.tu/default-metadata-provider]] uses the "outer" snippet (2).
+ ;; This adds a SQL transform that consumes card 7, and a card that selects from that transform.
+ tf2-query (lib/native-query mp "SELECT * FROM {{#7}}")
+ transform2 {:id 31
+ :name "SQL transform"
+ :source {:query (lib/->legacy-MBQL tf2-query)}
+ :target {:schema "Transformed"
+ :name "output_tf31"}}
+ tf2-output (-> (meta/table-metadata :orders)
+ (assoc :id 1234568
+ :schema "Transformed"
+ :name "output_tf31"
+ :display-name "Transform 31 Output"))
+ ;; Output columns for transform2 are derived from card 7. That's "SELECT * FROM {{snippet: outer}}";
+ ;; the outer snippet references {{snippet: inner}}; the inner snippet is {{#card-ref}} to card 1;
+ ;; card 1 is a basic MBQL query of products.
+ card7 (-> (lib.metadata/card mp 7)
+ (assoc :result-metadata (:result-metadata (:products/native (lib.tu/mock-cards)))))
+ mp (lib.tu/mock-metadata-provider mp {:cards [card7]})
+ tf2-cols (->> card7
+ (lib/query mp)
+ lib/returned-columns
+ (map-indexed (fn [i col]
+ (assoc col
+ :id (+ 123456800 i)
+ :table-id 1234568
+ :lib/source :source/table-defaults))))
+ mp (lib.tu/mock-metadata-provider mp {:tables [tf2-output]
+ :fields tf2-cols
+ :transforms [transform2]})
+
+ ;; MBQL card consuming tf2-output.
+ query3 (-> (lib/query mp tf2-output)
+ (lib/filter (lib/< (m/find-first #(= (:name %) "RATING") tf2-cols)
+ 3)))
+ ;; SQL card consuming tf2-output.
+ query4 (lib/native-query mp "SELECT * FROM Transformed.output_tf31")
+ mp (-> mp
+ (lib.tu/metadata-provider-with-card-from-query 303 query3)
+ (lib.tu/metadata-provider-with-card-from-query 304 query4))]
+ {:provider mp
+ ;; NOTE: This is a *downstream* graph! So it's a map of keys to those entities which depend on them.
+ :graph (graph/in-memory {[:card 101] #{[:card 102]
+ [:transform 30]}
+ [:transform 30] #{[:table 1234567]}
+ [:table 1234567] #{[:card 301]}
+
+ [:snippet 1] #{[:snippet 2]}
+ [:snippet 2] #{[:card 7]}
+ [:card 7] #{[:transform 31]}
+ [:transform 31] #{[:table 1234568]}
+ [:table 1234568] #{[:card 303]
+ [:card 304]}})
+ :mbql-base card1
+ :mbql-dependent (lib.metadata/card mp 102)
+ :mbql-transform transform1
+ :mbql-transform-output tf1-output
+ :mbql-transform-cols (m/index-by :name tf1-cols)
+ :mbql-transform-consumer (lib.metadata/card mp 301)
+
+ :snippet-inner (lib.metadata/native-query-snippet mp 1)
+ :snippet-outer (lib.metadata/native-query-snippet mp 2)
+ :sql-base (lib.metadata/card mp 7)
+ :sql-transform transform2
+ :sql-transform-output tf2-output
+ :sql-transform-cols (m/index-by :name tf2-cols)
+ :sql-transform-mbql-consumer (lib.metadata/card mp 303)
+ :sql-transform-sql-consumer (lib.metadata/card mp 304)}))
+
+(comment
+ *e
+ (:mbql-dependent (testbed))
+ (:mbql-transform-consumer (testbed)))
+
+(deftest ^:parallel basic-mbql-card-test
+ (testing "when changing an MBQL card with dependents"
+ (let [{:keys [provider graph mbql-base]
+ {tax-rate "Tax Rate"} :mbql-transform-cols
+ {downstream-card-id :id} :mbql-dependent
+ {transformed-card-id :id} :mbql-transform-consumer} (testbed)]
+ (testing "a column that no longer exists will cause errors when referenced"
+ (let [card' (-> mbql-base
+ (update-in [:dataset-query :query :expressions]
+ update-keys (constantly "Sales Taxes"))
+ (dissoc :result-metadata))
+ errors (dependencies/errors-from-proposed-edits provider graph {:card [card']})]
+ (is (=? {:card {downstream-card-id [[:field {} "Tax Rate"]]
+ transformed-card-id [[:field {} (:id tax-rate)]]}}
+ errors))
+ (is (= [:card] (keys errors)))
+ (is (= #{downstream-card-id transformed-card-id} (set (keys (:card errors)))))))
+
+ (testing "changing something unrelated will cause no errors"
+ (let [card' (-> mbql-base
+ (assoc-in [:dataset-query :query :filter]
+ [:> [:field (meta/id :orders :quantity) nil] 100])
+ (dissoc :result-metadata))]
+ (is (= {} (dependencies/errors-from-proposed-edits provider graph {:card [card']}))))))))
+
+(deftest ^:parallel sql-snippet->card->transform->cards-test
+ (testing "changing a snippet correctly finds downstream errors"
+ (let [{:keys [provider graph sql-transform snippet-inner sql-transform-cols]
+ {direct-sql-card-id :id} :sql-base
+ {transformed-sql-card-id :id} :sql-transform-sql-consumer
+ {transformed-mbql-card-id :id} :sql-transform-mbql-consumer} (testbed)]
+ (testing "when breaking the inner snippet with a nonexistent table"
+ (let [snippet' (assoc snippet-inner
+ :content "nonexistent_table"
+ :template-tags {})
+ rating (get sql-transform-cols "RATING")
+ errors (dependencies/errors-from-proposed-edits provider graph {:snippet [snippet']})]
+ ;; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL
+ ;; queries that consume the transform's table.
+ (is (=? {:card {direct-sql-card-id [{:table {:table "NONEXISTENT_TABLE"},
+ :type :all-columns,
+ :metabase.driver.sql/bad-reference true}]
+ transformed-sql-card-id [{:table {:schema "TRANSFORMED", :table "OUTPUT_TF31"},
+ :type :all-columns,
+ :metabase.driver.sql/bad-reference true}]
+ transformed-mbql-card-id [[:field {} (:id rating)]]}
+ :transform {(:id sql-transform) [{:table {:table "NONEXISTENT_TABLE"},
+ :type :all-columns,
+ :metabase.driver.sql/bad-reference true}]}}
+ errors))
+ (is (= #{:card :transform}
+ (set (keys errors))))
+ (is (= #{direct-sql-card-id transformed-sql-card-id transformed-mbql-card-id}
+ (set (keys (:card errors)))))
+ (is (= #{(:id sql-transform)}
+ (set (keys (:transform errors))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_provider_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_provider_test.clj
new file mode 100644
index 00000000000..e108a389f64
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_provider_test.clj
@@ -0,0 +1,131 @@
+(ns metabase-enterprise.dependencies.metadata-provider-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [medley.core :as m]
+ [metabase-enterprise.dependencies.metadata-provider :as deps.mp]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
+ [metabase.lib.test-util.macros :as lib.tu.macros]))
+
+(deftest basic-cards-test
+ (testing "overriding a single card, it is returned correctly"
+ (let [card (:orders (lib.tu/mock-cards))
+ mp (-> card
+ lib.tu/metadata-provider-with-mock-card
+ deps.mp/override-metadata-provider)]
+ (deps.mp/add-override mp :card (:id card)
+ (-> card
+ (assoc-in [:dataset-query :query :fields]
+ (lib.tu.macros/$ids orders
+ [$subtotal $created-at]))
+ (dissoc :result-metadata)))
+ (is (=? [(meta/field-metadata :orders :subtotal)
+ (meta/field-metadata :orders :created-at)]
+ (->> (lib.metadata/card mp (:id card))
+ (lib/query mp)
+ lib/returned-columns))))))
+
+(defn- replace-query [new-query card]
+ (-> card
+ (assoc :dataset-query (lib/->legacy-MBQL new-query))
+ (dissoc :result-metadata)))
+
+(deftest upstream-card-test
+ (testing "overriding an upstream card that's consumed by a downstream card"
+ (let [upstream (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/breakout (meta/field-metadata :products :category))
+ (lib/aggregate (lib/count)))
+ mp1 (lib.tu/metadata-provider-with-card-from-query 1 upstream)
+ card1 (lib.metadata/card mp1 1)
+ by-name (m/index-by :name (lib/returned-columns (lib/query mp1 card1)))
+ downstream (-> (lib/query mp1 card1)
+ (lib/filter (lib/= (get by-name "count") 1)))
+ mp (-> (lib.tu/metadata-provider-with-card-from-query mp1 2 downstream)
+ deps.mp/override-metadata-provider)]
+ (deps.mp/add-override mp :card 1
+ (-> upstream
+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :day))
+ (replace-query card1)))
+ ;; NOTE: This line is important: this is what would happen to all the dependents returned by the graph.
+ ;; There's no update (`nil`) but it still includes the downstream stuff in the scope of the
+ ;; OverrideMetadataProvider.
+ (deps.mp/add-override mp :card 2 nil)
+ (testing "upstream card's columns are correct"
+ (is (=? [{:name "CATEGORY"}
+ {:name "CREATED_AT"}
+ {:name "count"}]
+ (->> (lib.metadata/card mp 1)
+ (lib/query mp)
+ lib/returned-columns))))
+ (testing "downstream card (which selects all) is also correct"
+ (is (=? [{:name "CATEGORY"}
+ {:name "CREATED_AT"}
+ {:name "count"}]
+ (->> (lib.metadata/card mp 2)
+ (lib/query mp)
+ lib/returned-columns)))))))
+
+(deftest card->transform->card-test
+ (testing "changing the columns of an upstream card that is transformed and then used by a card"
+ (let [upstream (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/breakout (meta/field-metadata :products :category))
+ (lib/aggregate (lib/count)))
+ mp1 (lib.tu/metadata-provider-with-card-from-query 1 upstream)
+ card1 (lib.metadata/card mp1 1)
+ card1-cols (-> (lib/query mp1 card1)
+ lib/returned-columns)
+
+ transform {:lib/type :metadata/transform
+ :id 123
+ :name "The Transform"
+ :source {:type :query
+ :query (-> (lib/query mp1 (lib.metadata/card mp1 1))
+ lib/->legacy-MBQL)}
+ :target {:type "table"
+ :schema "transform_schema_123"
+ :name "autobots"}}
+ mp2 (lib.tu/mock-metadata-provider
+ mp1 {:tables [{:id 123000000
+ :lib/type :metadata/table
+ :name "autobots"
+ :schema "transform_schema_123"
+ :db-id (meta/id)
+ :visibility-type nil
+ :active true}]
+ :fields (map-indexed (fn [i col]
+ (assoc col
+ :id (+ 123000001 i)
+ :table-id 123000000))
+ card1-cols)})
+
+ xform-cols (m/index-by :name (lib.metadata/fields mp2 123000000))
+ downstream (-> (lib/query mp2 (lib.metadata/table mp2 123000000))
+ (lib/filter (lib/= (get xform-cols "count") 1)))
+ mp (-> (lib.tu/metadata-provider-with-card-from-query mp2 2 downstream)
+ deps.mp/override-metadata-provider)]
+ (deps.mp/add-override mp :card 1
+ (-> upstream
+ (lib/remove-clause (first (lib/breakouts upstream)))
+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :day))
+ (replace-query card1)))
+ ;; NOTE: Theses lines are important: this is what would happen to all the dependents returned by the graph.
+ ;; There's no update (`nil`) but it still includes the downstream stuff in the scope of the
+ ;; OverrideMetadataProvider.
+ (deps.mp/add-override mp :card 2 nil)
+ (deps.mp/add-override mp :transform 123 transform)
+
+ (testing "upstream card's columns are correct"
+ (is (=? [{:name "CREATED_AT"}
+ {:name "count"}]
+ (->> (lib.metadata/card mp 1)
+ (lib/query mp)
+ lib/returned-columns))))
+ (testing "downstream card (which selects all) is also correct"
+ (is (=? [{:name "CREATED_AT"}
+ {:name "count"}]
+ (->> (lib.metadata/card mp 2)
+ (lib/query mp)
+ lib/returned-columns
+ (sort-by :name))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
new file mode 100644
index 00000000000..af57d29840f
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
@@ -0,0 +1,216 @@
+(ns metabase-enterprise.dependencies.models.dependency-test
+ (:require
+ [clojure.set :as set]
+ [clojure.test :refer [deftest is testing]]
+ [metabase-enterprise.dependencies.models.dependency :as deps.graph]
+ [metabase.lib.core :as lib]
+ [metabase.queries.models.card :as card]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(defn- depends-on-> [from-type from-id to-type to-id]
+ {:from_entity_type from-type
+ :from_entity_id from-id
+ :to_entity_type to-type
+ :to_entity_id to-id})
+
+(defn- upstream-of [from-type from-id]
+ (t2/select-fn-set #(select-keys % [:from_entity_type :from_entity_id :to_entity_type :to_entity_id])
+ :model/Dependency :from_entity_type from-type :from_entity_id from-id))
+
+(defn- basic-orders []
+ {:name "Test card"
+ :database_id (mt/id)
+ :table_id (mt/id :orders)
+ :display :table
+ :query_type :query
+ :type :question
+ :dataset_query (mt/mbql-query orders)
+ :visualization_settings {}})
+
+(defn- wrap-card [inner-card]
+ {:name "Downstream card"
+ :database_id (mt/id)
+ :display :table
+ :query_type :query
+ :type :question
+ :dataset_query (mt/mbql-query nil
+ {:source-table (str "card__" (:id inner-card))})
+ :visualization_settings {}})
+
+(deftest ^:sequential card-deps-maintenance-test-1-new-card
+ (testing "upstream deps of a card are updated correctly"
+ (mt/dataset test-data
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (let [card1 (card/create-card! (basic-orders) user)]
+ (is (integer? (:id card1)))
+ (testing "when creating a new card"
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
+ (upstream-of :card (:id card1))))
+
+ (testing "that depends on another card"
+ (let [card2 (card/create-card! (wrap-card card1) user)]
+ (is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}
+ (upstream-of :card (:id card2))))
+ (testing "but that doesn't affect the upstream deps of the inner card"
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
+ (upstream-of :card (:id card1))))))))
+
+ (testing "when updating an existing card"
+ (testing "to add a new table dep"
+ (card/update-card! {:card-before-update card1
+ :card-updates
+ {:dataset_query (mt/mbql-query orders
+ {:joins [{:alias "Products"
+ :source-table (mt/id :products)
+ :condition [:= $id &Products.$products.id]}]})}})
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))
+ (depends-on-> :card (:id card1) :table (mt/id :products))}
+ (upstream-of :card (:id card1)))))
+ (testing "to remove a table dep"
+ (card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))
+ :card-updates
+ {:dataset_query (mt/mbql-query products)}})
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}
+ (upstream-of :card (:id card1))))))))))))
+
+(deftest ^:sequential card-deps-graph-test-1-mbql-card-chain
+ (testing "deps graph is connected properly for a chain of MBQL cards"
+ (mt/dataset test-data
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)
+ {id3 :id :as card3} (card/create-card! (wrap-card card2) user)]
+ (testing "raw deps are recorded correctly"
+ (is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
+ (is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))
+ (is (=? #{(depends-on-> :card id3 :card id2)} (upstream-of :card id3))))
+
+ (testing "transitive deps are computed correctly"
+ (testing "for each card"
+ (is (=? {:card #{id2 id3}}
+ (deps.graph/transitive-dependents {:card [card1]})))
+ (is (=? {:card #{id3}}
+ (deps.graph/transitive-dependents {:card [card2]})))
+ (is (=? {}
+ (deps.graph/transitive-dependents {:card [card3]}))))
+ (testing "for the table"
+ (is (set/subset? #{id1 id2 id3}
+ (-> {:table [{:id (mt/id :orders)}]}
+ deps.graph/transitive-dependents
+ :card)))))))))))
+
+(defn- sql-card [metadata-provider sql]
+ {:name "SQL card"
+ :database_id (mt/id)
+ :display :table
+ :query_type :query
+ :type :question
+ :dataset_query (-> (lib/native-query metadata-provider sql)
+ lib/->legacy-MBQL)
+ :visualization_settings {}})
+
+(deftest ^:sequential card-deps-graph-test-2-native-card-chain
+ (testing "deps graph is connected properly for a chain of native cards"
+ (mt/dataset test-data
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (let [mp (mt/metadata-provider)
+ {id1 :id :as card1} (card/create-card! (sql-card mp "SELECT * FROM orders;") user)
+ {id2 :id :as card2} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id1 "}}")) user)
+ {id3 :id :as card3} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id2 "}}")) user)]
+ (testing "raw deps are recorded correctly"
+ (is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
+ (is (=? #{(depends-on-> :card id2 :table (mt/id :orders))
+ (depends-on-> :card id2 :card id1)} (upstream-of :card id2)))
+ (is (=? #{(depends-on-> :card id3 :table (mt/id :orders))
+ (depends-on-> :card id3 :card id2)} (upstream-of :card id3))))
+ (testing "transitive deps are computed correctly"
+ (testing "for each card"
+ (is (=? {:card #{id2 id3}}
+ (deps.graph/transitive-dependents {:card [card1]})))
+ (is (=? {:card #{id3}}
+ (deps.graph/transitive-dependents {:card [card2]})))
+ (is (=? {}
+ (deps.graph/transitive-dependents {:card [card3]}))))
+ (testing "for the table"
+ (is (set/subset? #{id1 id2 id3}
+ (-> {:table [{:id (mt/id :orders)}]}
+ deps.graph/transitive-dependents
+ :card)))))))))))
+
+(deftest ^:sequential card-deps-graph-metric-test
+ (testing "deps graph is connected properly for a question using a metric"
+ (mt/dataset test-data
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (let [metric-card (card/create-card! {:name "Test Metric"
+ :database_id (mt/id)
+ :table_id (mt/id :orders)
+ :display :scalar
+ :query_type :query
+ :type :metric
+ :dataset_query (mt/mbql-query orders {:aggregation [[:count]]})
+ :visualization_settings {}}
+ user)
+ question-card (card/create-card! {:name "Question using Metric"
+ :database_id (mt/id)
+ :display :table
+ :query_type :query
+ :type :question
+ :dataset_query (mt/mbql-query orders
+ {:aggregation [[:metric (:id metric-card)]]})
+ :visualization_settings {}}
+ user)]
+ (testing "raw deps are recorded correctly for question using metric"
+ (is (=? #{(depends-on-> :card (:id question-card) :table (mt/id :orders))
+ (depends-on-> :card (:id question-card) :card (:id metric-card))}
+ (upstream-of :card (:id question-card)))))))))))
+
+(deftest dependency-analysis-version-test
+ (testing "dependency_analysis_version is updated when entities are created or updated"
+ (testing "cards"
+ (mt/with-temp [:model/Card card {:dependency_analysis_version 0}]
+ (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))
+ (mt/with-premium-features #{:dependencies}
+ (card/update-card! {:card-before-update card
+ :card-updates {:dataset_query (mt/mbql-query orders)}}))
+ (is (= deps.graph/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))))
+ (testing "transforms create"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Transform transform]
+ (is (= deps.graph/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))
+ (testing "transforms update"
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Transform transform]
+ (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform))))
+ (mt/with-premium-features #{:dependencies}
+ (t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}}))
+ (is (= deps.graph/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))
+ (testing "snippets create"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/NativeQuerySnippet snippet]
+ (is (= deps.graph/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))
+ (testing "snippets update"
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/NativeQuerySnippet snippet]
+ (is (zero? (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet))))
+ (mt/with-premium-features #{:dependencies}
+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))
+ (is (= deps.graph/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))))
+
+(deftest dependency-analysis-version-create-card-test
+ (testing "dependency_analysis_version is updated when a card is created"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/Card]
+ (mt/with-temp [:model/User user]
+ (let [card (card/create-card! (basic-orders) user)]
+ (is (= deps.graph/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Card :id (:id card))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
new file mode 100644
index 00000000000..a6e802b975e
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
@@ -0,0 +1,241 @@
+(ns metabase-enterprise.dependencies.native-validation-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.dependencies.native-validation :as deps.native-validation]
+ [metabase-enterprise.dependencies.test-util :as deps.tu]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-metadata :as meta]))
+
+(defn- fake-query
+ ([mp query]
+ (fake-query mp query {}))
+ ([mp query template-tags]
+ {:database (:id (lib.metadata/database mp))
+ :type :native
+ :native {:query query
+ :template-tags template-tags}}))
+
+(defn- validates?
+ [mp driver card-id expected]
+ (is (=? expected
+ (->> (lib.metadata/card mp card-id)
+ :dataset-query
+ (deps.native-validation/validate-native-query driver mp)))))
+
+(deftest basic-deps-test
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (is (= #{{:table (meta/id :products)}}
+ (->> (lib.metadata/card mp 4)
+ :dataset-query
+ (deps.native-validation/native-query-deps driver mp))))
+ (is (= #{{:table (meta/id :products)}
+ {:card 1}}
+ (->> (lib.metadata/card mp 5)
+ :dataset-query
+ (deps.native-validation/native-query-deps driver mp))))
+ (is (= #{{:table (meta/id :products)}
+ {:card 1}
+ {:snippet 1}}
+ (->> (lib.metadata/card mp 6)
+ :dataset-query
+ (deps.native-validation/native-query-deps driver mp))))
+ (is (= #{{:table (meta/id :products)}
+ {:card 1}
+ {:snippet 1}
+ {:snippet 2}}
+ (->> (lib.metadata/card mp 7)
+ :dataset-query
+ (deps.native-validation/native-query-deps driver mp))))
+ (is (= #{{:table (meta/id :products)}
+ {:table (meta/id :orders)}
+ {:card 1}
+ {:card 2}}
+ (->> (lib.metadata/card mp 9)
+ :dataset-query
+ (deps.native-validation/native-query-deps driver mp))))))
+
+(deftest validate-bad-queries-test
+ (testing "validate-native-query handles nonsense queries"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "complete nonsense query"
+ (is (= [{:error :bad-sql}]
+ (deps.native-validation/validate-native-query
+ driver mp
+ (fake-query mp "this is not a query")))))
+ (testing "bad table wildcard"
+ (is (= [{:type :invalid-table-wildcard,
+ :table "products",
+ :metabase.driver.sql/bad-reference true}]
+ (deps.native-validation/validate-native-query
+ driver mp
+ (fake-query mp "select products.* from orders")))))
+ (testing "bad col reference"
+ (is (= [{:column "BAD",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "PRODUCTS"}}]],
+ :metabase.driver.sql/bad-reference true}]
+ (deps.native-validation/validate-native-query
+ driver mp
+ (fake-query mp "select bad from products"))))))))
+
+(deftest validate-native-query-with-subquery-columns-test
+ (testing "validate-native-query should detect invalid columns in subqueries"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+
+ (testing "Valid query - selecting existing columns from subquery"
+ (validates? mp driver 10 empty?))
+
+ (testing "Invalid query - selecting non-existent column from subquery"
+ (validates? mp driver 11 [{:column "CATEGORY",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]],
+ :metabase.driver.sql/bad-reference true}])
+ (validates? mp driver 12 [{:column "CATEGORY",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]],
+ :metabase.driver.sql/bad-reference true}]))
+
+ (testing "Nested subqueries"
+ (validates? mp driver 13 empty?)
+ (validates? mp driver 14 [{:column "CATEGORY",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]],
+ :metabase.driver.sql/bad-reference true}]))
+
+ (testing "SELECT * from subquery expands to subquery columns"
+ (validates? mp driver 15 empty?)
+ (validates? mp driver 16 empty?)
+ (validates? mp driver 17 [{:column "EMAIL",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]],
+ :metabase.driver.sql/bad-reference true}])))))
+
+(deftest validate-card-reference-after-expansion-test
+ (testing "Validation of queries after card references have been expanded"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+
+ (testing "Card reference expanded to subquery - valid columns"
+ (validates? mp driver 18 empty?))
+
+ (testing "Card reference expanded to subquery - invalid column"
+ (validates? mp driver 19
+ [{:column "DESCRIPTION",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]],
+ :metabase.driver.sql/bad-reference true}]))
+
+ (testing "Card reference with alias - valid column"
+ (validates? mp driver 20 empty?))
+
+ (testing "Card reference with alias - invalid column"
+ (validates? mp driver 21
+ [{:column "PASSWORD",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]],
+ :metabase.driver.sql/bad-reference true}]))
+
+ (testing "Wildcard selection from card reference"
+ (validates? mp driver 22 empty?))
+
+ (testing "Invalid column from aliased card"
+ (validates? mp driver 23
+ [{:column "LATITUDE",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]],
+ :metabase.driver.sql/bad-reference true}])))))
+
+(defn- check-result-metadata [driver mp query expected]
+ (is (=? expected
+ (->> query
+ (fake-query mp)
+ (deps.native-validation/native-result-metadata driver mp)))))
+
+(defn- add-desired-column-alias [fields]
+ (map #(assoc % :lib/desired-column-alias (:name %)) fields))
+
+(deftest result-metadata-test
+ (testing "Calculates result metadata"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (testing "Selecting a wildcard"
+ (check-result-metadata
+ driver mp
+ "select * from orders"
+ (add-desired-column-alias (lib.metadata/fields mp (meta/id :orders)))))
+ (testing "Selecting a table wildcard"
+ (check-result-metadata
+ driver mp
+ "select orders.* from orders"
+ (add-desired-column-alias (lib.metadata/fields mp (meta/id :orders)))))
+ (testing "Selecting a single col"
+ (check-result-metadata
+ driver mp
+ "select total from orders"
+ (add-desired-column-alias [(lib.metadata/field mp (meta/id :orders :total))])))
+ (testing "Selecting a nonexistent col"
+ (check-result-metadata
+ driver mp
+ "select bad from orders"
+ []))
+ (testing "Selecting a col with an alias"
+ (check-result-metadata
+ driver mp
+ "select subtotal as new_subtotal from orders"
+ [{:lib/type :metadata/column,
+ :base-type :type/Float,
+ :semantic-type nil,
+ :name "SUBTOTAL",
+ :lib/desired-column-alias "NEW_SUBTOTAL"
+ :database-type "DOUBLE PRECISION",
+ :display-name "Subtotal"}]))
+ (testing "Selecting a custom col"
+ (check-result-metadata
+ driver mp
+ "select subtotal + tax as sum from orders"
+ [{:base-type :type/*,
+ :name "SUM",
+ :display-name "Sum",
+ :effective-type :type/*,
+ :semantic-type :Semantic/*}]))
+ (testing "Selecting a union"
+ (check-result-metadata
+ driver mp
+ "select total from orders union select subtotal from orders"
+ [{:name "TOTAL",
+ :lib/desired-column-alias "TOTAL",
+ :display-name "Total",
+ :base-type :type/Float,
+ :effective-type :type/Float,
+ :semantic-type :Semantic/*}]))
+ (testing "Selecting a union with different types"
+ (check-result-metadata
+ driver mp
+ "select category from products union select title from products"
+ [{:name "CATEGORY",
+ :lib/desired-column-alias "CATEGORY",
+ :display-name "Category",
+ :base-type :type/Text,
+ :effective-type :type/Text,
+ :semantic-type :type/Category}]))
+ (testing "Selecting a bad table wildcard"
+ (check-result-metadata
+ driver mp
+ "select orders.* from products"
+ []))
+ (testing "Using a nonsense query"
+ (check-result-metadata
+ driver mp
+ "this is not a query"
+ [])))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
new file mode 100644
index 00000000000..cd8c1aa223f
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
@@ -0,0 +1,231 @@
+(ns metabase-enterprise.dependencies.task.backfill-test
+ (:require
+ [clojure.test :refer :all]
+ [environ.core :as env]
+ [java-time.api :as t]
+ [metabase-enterprise.dependencies.models.dependency :as dependencies.model]
+ [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
+ [metabase.task.core :as task]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(defn- backfill-all-existing-entities
+ []
+ (mt/with-premium-features #{:dependencies}
+ (while (#'dependencies.backfill/backfill-dependencies))))
+
+(defn- backfill-dependencies-single-trigger
+ []
+ (mt/with-premium-features #{:dependencies}
+ (#'dependencies.backfill/backfill-dependencies)))
+
+(deftest ^:sequential backfill-dependency-analysis-test
+ (testing "Test that the backfill job correctly updates the dependency_analysis_version"
+ (backfill-all-existing-entities)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+ (let [query (mt/mbql-query orders)]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}
+ :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}
+ :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]
+ (let [card-count (fn [& args]
+ (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
+ current-version dependencies.model/current-dependency-analysis-version]
+ (is (= 3 (card-count)))
+ ;; first run, should process 2 cards
+ (is (true? (backfill-dependencies-single-trigger)))
+ (is (= 2 (card-count :dependency_analysis_version current-version)))
+ ;; second run, should process the last card
+ (is (false? (backfill-dependencies-single-trigger)))
+ (is (= 3 (card-count :dependency_analysis_version current-version)))
+ ;; third run, should not process anything
+ (is (false? (backfill-dependencies-single-trigger)))
+ (is (= 3 (card-count :dependency_analysis_version current-version))))))))))
+
+(deftest ^:sequential backfill-transform-test
+ (testing "Test that transforms are correctly backfilled"
+ (backfill-all-existing-entities)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)
+ :dependency_analysis_version 0}
+ :model/Transform {transform-id :id} {:name "Test Transform"
+ :source {:type "query"
+ :query (mt/mbql-query nil
+ {:source-table (str "card__" card-id)})}
+ :target {:type "table" :name "test_table"}
+ :dependency_analysis_version 0}]
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ (is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))
+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))
+ (is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))
+ (backfill-dependencies-single-trigger)
+ (is (t2/exists? :model/Card :id card-id
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (is (t2/exists? :model/Transform :id transform-id
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders)))
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :transform :from_entity_id transform-id
+ :to_entity_type :card :to_entity_id card-id)))))))
+
+(deftest ^:sequential backfill-snippet-test
+ (testing "Test that cards with snippets are correctly backfilled"
+ (backfill-all-existing-entities)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"
+ :content "SELECT 1"
+ :dependency_analysis_version 0}
+ :model/Card {card-id :id} {:dataset_query {:database (mt/id)
+ :type :native
+ :native {:query "SELECT * FROM {{my_snippet}}"
+ :template-tags
+ {:my_snippet
+ {:id (str (random-uuid))
+ :name "my_snippet"
+ :display-name "my_snippet"
+ :type :snippet
+ :snippet-name "my_snippet"
+ :snippet-id snippet-id}}}}
+ :dependency_analysis_version 0}]
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ (is (t2/exists? :model/NativeQuerySnippet :id snippet-id :dependency_analysis_version 0))
+ (is (false? (t2/exists? :model/Dependency
+ :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :snippet :to_entity_id snippet-id)))
+ (backfill-dependencies-single-trigger)
+ (is (t2/exists? :model/Card :id card-id
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (is (t2/exists? :model/NativeQuerySnippet :id snippet-id
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :snippet :to_entity_id snippet-id)))))))
+
+(deftest ^:sequential backfill-idempotency-test
+ (testing "Running the backfill multiple times should be idempotent"
+ (backfill-all-existing-entities)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)
+ :dependency_analysis_version 0}]
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders))))
+ ;; First run
+ (backfill-dependencies-single-trigger)
+ (is (t2/exists? :model/Card :id card-id
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ ;; Second run - should not change anything
+ (backfill-dependencies-single-trigger)
+ (is (t2/exists? :model/Card :id card-id
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders))))))))
+
+(defn- wait-for-condition
+ [predicate timeout-ms]
+ (let [limit (+ (System/currentTimeMillis) timeout-ms)]
+ (loop []
+ (Thread/sleep 100)
+ (or (predicate)
+ (when (< (System/currentTimeMillis) limit)
+ (recur))))))
+
+(deftest ^:sequential backfill-scheduling-test
+ (testing "Test that the backfill job schedules and reschedules itself correctly"
+ (backfill-all-existing-entities)
+ (mt/with-temp-scheduler!
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-batch-size "1"
+ :mb-dependency-backfill-delay-minutes "0"
+ :mb-dependency-backfill-variance-minutes "0")]
+ (let [card-data {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card1-id :id} card-data
+ :model/Card {card2-id :id} card-data
+ :model/Card {card3-id :id} card-data]
+ (let [card-count (fn [& args]
+ (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
+ current-version dependencies.model/current-dependency-analysis-version]
+ (is (= 3 (card-count :dependency_analysis_version 0)))
+ (mt/with-premium-features #{:dependencies}
+ ;; Initialize the task, which should schedule the first run
+ (task/init! ::dependencies.backfill/DependencyBackfill)
+ (wait-for-condition #(= 3 (card-count :dependency_analysis_version current-version)) 10000)
+ (is (= 3 (card-count :dependency_analysis_version current-version)))))))))))))
+
+(deftest ^:sequential backfill-terminal-failure-test
+ (testing "Entities should be marked as terminally broken after MAX_RETRIES failures"
+ (backfill-all-existing-entities)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+ (let [update-attempts (volatile! 0)
+ failures (inc @#'dependencies.backfill/MAX_RETRIES)]
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-delay-minutes "0"
+ :mb-dependency-backfill-variance-minutes "0")
+ t2/update! (fn [model-kw id & args]
+ (if (and (= model-kw :model/Card)
+ (= id card-id)
+ (< @update-attempts failures))
+ (do
+ (vswap! update-attempts inc)
+ (throw (ex-info "Simulated DB error" {:id id})))
+ (apply t2/update! model-kw id args)))]
+ ;; fail MAX_RETRIES + 1 times
+ (while (< @update-attempts failures)
+ (backfill-dependencies-single-trigger))))
+
+ ;; verify card is not processed and is terminally broken
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+ ;; verify subsequent runs don't process it
+ (backfill-dependencies-single-trigger)
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))))))
+
+(deftest ^:sequential backfill-delayed-retry-test
+ (testing "Failed entities should be retried after their delay period expires"
+ (backfill-all-existing-entities)
+ (let [current-version dependencies.model/current-dependency-analysis-version
+ update-attempts (volatile! 0)]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-delay-minutes "1") ; 1 minute delay for retries
+ t2/update! (fn [model-kw id & args]
+ (if (and (= model-kw :model/Card)
+ (= id card-id)
+ (zero? @update-attempts))
+ (do
+ (vswap! update-attempts inc)
+ (throw (ex-info "Simulated DB error" {:id id})))
+ (apply t2/update! model-kw id args)))]
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+ ;; first failure - should be put into retry state
+ ;; fail MAX_RETRIES + 1 times
+ (while (zero? @update-attempts)
+ (backfill-dependencies-single-trigger))
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0)))
+
+ ;; advance time by less than retry delay - should NOT be processed
+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))
+ (backfill-dependencies-single-trigger))
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+ ;; advance time by more than retry delay - should be processed
+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))
+ (backfill-dependencies-single-trigger))
+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version current-version)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
new file mode 100644
index 00000000000..57bd8f407b7
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
@@ -0,0 +1,125 @@
+(ns metabase-enterprise.dependencies.test-util
+ (:require
+ [medley.core :as m]
+ [metabase.lib.convert :as lib.convert]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util.metadata-providers.mock :as providers.mock]))
+
+(defn mock-card [metadata-provider {:keys [id query details]}]
+ (merge {:lib/type :metadata/card
+ :id id
+ :database-id (:id (lib.metadata/database metadata-provider))
+ :dataset-query (if (string? query)
+ {:database (:id (lib.metadata/database metadata-provider))
+ :type :native
+ :native {:query query
+ :template-tags (lib/extract-template-tags metadata-provider query)}}
+ (lib.convert/->legacy-MBQL query))
+ :name (str (gensym))
+ :result-metadata (when (not (string? query))
+ (->> (lib/returned-columns query)
+ (sort-by :id)))}
+ details))
+
+(defn mock-snippet [{:keys [id name content template-tags]}]
+ {:description nil,
+ :archived false,
+ :collection-id nil,
+ :id id
+ :name name
+ :content content
+ :template-tags template-tags
+ :lib/type :metadata/native-query-snippet})
+
+(defn mock-metadata-provider [{:keys [cards snippets]}]
+ (lib/composed-metadata-provider
+ meta/metadata-provider
+ (providers.mock/mock-metadata-provider
+ {:cards cards
+ :native-query-snippets snippets})))
+
+(defn default-metadata-provider []
+ (let [snippets (map mock-snippet
+ [{:id 1
+ :name "inner"
+ :content "{{#card-ref}}"
+ :template-tags {"#card-ref" {:card-id 1
+ :type :card
+ :name "#card-ref"
+ :display-name "#Card Ref"
+ :id (str (random-uuid))}}}
+ {:id 2
+ :name "outer"
+ :content "{{snippet: inner}}"
+ :template-tags {"snippet: inner" {:snippet-id 1
+ :snippet-name "inner"
+ :type :snippet
+ :name "snippet: inner"
+ :display-name "Snippet: Inner"
+ :id (str (random-uuid))}}}])
+ initial-mock-mp (mock-metadata-provider {:snippets snippets})
+ initial-card-a (mock-card initial-mock-mp
+ {:id 1
+ :query (lib/query initial-mock-mp (meta/table-metadata :products))})
+ initial-card-b (mock-card initial-mock-mp
+ {:id 2
+ :query (lib/query initial-mock-mp (meta/table-metadata :orders))})]
+ (mock-metadata-provider
+ {:cards (into [initial-card-a
+ initial-card-b]
+ (map #(mock-card initial-mock-mp %))
+ [{:id 3
+ :query (lib/query initial-mock-mp initial-card-a)}
+ {:id 4
+ :query "select * from products"}
+ {:id 5
+ :query "select * from {{#1}}"}
+ {:id 6
+ :query "select * from {{snippet: inner}}"}
+ {:id 7
+ :query "select * from {{snippet: outer}}"}
+ {:id 8
+ :query (as-> (lib/query initial-mock-mp initial-card-b) q
+ (lib/join q (-> (lib/join-clause initial-card-a)
+ (lib/with-join-conditions
+ [(lib/=
+ (m/find-first #(= (:name %) "PRODUCT_ID")
+ (lib/join-condition-lhs-columns
+ q initial-card-b nil nil))
+ (m/find-first #(= (:name %) "ID")
+ (lib/join-condition-lhs-columns
+ q initial-card-a nil nil)))])
+ (lib/with-join-fields :all))))}
+ {:id 9
+ :query "select * from {{#1}} inner join {{#2}}"}
+ {:id 10
+ :query "SELECT id, name FROM (SELECT id, name, email FROM PEOPLE)"}
+ {:id 11
+ :query "SELECT category FROM (SELECT id, name FROM PEOPLE)"}
+ {:id 12
+ :query "SELECT id, category FROM (SELECT id, name FROM PEOPLE)"}
+ {:id 13
+ :query "SELECT id FROM (SELECT id, name FROM (SELECT id, name, email FROM PEOPLE))"}
+ {:id 14
+ :query "SELECT category FROM (SELECT id, name FROM (SELECT id, name, email FROM PEOPLE))"}
+ {:id 15
+ :query "SELECT * FROM (SELECT id, name FROM PEOPLE)"}
+ {:id 16
+ :query "SELECT name FROM (SELECT * FROM (SELECT id, name FROM PEOPLE))"}
+ {:id 17
+ :query "SELECT email FROM (SELECT * FROM (SELECT id, name FROM PEOPLE))"}
+ {:id 18
+ :query "SELECT ID, TITLE FROM {{#1}}"}
+ {:id 19
+ :query "SELECT DESCRIPTION FROM {{#1}}"}
+ {:id 20
+ :query "SELECT ID FROM {{#1}} AS p"}
+ {:id 21
+ :query "SELECT PASSWORD FROM {{#1}} AS p"}
+ {:id 22
+ :query "SELECT * FROM {{#1}}"}
+ {:id 23
+ :query "SELECT p.LATITUDE FROM {{#1}} AS p"}])
+ :snippets snippets})))
diff --git a/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj
index b080f26e0ed..18f036505b9 100644
--- a/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj
@@ -47,6 +47,7 @@
:model/DashboardBookmark
:model/DataPermissions
:model/DatabaseRouter
+ :model/Dependency
:model/DocumentBookmark
:model/CollectionPermissionGraphRevision
:model/DashboardCardSeries
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index 79bd31a85bd..d91fc3a11b5 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -224,8 +224,14 @@
:model/Transform child (-> (->transform "transform2" (mt/mbql-query nil {:source-table table}))
(assoc-in [:target :name] "orders_3"))]
(mt/with-premium-features #{:transforms}
- (is (= [parent]
- (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s/dependencies" (:id child)))))))))
+ (is (=? [{:name "transform1"
+ :source {:type "query", :query {:database (mt/id)
+ :type "query"
+ :query {:source-table (mt/id :orders)}}}
+ :id (:id parent)
+ :entity_id (:entity_id parent)
+ :target {:schema "public", :name "orders_2", :type "table"}}]
+ (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s/dependencies" (:id child)))))))))
(deftest put-transforms-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
diff --git a/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsx b/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsx
index c1abe7d2067..b6bac3b4cdf 100644
--- a/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsx
+++ b/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsx
@@ -44,6 +44,7 @@ export const SdkSaveQuestionForm = ({ onCancel }: SaveQuestionFormProps) => {
originalQuestion={originalQuestion ?? null}
onCreate={onCreate}
onSave={onSave}
+ onCancel={onCancel}
multiStep={false}
targetCollection={targetCollection ?? undefined}
>
diff --git a/enterprise/frontend/src/metabase-enterprise/api/dependencies.ts b/enterprise/frontend/src/metabase-enterprise/api/dependencies.ts
new file mode 100644
index 00000000000..0df4d751f84
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/api/dependencies.ts
@@ -0,0 +1,49 @@
+import type {
+ CheckCardDependenciesRequest,
+ CheckDependenciesResponse,
+ CheckSnippetDependenciesRequest,
+ CheckTransformDependenciesRequest,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+
+export const dependencyApi = EnterpriseApi.injectEndpoints({
+ endpoints: (builder) => ({
+ checkCardDependencies: builder.query<
+ CheckDependenciesResponse,
+ CheckCardDependenciesRequest
+ >({
+ query: (body) => ({
+ method: "POST",
+ url: "/api/ee/dependencies/check_card",
+ body,
+ }),
+ }),
+ checkSnippetDependencies: builder.query<
+ CheckDependenciesResponse,
+ CheckSnippetDependenciesRequest
+ >({
+ query: (body) => ({
+ method: "POST",
+ url: "/api/ee/dependencies/check_snippet",
+ body,
+ }),
+ }),
+ checkTransformDependencies: builder.query<
+ CheckDependenciesResponse,
+ CheckTransformDependenciesRequest
+ >({
+ query: (body) => ({
+ method: "POST",
+ url: "/api/ee/dependencies/check_transform",
+ body,
+ }),
+ }),
+ }),
+});
+
+export const {
+ useLazyCheckCardDependenciesQuery,
+ useLazyCheckSnippetDependenciesQuery,
+ useLazyCheckTransformDependenciesQuery,
+} = dependencyApi;
diff --git a/enterprise/frontend/src/metabase-enterprise/api/index.ts b/enterprise/frontend/src/metabase-enterprise/api/index.ts
index ffeadd11cc7..01cedf6e0b6 100644
--- a/enterprise/frontend/src/metabase-enterprise/api/index.ts
+++ b/enterprise/frontend/src/metabase-enterprise/api/index.ts
@@ -5,6 +5,7 @@ export * from "./billing-info";
export * from "./comment";
export * from "./content-translation";
export * from "./db-routing";
+export * from "./dependencies";
export * from "./document";
export * from "./gdrive";
export * from "./metabot";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.module.css b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.module.css
new file mode 100644
index 00000000000..de3cba6c740
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.module.css
@@ -0,0 +1,12 @@
+.card {
+ flex-shrink: 0;
+
+ &:hover {
+ background-color: var(--mb-color-background-light);
+ }
+}
+
+.list {
+ overflow: auto;
+ max-height: 60vh;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.tsx
new file mode 100644
index 00000000000..60319e50cbb
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.tsx
@@ -0,0 +1,239 @@
+import { Fragment } from "react";
+import { Link } from "react-router";
+import { t } from "ttag";
+
+import {
+ Form,
+ FormErrorMessage,
+ FormProvider,
+ FormSubmitButton,
+} from "metabase/forms";
+import * as Urls from "metabase/lib/urls";
+import type { CheckDependenciesFormProps } from "metabase/plugins";
+import {
+ Anchor,
+ Box,
+ Button,
+ Card,
+ FixedSizeIcon,
+ Group,
+ type IconName,
+ Stack,
+ Text,
+} from "metabase/ui";
+import visualizations from "metabase/visualizations";
+import { getTransformUrl } from "metabase-enterprise/transforms/urls";
+import type {
+ Card as ApiCard,
+ CheckDependenciesResponse,
+ Transform,
+} from "metabase-types/api";
+
+import S from "./CheckDependenciesForm.module.css";
+
+type DependencyItem = CardDependencyItem | TransformDependencyItem;
+
+type CardDependencyItem = {
+ type: "card";
+ card: ApiCard;
+};
+
+type TransformDependencyItem = {
+ type: "transform";
+ transform: Transform;
+};
+
+export function CheckDependenciesForm({
+ checkData,
+ onSave,
+ onCancel,
+}: CheckDependenciesFormProps) {
+ const items = getDependencyItems(checkData);
+
+ return (
+ <FormProvider initialValues={{}} onSubmit={onSave}>
+ <Form>
+ <Text mb="md" px="xl">
+ {t`The items below will break because of these changes:`}
+ </Text>
+ <Stack className={S.list} mb="xl" px="xl">
+ {items.map((item, index) => (
+ <DependencyItemCard key={index} item={item} />
+ ))}
+ </Stack>
+ <Group px="xl">
+ <Box flex={1}>
+ <FormErrorMessage />
+ </Box>
+ <Button onClick={onCancel}>{t`Cancel`}</Button>
+ <FormSubmitButton label={t`Save anyway`} variant="filled" />
+ </Group>
+ </Form>
+ </FormProvider>
+ );
+}
+
+function getDependencyItems({
+ bad_cards = [],
+ bad_transforms = [],
+}: CheckDependenciesResponse): DependencyItem[] {
+ return [
+ ...bad_transforms.map(
+ (transform): TransformDependencyItem => ({
+ type: "transform",
+ transform,
+ }),
+ ),
+ ...bad_cards.map(
+ (card): CardDependencyItem => ({
+ type: "card",
+ card,
+ }),
+ ),
+ ];
+}
+
+type DependencyItemCardProps = {
+ item: DependencyItem;
+};
+
+function DependencyItemCard({ item }: DependencyItemCardProps) {
+ return (
+ <Card
+ className={S.card}
+ component={Link}
+ to={getItemLink(item)}
+ p="md"
+ shadow="none"
+ withBorder
+ >
+ <Stack gap="xs">
+ <Group c="brand" gap="sm" wrap="nowrap">
+ <FixedSizeIcon name={getItemIcon(item)} />
+ <Box fw="bold" lh="h4">
+ {getItemName(item)}
+ </Box>
+ </Group>
+ {getItemDescription(item)}
+ </Stack>
+ </Card>
+ );
+}
+
+function getItemIcon(item: DependencyItem): IconName {
+ if (item.type === "card") {
+ const { card } = item;
+
+ switch (card.type) {
+ case "question":
+ return visualizations.get(card.display)?.iconName ?? "table2";
+ case "model":
+ return "model";
+ case "metric":
+ return "metric";
+ }
+ }
+
+ if (item.type === "transform") {
+ return "refresh_downstream";
+ }
+
+ return "unknown";
+}
+
+function getItemName(item: DependencyItem) {
+ switch (item.type) {
+ case "card":
+ return item.card.name;
+ case "transform":
+ return item.transform.name;
+ default:
+ return null;
+ }
+}
+
+function getItemLink(item: DependencyItem) {
+ switch (item.type) {
+ case "card":
+ return Urls.question(item.card);
+ default:
+ return getTransformUrl(item.transform.id);
+ }
+}
+
+function getItemDescription(item: DependencyItem) {
+ if (item.type === "card") {
+ const { collection, dashboard } = item.card;
+
+ if (collection != null) {
+ const ancestors = collection.effective_ancestors ?? [];
+ const breadcrumbs = [
+ ...ancestors.map((ancestor) => ({
+ title: ancestor.name,
+ to: Urls.collection(ancestor),
+ })),
+ { title: collection.name, to: Urls.collection(collection) },
+ ...(dashboard != null
+ ? [{ title: dashboard.name, to: Urls.dashboard(dashboard) }]
+ : []),
+ ];
+
+ return (
+ <Group gap="sm" wrap="nowrap">
+ <FixedSizeIcon
+ c="text-secondary"
+ name={dashboard != null ? "dashboard" : "collection"}
+ flex="0 0 auto"
+ />
+ <BreadcrumbList items={breadcrumbs} />
+ </Group>
+ );
+ }
+ }
+
+ if (item.type === "transform") {
+ return (
+ <Box fz="sm" lh="h5">
+ {t`Transform`}
+ </Box>
+ );
+ }
+
+ return null;
+}
+
+type BreadcrumbItem = {
+ title: string;
+ to: string;
+};
+
+type BreadcrumbProps = {
+ item: BreadcrumbItem;
+};
+
+function Breadcrumb({ item }: BreadcrumbProps) {
+ return (
+ <Anchor component={Link} to={item.to} c="text-secondary" fz="sm" lh="h5">
+ {item.title}
+ </Anchor>
+ );
+}
+
+type BreadcrumbListProps = {
+ items: BreadcrumbItem[];
+};
+
+function BreadcrumbList({ items }: BreadcrumbListProps) {
+ return (
+ <Group align="center" gap="xs">
+ {items.map((item, itemIndex) => (
+ <Fragment key={itemIndex}>
+ {itemIndex > 0 && (
+ <FixedSizeIcon name="chevronright" c="text-secondary" size={8} />
+ )}
+ <Breadcrumb item={item} />
+ </Fragment>
+ ))}
+ </Group>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.unit.spec.tsx
new file mode 100644
index 00000000000..91a1db194b1
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.unit.spec.tsx
@@ -0,0 +1,194 @@
+import { Route } from "react-router";
+
+import { getIcon, renderWithProviders, screen } from "__support__/ui";
+import type { IconName } from "metabase/ui";
+import registerVisualizations from "metabase/visualizations/register";
+import type {
+ CardDisplayType,
+ CardType,
+ CheckDependenciesResponse,
+} from "metabase-types/api";
+import {
+ createMockCard,
+ createMockCheckDependenciesResponse,
+ createMockCollection,
+ createMockTransform,
+} from "metabase-types/api/mocks";
+
+import { CheckDependenciesForm } from "./CheckDependenciesForm";
+
+type SetupOpts = {
+ checkData?: CheckDependenciesResponse;
+};
+
+function setup({
+ checkData = createMockCheckDependenciesResponse(),
+}: SetupOpts) {
+ const onSave = jest.fn();
+ const onCancel = jest.fn();
+ renderWithProviders(
+ <Route
+ path="/"
+ component={() => (
+ <CheckDependenciesForm
+ checkData={checkData}
+ onSave={onSave}
+ onCancel={onCancel}
+ />
+ )}
+ />,
+ { withRouter: true, initialRoute: "/" },
+ );
+ return { onSave, onCancel };
+}
+
+registerVisualizations();
+
+describe("CheckDependenciesForm", () => {
+ describe.each<CardType>(["question", "model", "metric"])("%s", (type) => {
+ it("should display broken cards", () => {
+ setup({
+ checkData: createMockCheckDependenciesResponse({
+ success: false,
+ bad_cards: [
+ createMockCard({
+ name: "Card",
+ type,
+ }),
+ ],
+ }),
+ });
+ expect(screen.getByText("Card")).toBeInTheDocument();
+ expect(screen.getByRole("link", { name: /Card/ })).toHaveAttribute(
+ "href",
+ `/${type}/1-card`,
+ );
+ });
+
+ it("should display the root collection", () => {
+ setup({
+ checkData: createMockCheckDependenciesResponse({
+ success: false,
+ bad_cards: [
+ createMockCard({
+ name: "Card",
+ type,
+ collection: createMockCollection({
+ id: "root",
+ name: "Our analytics",
+ }),
+ }),
+ ],
+ }),
+ });
+ expect(
+ screen.getByRole("link", { name: "Our analytics" }),
+ ).toHaveAttribute("href", "/collection/root");
+ });
+
+ it("should display the nested collection with parent collections", () => {
+ setup({
+ checkData: createMockCheckDependenciesResponse({
+ success: false,
+ bad_cards: [
+ createMockCard({
+ name: "Card",
+ type,
+ collection: createMockCollection({
+ id: 2,
+ name: "Second collection",
+ effective_ancestors: [
+ createMockCollection({ id: 1, name: "First collection" }),
+ createMockCollection({ id: "root", name: "Our analytics" }),
+ ],
+ }),
+ }),
+ ],
+ }),
+ });
+ expect(
+ screen.getByRole("link", { name: "Our analytics" }),
+ ).toHaveAttribute("href", "/collection/root");
+ expect(
+ screen.getByRole("link", { name: "First collection" }),
+ ).toHaveAttribute("href", "/collection/1-first-collection");
+ expect(
+ screen.getByRole("link", { name: "Second collection" }),
+ ).toHaveAttribute("href", "/collection/2-second-collection");
+ });
+ });
+
+ it.each<{ display: CardDisplayType; icon: IconName }>([
+ { display: "line", icon: "line" },
+ { display: "pie", icon: "pie" },
+ ])(
+ "should display the icon that corresponds to the display type",
+ ({ display, icon }) => {
+ setup({
+ checkData: createMockCheckDependenciesResponse({
+ success: false,
+ bad_cards: [
+ createMockCard({
+ type: "question",
+ display,
+ }),
+ ],
+ }),
+ });
+ expect(getIcon(icon)).toBeInTheDocument();
+ },
+ );
+
+ it("should display the dashboard that the question belongs to", () => {
+ setup({
+ checkData: createMockCheckDependenciesResponse({
+ success: false,
+ bad_cards: [
+ createMockCard({
+ name: "Card",
+ type: "question",
+ collection: createMockCollection({
+ id: 1,
+ name: "First collection",
+ effective_ancestors: [
+ createMockCollection({ id: "root", name: "Our analytics" }),
+ ],
+ }),
+ dashboard: {
+ id: 2,
+ name: "Card dashboard",
+ },
+ }),
+ ],
+ }),
+ });
+ expect(screen.getByRole("link", { name: "Our analytics" })).toHaveAttribute(
+ "href",
+ "/collection/root",
+ );
+ expect(
+ screen.getByRole("link", { name: "First collection" }),
+ ).toHaveAttribute("href", "/collection/1-first-collection");
+ expect(
+ screen.getByRole("link", { name: "Card dashboard" }),
+ ).toHaveAttribute("href", "/dashboard/2-card-dashboard");
+ });
+
+ it("should display transforms", () => {
+ setup({
+ checkData: createMockCheckDependenciesResponse({
+ success: false,
+ bad_transforms: [
+ createMockTransform({
+ id: 1,
+ name: "SQL transform",
+ }),
+ ],
+ }),
+ });
+ expect(screen.getByRole("link", { name: /SQL transform/ })).toHaveAttribute(
+ "href",
+ "/admin/transforms/1",
+ );
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/index.ts
new file mode 100644
index 00000000000..420134c7ca9
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/index.ts
@@ -0,0 +1 @@
+export * from "./CheckDependenciesForm";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/CheckDependenciesModal.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/CheckDependenciesModal.tsx
new file mode 100644
index 00000000000..bff78b63a82
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/CheckDependenciesModal.tsx
@@ -0,0 +1,35 @@
+import type { CheckDependenciesModalProps } from "metabase/plugins";
+import { Flex, Modal } from "metabase/ui";
+
+import { CheckDependenciesForm } from "../CheckDependenciesForm";
+import { CheckDependenciesTitle } from "../CheckDependenciesTitle";
+
+export function CheckDependenciesModal({
+ checkData,
+ opened,
+ onSave,
+ onClose,
+}: CheckDependenciesModalProps) {
+ return (
+ <Modal.Root size="xl" padding="xl" opened={opened} onClose={onClose}>
+ <Modal.Overlay />
+ <Modal.Content>
+ <Modal.Header px="xl">
+ <Modal.Title>
+ <CheckDependenciesTitle />
+ </Modal.Title>
+ <Flex justify="flex-end">
+ <Modal.CloseButton />
+ </Flex>
+ </Modal.Header>
+ <Modal.Body px={0}>
+ <CheckDependenciesForm
+ checkData={checkData}
+ onSave={onSave}
+ onCancel={onClose}
+ />
+ </Modal.Body>
+ </Modal.Content>
+ </Modal.Root>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/index.ts
new file mode 100644
index 00000000000..6f83f37598d
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/index.ts
@@ -0,0 +1 @@
+export * from "./CheckDependenciesModal";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/CheckDependenciesTitle.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/CheckDependenciesTitle.tsx
new file mode 100644
index 00000000000..82f20ab86d3
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/CheckDependenciesTitle.tsx
@@ -0,0 +1,11 @@
+import { t } from "ttag";
+
+import { Box } from "metabase/ui";
+
+export function CheckDependenciesTitle() {
+ return (
+ <Box fz="h3" lh="h3">
+ {t`These changes will break some other things. Save anyway?`}
+ </Box>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/index.ts
new file mode 100644
index 00000000000..815b05cb08e
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/index.ts
@@ -0,0 +1 @@
+export * from "./CheckDependenciesTitle";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/index.ts
new file mode 100644
index 00000000000..bb1a1c1d203
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/index.ts
@@ -0,0 +1 @@
+export * from "./use-check-card-dependencies";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.ts
new file mode 100644
index 00000000000..27dadd7ba9c
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.ts
@@ -0,0 +1,40 @@
+import { useCallback } from "react";
+
+import { useStore } from "metabase/lib/redux";
+import type {
+ UseCheckDependenciesProps,
+ UseCheckDependenciesResult,
+} from "metabase/plugins";
+import { getSubmittableQuestion } from "metabase/query_builder/selectors";
+import { useLazyCheckCardDependenciesQuery } from "metabase-enterprise/api";
+import type Question from "metabase-lib/v1/Question";
+import type { CheckCardDependenciesRequest } from "metabase-types/api";
+
+import { useCheckDependencies } from "../use-check-dependencies";
+
+export function useCheckCardDependencies({
+ onSave,
+ onError,
+}: UseCheckDependenciesProps<Question>): UseCheckDependenciesResult<Question> {
+ const store = useStore();
+
+ const getCheckDependenciesRequest = useCallback(
+ (question: Question): CheckCardDependenciesRequest => {
+ const submittableQuestion = getSubmittableQuestion(
+ store.getState(),
+ question,
+ );
+ const { id, type, dataset_query, result_metadata } =
+ submittableQuestion.card();
+ return { id, type, dataset_query, result_metadata };
+ },
+ [store],
+ );
+
+ return useCheckDependencies({
+ getCheckDependenciesRequest,
+ useLazyCheckDependenciesQuery: useLazyCheckCardDependenciesQuery,
+ onSave,
+ onError,
+ });
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.unit.spec.ts
new file mode 100644
index 00000000000..80f6242dc1c
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.unit.spec.ts
@@ -0,0 +1,77 @@
+import { setupCheckCardDependenciesEndpoint } from "__support__/server-mocks/dependencies";
+import { act, renderHookWithProviders } from "__support__/ui";
+import { SAMPLE_METADATA } from "metabase-lib/test-helpers";
+import Question from "metabase-lib/v1/Question";
+import type { CheckDependenciesResponse } from "metabase-types/api";
+import { createMockCheckDependenciesResponse } from "metabase-types/api/mocks";
+
+import { useCheckCardDependencies } from "./use-check-card-dependencies";
+
+type SetupOpts = {
+ response?: CheckDependenciesResponse;
+};
+
+function setup({
+ response = createMockCheckDependenciesResponse(),
+}: SetupOpts = {}) {
+ setupCheckCardDependenciesEndpoint(response);
+
+ const onSave = jest.fn();
+ const onError = jest.fn();
+ const { result } = renderHookWithProviders(
+ () => useCheckCardDependencies({ onSave, onError }),
+ {},
+ );
+
+ return { result, onSave, onError };
+}
+
+describe("useCheckCardDependencies", () => {
+ it("should save if there are no broken dependencies", async () => {
+ const question = Question.create({ metadata: SAMPLE_METADATA });
+ const response = createMockCheckDependenciesResponse({ success: true });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(question));
+
+ expect(onSave).toHaveBeenCalledWith(question);
+ });
+
+ it("should not save if there are broken dependencies", async () => {
+ const question = Question.create({ metadata: SAMPLE_METADATA });
+ const response = createMockCheckDependenciesResponse({ success: false });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(question));
+
+ expect(result.current.checkData).toEqual(response);
+ expect(result.current.isConfirmationShown).toBe(true);
+ expect(onSave).not.toHaveBeenCalled();
+ });
+
+ it("should allow to confirm saving", async () => {
+ const question = Question.create({ metadata: SAMPLE_METADATA });
+ const response = createMockCheckDependenciesResponse({ success: false });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(question));
+ await act(() => result.current.handleSaveAfterConfirmation());
+
+ expect(result.current.checkData).toEqual(response);
+ expect(result.current.isConfirmationShown).toBe(false);
+ expect(onSave).toHaveBeenCalledWith(question);
+ });
+
+ it("should allow to cancel the confirmation screen", async () => {
+ const question = Question.create({ metadata: SAMPLE_METADATA });
+ const response = createMockCheckDependenciesResponse({ success: false });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(question));
+ await act(() => result.current.handleCloseConfirmation());
+
+ expect(result.current.checkData).toEqual(response);
+ expect(result.current.isConfirmationShown).toBe(false);
+ expect(onSave).not.toHaveBeenCalled();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/index.ts
new file mode 100644
index 00000000000..db8905bf067
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/index.ts
@@ -0,0 +1 @@
+export * from "./use-check-dependencies";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/use-check-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/use-check-dependencies.ts
new file mode 100644
index 00000000000..bc80ac71d41
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/use-check-dependencies.ts
@@ -0,0 +1,80 @@
+import { useCallback, useState } from "react";
+
+import type { UseCheckDependenciesResult } from "metabase/plugins";
+import type { CheckDependenciesResponse } from "metabase-types/api";
+
+type UseCheckDependenciesQueryData = {
+ data?: CheckDependenciesResponse;
+ error?: unknown;
+ isLoading: boolean;
+};
+
+type UseCheckDependenciesArgsData<TRequest> = {
+ lastArg?: TRequest;
+};
+
+type UseCheckDependenciesQueryResult<TRequest> = [
+ (request: TRequest) => Promise<UseCheckDependenciesQueryData>,
+ UseCheckDependenciesQueryData,
+ UseCheckDependenciesArgsData<TRequest>,
+];
+
+type UseCheckDependenciesProps<TChange, TRequest> = {
+ getCheckDependenciesRequest: (change: TChange) => TRequest;
+ useLazyCheckDependenciesQuery: () => UseCheckDependenciesQueryResult<TRequest>;
+ onSave: (change: TChange) => Promise<void>;
+ onError: (error: unknown) => void;
+};
+
+export function useCheckDependencies<TChange, TRequest>({
+ getCheckDependenciesRequest,
+ useLazyCheckDependenciesQuery,
+ onSave,
+ onError,
+}: UseCheckDependenciesProps<
+ TChange,
+ TRequest
+>): UseCheckDependenciesResult<TChange> {
+ const [change, setChange] = useState<TChange | null>(null);
+ const [isConfirmationShown, setIsConfirmationShown] = useState(false);
+ const [checkDependencies, { data, isLoading }] =
+ useLazyCheckDependenciesQuery();
+
+ const handleInitialSave = useCallback(
+ async (change: TChange) => {
+ const { data, error } = await checkDependencies(
+ getCheckDependenciesRequest(change),
+ );
+ if (error != null) {
+ onError(error);
+ } else if (data != null && !data.success) {
+ setChange(change);
+ setIsConfirmationShown(true);
+ } else {
+ await onSave(change);
+ }
+ },
+ [getCheckDependenciesRequest, checkDependencies, onSave, onError],
+ );
+
+ const handleCloseConfirmation = useCallback(() => {
+ setChange(null);
+ setIsConfirmationShown(false);
+ }, []);
+
+ const handleSaveAfterConfirmation = useCallback(async () => {
+ if (change != null) {
+ await onSave(change);
+ handleCloseConfirmation();
+ }
+ }, [change, onSave, handleCloseConfirmation]);
+
+ return {
+ checkData: data,
+ isCheckingDependencies: isLoading,
+ isConfirmationShown,
+ handleInitialSave,
+ handleSaveAfterConfirmation,
+ handleCloseConfirmation,
+ };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/index.ts
new file mode 100644
index 00000000000..d100a598a59
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/index.ts
@@ -0,0 +1 @@
+export * from "./use-check-snippet-dependencies";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.ts
new file mode 100644
index 00000000000..8221931c6cf
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.ts
@@ -0,0 +1,31 @@
+import type {
+ UseCheckDependenciesProps,
+ UseCheckDependenciesResult,
+} from "metabase/plugins";
+import { useLazyCheckSnippetDependenciesQuery } from "metabase-enterprise/api";
+import type {
+ CheckSnippetDependenciesRequest,
+ UpdateSnippetRequest,
+} from "metabase-types/api";
+
+import { useCheckDependencies } from "../use-check-dependencies";
+
+export function useCheckSnippetDependencies({
+ onSave,
+ onError,
+}: UseCheckDependenciesProps<UpdateSnippetRequest>): UseCheckDependenciesResult<UpdateSnippetRequest> {
+ return useCheckDependencies({
+ getCheckDependenciesRequest,
+ useLazyCheckDependenciesQuery: useLazyCheckSnippetDependenciesQuery,
+ onSave,
+ onError,
+ });
+}
+
+function getCheckDependenciesRequest({
+ id,
+ name,
+ content,
+}: UpdateSnippetRequest): CheckSnippetDependenciesRequest {
+ return { id, name, content };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.unit.spec.ts
new file mode 100644
index 00000000000..6b0c28be68c
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.unit.spec.ts
@@ -0,0 +1,52 @@
+import { setupCheckSnippetDependenciesEndpoint } from "__support__/server-mocks/dependencies";
+import { act, renderHookWithProviders } from "__support__/ui";
+import type { CheckDependenciesResponse } from "metabase-types/api";
+import {
+ createMockCheckDependenciesResponse,
+ createMockUpdateSnippetRequest,
+} from "metabase-types/api/mocks";
+
+import { useCheckSnippetDependencies } from "./use-check-snippet-dependencies";
+
+type SetupOpts = {
+ response?: CheckDependenciesResponse;
+};
+
+function setup({
+ response = createMockCheckDependenciesResponse(),
+}: SetupOpts = {}) {
+ setupCheckSnippetDependenciesEndpoint(response);
+
+ const onSave = jest.fn();
+ const onError = jest.fn();
+ const { result } = renderHookWithProviders(
+ () => useCheckSnippetDependencies({ onSave, onError }),
+ {},
+ );
+
+ return { result, onSave, onError };
+}
+
+describe("useCheckSnippetDependencies", () => {
+ it("should save if there are no broken dependencies", async () => {
+ const request = createMockUpdateSnippetRequest();
+ const response = createMockCheckDependenciesResponse({ success: true });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(request));
+
+ expect(onSave).toHaveBeenCalledWith(request);
+ });
+
+ it("should not save if there are broken dependencies", async () => {
+ const request = createMockUpdateSnippetRequest();
+ const response = createMockCheckDependenciesResponse({ success: false });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(request));
+
+ expect(result.current.checkData).toEqual(response);
+ expect(result.current.isConfirmationShown).toBe(true);
+ expect(onSave).not.toHaveBeenCalled();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/index.ts
new file mode 100644
index 00000000000..8b2fd65fcaa
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/index.ts
@@ -0,0 +1 @@
+export * from "./use-check-transform-dependencies";
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.ts
new file mode 100644
index 00000000000..fb55a754d46
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.ts
@@ -0,0 +1,30 @@
+import type {
+ UseCheckDependenciesProps,
+ UseCheckDependenciesResult,
+} from "metabase/plugins";
+import { useLazyCheckTransformDependenciesQuery } from "metabase-enterprise/api";
+import type {
+ CheckTransformDependenciesRequest,
+ UpdateTransformRequest,
+} from "metabase-types/api";
+
+import { useCheckDependencies } from "../use-check-dependencies";
+
+export function useCheckTransformDependencies({
+ onSave,
+ onError,
+}: UseCheckDependenciesProps<UpdateTransformRequest>): UseCheckDependenciesResult<UpdateTransformRequest> {
+ return useCheckDependencies({
+ getCheckDependenciesRequest,
+ useLazyCheckDependenciesQuery: useLazyCheckTransformDependenciesQuery,
+ onSave,
+ onError,
+ });
+}
+
+function getCheckDependenciesRequest({
+ id,
+ source,
+}: UpdateTransformRequest): CheckTransformDependenciesRequest {
+ return { id, source };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.unit.spec.ts
new file mode 100644
index 00000000000..19276764a62
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.unit.spec.ts
@@ -0,0 +1,52 @@
+import { setupCheckTransformDependenciesEndpoint } from "__support__/server-mocks/dependencies";
+import { act, renderHookWithProviders } from "__support__/ui";
+import type { CheckDependenciesResponse } from "metabase-types/api";
+import {
+ createMockCheckDependenciesResponse,
+ createMockUpdateTransformRequest,
+} from "metabase-types/api/mocks";
+
+import { useCheckTransformDependencies } from "./use-check-transform-dependencies";
+
+type SetupOpts = {
+ response?: CheckDependenciesResponse;
+};
+
+function setup({
+ response = createMockCheckDependenciesResponse(),
+}: SetupOpts = {}) {
+ setupCheckTransformDependenciesEndpoint(response);
+
+ const onSave = jest.fn();
+ const onError = jest.fn();
+ const { result } = renderHookWithProviders(
+ () => useCheckTransformDependencies({ onSave, onError }),
+ {},
+ );
+
+ return { result, onSave, onError };
+}
+
+describe("useCheckTransformDependencies", () => {
+ it("should save if there are no broken dependencies", async () => {
+ const request = createMockUpdateTransformRequest();
+ const response = createMockCheckDependenciesResponse({ success: true });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(request));
+
+ expect(onSave).toHaveBeenCalledWith(request);
+ });
+
+ it("should not save if there are broken dependencies", async () => {
+ const request = createMockUpdateTransformRequest();
+ const response = createMockCheckDependenciesResponse({ success: false });
+ const { result, onSave } = setup({ response });
+
+ await act(() => result.current.handleInitialSave(request));
+
+ expect(result.current.checkData).toEqual(response);
+ expect(result.current.isConfirmationShown).toBe(true);
+ expect(onSave).not.toHaveBeenCalled();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/index.ts
new file mode 100644
index 00000000000..3e103cc4c9d
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/index.ts
@@ -0,0 +1,19 @@
+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";
+import { hasPremiumFeature } from "metabase-enterprise/settings";
+
+import { CheckDependenciesForm } from "./components/CheckDependenciesForm";
+import { CheckDependenciesModal } from "./components/CheckDependenciesModal";
+import { CheckDependenciesTitle } from "./components/CheckDependenciesTitle";
+import { useCheckCardDependencies } from "./hooks/use-check-card-dependencies";
+import { useCheckSnippetDependencies } from "./hooks/use-check-snippet-dependencies";
+import { useCheckTransformDependencies } from "./hooks/use-check-transform-dependencies";
+
+if (hasPremiumFeature("dependencies")) {
+ PLUGIN_DEPENDENCIES.CheckDependenciesForm = CheckDependenciesForm;
+ PLUGIN_DEPENDENCIES.CheckDependenciesModal = CheckDependenciesModal;
+ PLUGIN_DEPENDENCIES.CheckDependenciesTitle = CheckDependenciesTitle;
+ PLUGIN_DEPENDENCIES.useCheckCardDependencies = useCheckCardDependencies;
+ PLUGIN_DEPENDENCIES.useCheckSnippetDependencies = useCheckSnippetDependencies;
+ PLUGIN_DEPENDENCIES.useCheckTransformDependencies =
+ useCheckTransformDependencies;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/plugins.js b/enterprise/frontend/src/metabase-enterprise/plugins.js
index 0a676e1f976..329d6bca338 100644
--- a/enterprise/frontend/src/metabase-enterprise/plugins.js
+++ b/enterprise/frontend/src/metabase-enterprise/plugins.js
@@ -39,6 +39,7 @@ import "./clean_up";
import "./metabot";
import "./database_replication";
import "./table-editing";
+import "./dependencies";
import "./documents";
import "./semantic_search";
import "./transforms";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
index 87fac2e71e8..7a50a218344 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
@@ -6,7 +6,10 @@ import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErr
import { useDispatch } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { useMetadataToasts } from "metabase/metadata/hooks";
-import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
+import {
+ PLUGIN_DEPENDENCIES,
+ PLUGIN_TRANSFORMS_PYTHON,
+} from "metabase/plugins";
import {
useGetTransformQuery,
useUpdateTransformMutation,
@@ -50,22 +53,38 @@ type TransformQueryPageBodyProps = {
export function TransformQueryPageBody({
transform,
}: TransformQueryPageBodyProps) {
- const [updateTransform, { isLoading }] = useUpdateTransformMutation();
+ const [updateTransform, { isLoading: isSaving }] =
+ useUpdateTransformMutation();
const dispatch = useDispatch();
- const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
+ const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
- const handleSourceSave = async (source: TransformSource) => {
- const { error } = await updateTransform({
+ const {
+ checkData,
+ isCheckingDependencies,
+ isConfirmationShown,
+ handleInitialSave,
+ handleSaveAfterConfirmation,
+ handleCloseConfirmation,
+ } = PLUGIN_DEPENDENCIES.useCheckTransformDependencies({
+ onSave: async (request) => {
+ const { error } = await updateTransform(request);
+ if (error) {
+ sendErrorToast(t`Failed to update transform query`);
+ } else {
+ sendSuccessToast(t`Transform query updated`);
+ dispatch(push(getTransformUrl(transform.id)));
+ }
+ },
+ onError: () => {
+ sendErrorToast(t`Failed to update transform query`);
+ },
+ });
+
+ const handleSaveSource = async (source: TransformSource) => {
+ await handleInitialSave({
id: transform.id,
source,
});
-
- if (error) {
- sendErrorToast(t`Failed to update transform query`);
- } else {
- sendSuccessToast(t`Transform query updated`);
- dispatch(push(getTransformUrl(transform.id)));
- }
};
const handleCancel = () => {
@@ -77,19 +96,29 @@ export function TransformQueryPageBody({
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
initialSource={transform.source}
isNew={false}
- isSaving={isLoading}
- onSave={handleSourceSave}
+ isSaving={isSaving}
+ onSave={handleSaveSource}
onCancel={handleCancel}
/>
);
}
return (
- <QueryEditor
- initialSource={transform.source}
- isNew={false}
- isSaving={isLoading}
- onSave={handleSourceSave}
- onCancel={handleCancel}
- />
+ <>
+ <QueryEditor
+ initialSource={transform.source}
+ isNew={false}
+ isSaving={isSaving || isCheckingDependencies}
+ onSave={handleSaveSource}
+ onCancel={handleCancel}
+ />
+ {isConfirmationShown && checkData != null && (
+ <PLUGIN_DEPENDENCIES.CheckDependenciesModal
+ checkData={checkData}
+ opened
+ onSave={handleSaveAfterConfirmation}
+ onClose={handleCloseConfirmation}
+ />
+ )}
+ </>
);
}
diff --git a/frontend/src/metabase-types/api/dependencies.ts b/frontend/src/metabase-types/api/dependencies.ts
new file mode 100644
index 00000000000..968cedea519
--- /dev/null
+++ b/frontend/src/metabase-types/api/dependencies.ts
@@ -0,0 +1,19 @@
+import type { NativeQuerySnippet } from "metabase-types/api/snippets";
+
+import type { Card } from "./card";
+import type { Transform } from "./transform";
+
+export type CheckDependenciesResponse = {
+ success: boolean;
+ bad_cards?: Card[];
+ bad_transforms?: Transform[];
+};
+
+export type CheckCardDependenciesRequest = Pick<Card, "id"> &
+ Partial<Pick<Card, "type" | "dataset_query" | "result_metadata">>;
+
+export type CheckSnippetDependenciesRequest = Pick<NativeQuerySnippet, "id"> &
+ Partial<Pick<NativeQuerySnippet, "name" | "content">>;
+
+export type CheckTransformDependenciesRequest = Pick<Transform, "id"> &
+ Partial<Pick<Transform, "source">>;
diff --git a/frontend/src/metabase-types/api/index.ts b/frontend/src/metabase-types/api/index.ts
index 8b7fb609c6b..882c2b7fc3b 100644
--- a/frontend/src/metabase-types/api/index.ts
+++ b/frontend/src/metabase-types/api/index.ts
@@ -16,6 +16,7 @@ export * from "./content-translation";
export * from "./dashboard";
export * from "./database";
export * from "./dataset";
+export * from "./dependencies";
export * from "./email";
export * from "./entity-id";
export * from "./field";
diff --git a/frontend/src/metabase-types/api/mocks/dependencies.ts b/frontend/src/metabase-types/api/mocks/dependencies.ts
new file mode 100644
index 00000000000..9552e424e4d
--- /dev/null
+++ b/frontend/src/metabase-types/api/mocks/dependencies.ts
@@ -0,0 +1,42 @@
+import type {
+ CheckCardDependenciesRequest,
+ CheckDependenciesResponse,
+ CheckSnippetDependenciesRequest,
+ CheckTransformDependenciesRequest,
+} from "metabase-types/api";
+
+export function createMockCheckCardDependenciesRequest(
+ opts?: Partial<CheckCardDependenciesRequest>,
+): CheckCardDependenciesRequest {
+ return {
+ id: 1,
+ ...opts,
+ };
+}
+
+export function createMockCheckSnippetDependenciesRequest(
+ opts?: Partial<CheckSnippetDependenciesRequest>,
+): CheckSnippetDependenciesRequest {
+ return {
+ id: 1,
+ ...opts,
+ };
+}
+
+export function createMockCheckTransformDependenciesRequest(
+ opts?: Partial<CheckTransformDependenciesRequest>,
+): CheckTransformDependenciesRequest {
+ return {
+ id: 1,
+ ...opts,
+ };
+}
+
+export function createMockCheckDependenciesResponse(
+ opts?: Partial<CheckDependenciesResponse>,
+): CheckDependenciesResponse {
+ return {
+ success: true,
+ ...opts,
+ };
+}
diff --git a/frontend/src/metabase-types/api/mocks/index.ts b/frontend/src/metabase-types/api/mocks/index.ts
index 310b153c05a..344e437eee7 100644
--- a/frontend/src/metabase-types/api/mocks/index.ts
+++ b/frontend/src/metabase-types/api/mocks/index.ts
@@ -9,6 +9,7 @@ export * from "./collection";
export * from "./dashboard";
export * from "./database";
export * from "./dataset";
+export * from "./dependencies";
export * from "./document";
export * from "./field";
export * from "./geojson";
diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.ts
index fc1a8e7b88f..98c57267de4 100644
--- a/frontend/src/metabase-types/api/mocks/settings.ts
+++ b/frontend/src/metabase-types/api/mocks/settings.ts
@@ -101,6 +101,7 @@ export const createMockTokenFeatures = (
advanced_permissions: false,
audit_app: false,
cache_granular_controls: false,
+ dependencies: false,
disable_password_login: false,
cloud_custom_smtp: false,
content_translation: false,
diff --git a/frontend/src/metabase-types/api/mocks/snippets.ts b/frontend/src/metabase-types/api/mocks/snippets.ts
index ba9246beb24..68a4b4c4132 100644
--- a/frontend/src/metabase-types/api/mocks/snippets.ts
+++ b/frontend/src/metabase-types/api/mocks/snippets.ts
@@ -1,4 +1,7 @@
-import type { NativeQuerySnippet } from "metabase-types/api";
+import type {
+ NativeQuerySnippet,
+ UpdateSnippetRequest,
+} from "metabase-types/api";
import { createMockEntityId } from "./entity-id";
import { createMockUser } from "./user";
@@ -22,3 +25,12 @@ export const createMockNativeQuerySnippet = ({
updated_at: new Date().toISOString(),
...opts,
});
+
+export function createMockUpdateSnippetRequest(
+ opts?: Partial<UpdateSnippetRequest>,
+): UpdateSnippetRequest {
+ return {
+ id: 1,
+ ...opts,
+ };
+}
diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.ts
index ef8c1333be4..c5651f120ed 100644
--- a/frontend/src/metabase-types/api/mocks/transform.ts
+++ b/frontend/src/metabase-types/api/mocks/transform.ts
@@ -7,6 +7,7 @@ import type {
TransformSource,
TransformTag,
TransformTarget,
+ UpdateTransformRequest,
} from "metabase-types/api";
import { createMockStructuredDatasetQuery } from "./query";
@@ -100,3 +101,12 @@ export function createMockTransformJob(
...opts,
};
}
+
+export function createMockUpdateTransformRequest(
+ opts?: Partial<UpdateTransformRequest>,
+): UpdateTransformRequest {
+ return {
+ id: 1,
+ ...opts,
+ };
+}
diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.ts
index 035b0290445..cb9d4d39387 100644
--- a/frontend/src/metabase-types/api/settings.ts
+++ b/frontend/src/metabase-types/api/settings.ts
@@ -321,6 +321,7 @@ export const tokenFeatures = [
"etl_connections",
"etl_connections_pg",
"table_data_editing",
+ "dependencies",
"documents",
"semantic_search",
"transforms",
diff --git a/frontend/src/metabase-types/api/snippets.ts b/frontend/src/metabase-types/api/snippets.ts
index 1733a0a174d..e81bb9e7d7e 100644
--- a/frontend/src/metabase-types/api/snippets.ts
+++ b/frontend/src/metabase-types/api/snippets.ts
@@ -25,17 +25,17 @@ export interface ListSnippetsParams {
}
export interface CreateSnippetRequest {
- content: string;
name: string;
- description?: string;
+ content: string;
+ description?: string | null;
collection_id?: RegularCollectionId | null;
}
export interface UpdateSnippetRequest {
id: NativeQuerySnippetId;
- archived?: boolean;
- content?: string | null;
- name?: string | null;
+ name?: string;
description?: string | null;
+ content?: string;
collection_id?: RegularCollectionId | null;
+ archived?: boolean;
}
diff --git a/frontend/src/metabase/common/components/SaveQuestionForm/types.ts b/frontend/src/metabase/common/components/SaveQuestionForm/types.ts
index 2ac9148d41b..4bd6f4c5021 100644
--- a/frontend/src/metabase/common/components/SaveQuestionForm/types.ts
+++ b/frontend/src/metabase/common/components/SaveQuestionForm/types.ts
@@ -15,6 +15,7 @@ export type SaveQuestionProps<C = CollectionId> = {
},
) => Promise<Question>;
onSave: (question: Question) => Promise<void>;
+ onCancel?: () => void;
closeOnSuccess?: boolean;
multiStep?: boolean;
diff --git a/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx b/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx
index 2e3ce0d4286..48332900d26 100644
--- a/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx
+++ b/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx
@@ -5,6 +5,7 @@ import {
import { SaveQuestionProvider } from "metabase/common/components/SaveQuestionForm/context";
import type { SaveQuestionProps } from "metabase/common/components/SaveQuestionForm/types";
import { useEscapeToCloseModal } from "metabase/common/hooks/use-escape-to-close-modal";
+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";
import { Flex, Modal, type ModalProps } from "metabase/ui";
type SaveQuestionModalProps = Omit<SaveQuestionProps, "initialDashboardTabId"> &
@@ -14,6 +15,7 @@ export const SaveQuestionModal = ({
multiStep,
onCreate,
onSave,
+ onClose,
originalQuestion,
question,
closeOnSuccess,
@@ -21,7 +23,19 @@ export const SaveQuestionModal = ({
targetCollection,
...modalProps
}: SaveQuestionModalProps) => {
- useEscapeToCloseModal(modalProps.onClose);
+ const {
+ checkData,
+ isConfirmationShown,
+ handleInitialSave,
+ handleSaveAfterConfirmation,
+ } = PLUGIN_DEPENDENCIES.useCheckCardDependencies({
+ onSave,
+ onError: (error) => {
+ throw error;
+ },
+ });
+ useEscapeToCloseModal(onClose);
+
return (
<SaveQuestionProvider
question={question}
@@ -30,32 +44,51 @@ export const SaveQuestionModal = ({
const newQuestion = await onCreate(question, options);
if (closeOnSuccess) {
- modalProps.onClose();
+ onClose();
}
return newQuestion;
}}
- onSave={onSave}
+ onSave={handleInitialSave}
+ onCancel={onClose}
multiStep={multiStep}
initialCollectionId={initialCollectionId}
targetCollection={targetCollection}
>
- <Modal.Root padding="2.5rem" {...modalProps} closeOnEscape={false}>
+ <Modal.Root
+ padding="xl"
+ {...modalProps}
+ size={isConfirmationShown ? "xl" : undefined}
+ closeOnEscape={false}
+ onClose={onClose}
+ >
<Modal.Overlay />
<Modal.Content data-testid="save-question-modal">
- <Modal.Header>
+ <Modal.Header px={isConfirmationShown ? "xl" : undefined}>
<Modal.Title>
- <SaveQuestionTitle />
+ {isConfirmationShown ? (
+ <PLUGIN_DEPENDENCIES.CheckDependenciesTitle />
+ ) : (
+ <SaveQuestionTitle />
+ )}
</Modal.Title>
- <Flex align="center" justify="flex-end" gap="sm">
+ <Flex justify="flex-end">
<Modal.CloseButton />
</Flex>
</Modal.Header>
- <Modal.Body>
- <SaveQuestionForm
- onSaveSuccess={() => closeOnSuccess && modalProps.onClose()}
- onCancel={modalProps.onClose}
- />
+ <Modal.Body px={isConfirmationShown ? 0 : undefined}>
+ {checkData != null && isConfirmationShown ? (
+ <PLUGIN_DEPENDENCIES.CheckDependenciesForm
+ checkData={checkData}
+ onSave={handleSaveAfterConfirmation}
+ onCancel={onClose}
+ />
+ ) : (
+ <SaveQuestionForm
+ onSaveSuccess={() => closeOnSuccess && onClose()}
+ onCancel={onClose}
+ />
+ )}
</Modal.Body>
</Modal.Content>
</Modal.Root>
diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.ts
index a940c7f5731..7677c7e36b4 100644
--- a/frontend/src/metabase/plugins/index.ts
+++ b/frontend/src/metabase/plugins/index.ts
@@ -58,6 +58,7 @@ import type {
Bookmark,
CacheableDashboard,
CacheableModel,
+ CheckDependenciesResponse,
Collection,
CollectionAuthorityLevelConfig,
CollectionEssentials,
@@ -86,6 +87,8 @@ import type {
Timeline,
TimelineEvent,
Transform,
+ UpdateSnippetRequest,
+ UpdateTransformRequest,
User,
VisualizationDisplay,
} from "metabase-types/api";
@@ -871,3 +874,66 @@ export const PLUGIN_TRANSFORMS_PYTHON: PythonTransformsPlugin = {
getTransformsNavLinks: () => null,
getCreateTransformsMenuItems: () => null,
};
+
+type DependenciesPlugin = {
+ CheckDependenciesForm: ComponentType<CheckDependenciesFormProps>;
+ CheckDependenciesModal: ComponentType<CheckDependenciesModalProps>;
+ CheckDependenciesTitle: ComponentType;
+ useCheckCardDependencies: (
+ props: UseCheckDependenciesProps<Question>,
+ ) => UseCheckDependenciesResult<Question>;
+ useCheckSnippetDependencies: (
+ props: UseCheckDependenciesProps<UpdateSnippetRequest>,
+ ) => UseCheckDependenciesResult<UpdateSnippetRequest>;
+ useCheckTransformDependencies: (
+ props: UseCheckDependenciesProps<UpdateTransformRequest>,
+ ) => UseCheckDependenciesResult<UpdateTransformRequest>;
+};
+
+export type CheckDependenciesFormProps = {
+ checkData: CheckDependenciesResponse;
+ onSave: () => void | Promise<void>;
+ onCancel: () => void;
+};
+
+export type CheckDependenciesModalProps = {
+ checkData: CheckDependenciesResponse;
+ opened: boolean;
+ onSave: () => void | Promise<void>;
+ onClose: () => void;
+};
+
+export type UseCheckDependenciesProps<TChange> = {
+ onSave: (change: TChange) => Promise<void>;
+ onError: (error: unknown) => void;
+};
+
+export type UseCheckDependenciesResult<TChange> = {
+ checkData?: CheckDependenciesResponse;
+ isCheckingDependencies: boolean;
+ isConfirmationShown: boolean;
+ handleInitialSave: (change: TChange) => Promise<void>;
+ handleSaveAfterConfirmation: () => Promise<void>;
+ handleCloseConfirmation: () => void;
+};
+
+function useCheckDependencies<TChange>({
+ onSave,
+}: UseCheckDependenciesProps<TChange>): UseCheckDependenciesResult<TChange> {
+ return {
+ isConfirmationShown: false,
+ isCheckingDependencies: false,
+ handleInitialSave: onSave,
+ handleSaveAfterConfirmation: () => Promise.resolve(),
+ handleCloseConfirmation: () => undefined,
+ };
+}
+
+export const PLUGIN_DEPENDENCIES: DependenciesPlugin = {
+ CheckDependenciesForm: PluginPlaceholder,
+ CheckDependenciesModal: PluginPlaceholder,
+ CheckDependenciesTitle: PluginPlaceholder,
+ useCheckCardDependencies: useCheckDependencies,
+ useCheckSnippetDependencies: useCheckDependencies,
+ useCheckTransformDependencies: useCheckDependencies,
+};
diff --git a/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsx b/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsx
index e8e0aa53dd3..5eb24df3f43 100644
--- a/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsx
+++ b/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsx
@@ -5,14 +5,18 @@ import {
type ReactNode,
useCallback,
useEffect,
+ useLayoutEffect,
useMemo,
+ useRef,
useState,
} from "react";
import { useMount, usePrevious } from "react-use";
import { t } from "ttag";
import { useListModelIndexesQuery } from "metabase/api";
-import ActionButton from "metabase/common/components/ActionButton";
+import ActionButton, {
+ type ActionButtonHandle,
+} from "metabase/common/components/ActionButton";
import Button from "metabase/common/components/Button";
import DebouncedFrame from "metabase/common/components/DebouncedFrame";
import EditBar from "metabase/common/components/EditBar";
@@ -21,6 +25,7 @@ import ButtonsS from "metabase/css/components/buttons.module.css";
import CS from "metabase/css/core/index.css";
import { connect, useDispatch } from "metabase/lib/redux";
import { getSemanticTypeIcon } from "metabase/lib/schema_metadata";
+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";
import {
setDatasetEditorTab,
setUIControls,
@@ -474,6 +479,28 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {
}
};
+ const saveButtonRef = useRef<ActionButtonHandle>(null);
+ const {
+ checkData,
+ isConfirmationShown,
+ handleInitialSave,
+ handleSaveAfterConfirmation,
+ handleCloseConfirmation,
+ } = PLUGIN_DEPENDENCIES.useCheckCardDependencies({
+ onSave: async (question) => {
+ await onSave(question, { rerunQuery: true });
+ await setQueryBuilderMode("view");
+ runQuestionQuery();
+ },
+ onError: (error) => {
+ throw error;
+ },
+ });
+
+ useLayoutEffect(() => {
+ saveButtonRef.current?.resetState();
+ }, [isConfirmationShown]);
+
const handleSave = useCallback(async () => {
const canBeDataset = checkCanBeModel(question);
const isBrandNewDataset = !question.id();
@@ -495,9 +522,7 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {
});
onOpenModal(MODAL_TYPES.SAVE);
} else if (canBeDataset) {
- await onSave(questionWithMetadata, { rerunQuery: true });
- await setQueryBuilderMode("view");
- runQuestionQuery();
+ await handleInitialSave(questionWithMetadata);
} else {
onOpenModal(MODAL_TYPES.CAN_NOT_CREATE_MODEL);
throw new Error(t`Variables in models aren't supported yet`);
@@ -510,9 +535,7 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {
dispatch,
updateQuestion,
onOpenModal,
- onSave,
- setQueryBuilderMode,
- runQuestionQuery,
+ handleInitialSave,
]);
const handleColumnSelect = useCallback(
@@ -670,6 +693,7 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {
disabled={!saveButtonTooltipLabel}
>
<ActionButton
+ ref={saveButtonRef}
key="save"
disabled={!canSaveChanges}
actionFn={handleSave}
@@ -744,6 +768,15 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {
onConfirm={handleCancelEdit}
onClose={closeModal}
/>
+
+ {isConfirmationShown && checkData != null && (
+ <PLUGIN_DEPENDENCIES.CheckDependenciesModal
+ checkData={checkData}
+ opened
+ onSave={handleSaveAfterConfirmation}
+ onClose={handleCloseConfirmation}
+ />
+ )}
</>
);
};
diff --git a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx
index 497db4ab688..970e8213331 100644
--- a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx
+++ b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx
@@ -10,7 +10,6 @@ import { ResizableBox, type ResizableBoxProps } from "react-resizable";
import _ from "underscore";
import ExplicitSize from "metabase/common/components/ExplicitSize";
-import Modal from "metabase/common/components/Modal";
import Databases from "metabase/entities/databases";
import SnippetCollections from "metabase/entities/snippet-collections";
import Snippets from "metabase/entities/snippets";
@@ -20,7 +19,7 @@ import {
setIsNativeEditorOpen,
setUIControls,
} from "metabase/query_builder/actions";
-import SnippetFormModal from "metabase/query_builder/components/template_tags/SnippetFormModal";
+import { SnippetFormModal } from "metabase/query_builder/components/template_tags/SnippetFormModal";
import type { QueryModalType } from "metabase/query_builder/constants";
import { useNotebookScreenSize } from "metabase/query_builder/hooks/use-notebook-screen-size";
import { Flex } from "metabase/ui";
@@ -372,8 +371,9 @@ class NativeQueryEditor extends Component<Props, NativeQueryEditorState> {
canSaveSnippets={canSaveSnippets}
/>
- {this.props.modalSnippet && (
- <Modal onClose={this.props.closeSnippetModal}>
+ {this.props.modalSnippet &&
+ this.props.insertSnippet &&
+ this.props.closeSnippetModal && (
<SnippetFormModal
snippet={this.props.modalSnippet}
onCreate={this.props.insertSnippet}
@@ -387,8 +387,7 @@ class NativeQueryEditor extends Component<Props, NativeQueryEditorState> {
}}
onClose={this.props.closeSnippetModal}
/>
- </Modal>
- )}
+ )}
</div>
);
}
diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsx b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsx
index 6a6b99db2eb..653d668dd90 100644
--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsx
+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsx
@@ -1,26 +1,19 @@
-import { useCallback, useMemo } from "react";
+import { useMemo } from "react";
import { t } from "ttag";
import _ from "underscore";
import * as Yup from "yup";
import FormCollectionPicker from "metabase/collections/containers/FormCollectionPicker";
-import { canonicalCollectionId } from "metabase/collections/utils";
import Button from "metabase/common/components/Button";
import FormErrorMessage from "metabase/common/components/FormErrorMessage";
import FormInput from "metabase/common/components/FormInput";
import FormSubmitButton from "metabase/common/components/FormSubmitButton";
import FormTextArea from "metabase/common/components/FormTextArea";
import SnippetCollections from "metabase/entities/snippet-collections";
-import Snippets from "metabase/entities/snippets";
import { Form, FormProvider } from "metabase/forms";
import * as Errors from "metabase/lib/errors";
-import { connect } from "metabase/lib/redux";
import { Flex } from "metabase/ui";
-import type {
- Collection,
- NativeQuerySnippet,
- NativeQuerySnippetId,
-} from "metabase-types/api";
+import type { Collection, NativeQuerySnippet } from "metabase-types/api";
import S from "./SnippetForm.module.css";
@@ -37,60 +30,34 @@ const SNIPPET_SCHEMA = Yup.object({
collection_id: Yup.number().nullable().default(null),
});
-type SnippetFormValues = Pick<
+export type SnippetFormValues = Pick<
NativeQuerySnippet,
"name" | "description" | "content" | "collection_id"
>;
-type UpdateSnippetFormValues = Partial<SnippetFormValues> &
- Pick<NativeQuerySnippet, "id"> & {
- archived?: boolean;
- };
-
export interface SnippetFormOwnProps {
- snippet: Partial<NativeQuerySnippet>;
- onCreate?: (snippet: NativeQuerySnippet) => void;
- onUpdate?: (
- nextSnippet: NativeQuerySnippet,
- originalSnippet: NativeQuerySnippet,
- ) => void;
- onArchive?: () => void;
+ snippet?: Partial<SnippetFormValues>;
+ isEditing: boolean;
+ isDirty?: boolean;
+ onSubmit: (snippet: SnippetFormValues) => void | Promise<void>;
+ onArchive?: () => void | Promise<void>;
onCancel?: () => void;
}
interface SnippetLoaderProps {
snippetCollections: Collection[];
}
-
-interface SnippetFormDispatchProps {
- handleCreateSnippet: (
- snippet: SnippetFormValues,
- ) => Promise<NativeQuerySnippet>;
- handleUpdateSnippet: (
- snippet: UpdateSnippetFormValues,
- ) => Promise<NativeQuerySnippet>;
-}
-
-type SnippetFormProps = SnippetFormOwnProps &
- SnippetLoaderProps &
- SnippetFormDispatchProps;
-
-const mapDispatchToProps = {
- handleCreateSnippet: Snippets.actions.create,
- handleUpdateSnippet: Snippets.actions.update,
-};
+type SnippetFormProps = SnippetFormOwnProps & SnippetLoaderProps;
function SnippetForm({
snippet,
snippetCollections,
- handleCreateSnippet,
- handleUpdateSnippet,
- onCreate,
- onUpdate,
+ isEditing,
+ isDirty: isInitiallyDirty = false,
+ onSubmit,
onArchive,
onCancel,
}: SnippetFormProps) {
- const isEditing = snippet.id != null;
const hasManyCollections = snippetCollections.length > 1;
const initialValues = useMemo(
@@ -98,65 +65,26 @@ function SnippetForm({
SNIPPET_SCHEMA.cast(
{
...snippet,
- content: snippet.content || "",
- parent_id: canonicalCollectionId(snippet.id),
+ content: snippet?.content || "",
},
{ stripUnknown: true },
),
[snippet],
);
- const handleCreate = useCallback(
- async (values: SnippetFormValues) => {
- const action = await handleCreateSnippet(values);
- const snippet = Snippets.HACK_getObjectFromAction(action);
- onCreate?.(snippet);
- },
- [handleCreateSnippet, onCreate],
- );
-
- const handleUpdate = useCallback(
- async (values: UpdateSnippetFormValues) => {
- const action = await handleUpdateSnippet(values);
- const nextSnippet = Snippets.HACK_getObjectFromAction(action);
- onUpdate?.(nextSnippet, snippet as NativeQuerySnippet);
- },
- [snippet, handleUpdateSnippet, onUpdate],
- );
-
- const handleSubmit = useCallback(
- async (values: SnippetFormValues) => {
- if (isEditing && snippet.id) {
- await handleUpdate({ ...values, id: snippet.id });
- } else {
- await handleCreate(values);
- }
- },
- [snippet.id, isEditing, handleCreate, handleUpdate],
- );
-
- const handleArchive = useCallback(async () => {
- await handleUpdateSnippet({
- id: snippet.id as NativeQuerySnippetId,
- archived: true,
- });
- onArchive?.();
- }, [snippet.id, handleUpdateSnippet, onArchive]);
-
return (
<FormProvider
initialValues={initialValues}
validationSchema={SNIPPET_SCHEMA}
- onSubmit={handleSubmit}
+ onSubmit={onSubmit}
>
{({ dirty }) => (
- <Form disabled={!dirty} className={S.SnippetForm}>
+ <Form disabled={!dirty && !isInitiallyDirty} className={S.SnippetForm}>
<FormTextArea
inputClassName={S.FormSnippetTextArea}
name="content"
title={t`Enter some SQL here so you can reuse it later`}
placeholder="AND canceled_at IS null\nAND account_type = 'PAID'"
- autoFocus
rows={4}
/>
<FormInput
@@ -184,8 +112,10 @@ function SnippetForm({
type="button"
icon="archive"
borderless
- onClick={handleArchive}
- >{t`Archive`}</Button>
+ onClick={onArchive}
+ >
+ {t`Archive`}
+ </Button>
)}
<FormErrorMessage inline />
</Flex>
@@ -193,7 +123,11 @@ function SnippetForm({
{!!onCancel && (
<Button type="button" onClick={onCancel}>{t`Cancel`}</Button>
)}
- <FormSubmitButton title={t`Save`} disabled={!dirty} primary />
+ <FormSubmitButton
+ title={t`Save`}
+ disabled={!dirty && !isInitiallyDirty}
+ primary
+ />
</Flex>
</Flex>
</Form>
@@ -203,7 +137,4 @@ function SnippetForm({
}
// eslint-disable-next-line import/no-default-export -- deprecated usage
-export default _.compose(
- SnippetCollections.loadList(),
- connect(null, mapDispatchToProps),
-)(SnippetForm);
+export default _.compose(SnippetCollections.loadList())(SnippetForm);
diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.ts b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.ts
index d01b3b192e1..064e4847c96 100644
--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.ts
+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.ts
@@ -1,3 +1,3 @@
// eslint-disable-next-line import/no-default-export -- deprecated usage
export { default } from "./SnippetForm";
-export type { SnippetFormOwnProps } from "./SnippetForm";
+export type { SnippetFormValues } from "./SnippetForm";
diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsx b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsx
index c0664fcc887..d25dea8265f 100644
--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsx
+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsx
@@ -1,60 +1,138 @@
-import { useCallback } from "react";
+import { useCallback, useState } from "react";
import { t } from "ttag";
-import ModalContent from "metabase/common/components/ModalContent";
-import type { NativeQuerySnippet } from "metabase-types/api";
+import Snippets from "metabase/entities/snippets";
+import { useDispatch } from "metabase/lib/redux";
+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";
+import { Flex, Modal } from "metabase/ui";
+import type {
+ CreateSnippetRequest,
+ NativeQuerySnippet,
+ UpdateSnippetRequest,
+} from "metabase-types/api";
-import type { SnippetFormOwnProps } from "../SnippetForm";
-import SnippetForm from "../SnippetForm";
+import SnippetForm, { type SnippetFormValues } from "../SnippetForm";
-interface SnippetFormModalOwnProps
- extends Omit<SnippetFormOwnProps, "onCancel"> {
- onClose?: () => void;
-}
-
-type SnippetModalProps = SnippetFormModalOwnProps;
+type SnippetModalProps = {
+ snippet: NativeQuerySnippet | Partial<Omit<NativeQuerySnippet, "id">>;
+ onCreate: (snippet: NativeQuerySnippet) => void;
+ onUpdate: (
+ nextSnippet: NativeQuerySnippet,
+ originalSnippet: NativeQuerySnippet,
+ ) => void;
+ onClose: () => void;
+};
-function SnippetFormModal({
- snippet,
+export function SnippetFormModal({
+ snippet: initialSnippet,
onCreate,
onUpdate,
onClose,
- ...props
}: SnippetModalProps) {
- const isEditing = snippet.id != null;
- const title = isEditing
+ const [snippet, setSnippet] = useState(initialSnippet);
+ const dispatch = useDispatch();
+ const isEditing = isSavedSnippet(snippet);
+ const modalTitle = isEditing
? t`Editing ${snippet.name}`
: t`Create your new snippet`;
const handleCreate = useCallback(
- (snippet: NativeQuerySnippet) => {
+ async (values: CreateSnippetRequest) => {
+ const action = await dispatch(Snippets.actions.create(values));
+ const snippet = Snippets.HACK_getObjectFromAction(action);
onCreate?.(snippet);
onClose?.();
},
- [onCreate, onClose],
+ [dispatch, onCreate, onClose],
);
const handleUpdate = useCallback(
- (nextSnippet: NativeQuerySnippet, originalSnippet: NativeQuerySnippet) => {
- onUpdate?.(nextSnippet, originalSnippet);
+ async (values: UpdateSnippetRequest) => {
+ if (!isSavedSnippet(snippet)) {
+ return;
+ }
+ const action = await dispatch(Snippets.actions.update(snippet, values));
+ const nextSnippet = Snippets.HACK_getObjectFromAction(action);
+ onUpdate?.(nextSnippet, snippet);
onClose?.();
},
- [onUpdate, onClose],
+ [snippet, dispatch, onUpdate, onClose],
+ );
+
+ const handleArchive = useCallback(async () => {
+ if (!isSavedSnippet(snippet)) {
+ return;
+ }
+ await dispatch(Snippets.actions.update({ id: snippet.id, archived: true }));
+ onClose?.();
+ }, [snippet, dispatch, onClose]);
+
+ const {
+ checkData,
+ isConfirmationShown,
+ handleInitialSave,
+ handleSaveAfterConfirmation,
+ handleCloseConfirmation,
+ } = PLUGIN_DEPENDENCIES.useCheckSnippetDependencies({
+ onSave: handleUpdate,
+ onError: (error) => {
+ throw error;
+ },
+ });
+
+ const handleSubmit = useCallback(
+ async (values: SnippetFormValues) => {
+ if (isSavedSnippet(snippet)) {
+ setSnippet({ ...snippet, ...values });
+ await handleInitialSave({ ...values, id: snippet.id });
+ } else {
+ await handleCreate(values);
+ }
+ },
+ [snippet, handleCreate, handleInitialSave],
);
return (
- <ModalContent title={title} onClose={onClose}>
- <SnippetForm
- {...props}
- snippet={snippet}
- onCreate={handleCreate}
- onUpdate={handleUpdate}
- onArchive={onClose}
- onCancel={onClose}
- />
- </ModalContent>
+ <Modal.Root padding="xl" opened onClose={onClose}>
+ <Modal.Overlay />
+ <Modal.Content>
+ <Modal.Header px="xl">
+ <Modal.Title>
+ {isConfirmationShown ? (
+ <PLUGIN_DEPENDENCIES.CheckDependenciesTitle />
+ ) : (
+ modalTitle
+ )}
+ </Modal.Title>
+ <Flex justify="flex-end">
+ <Modal.CloseButton />
+ </Flex>
+ </Modal.Header>
+ <Modal.Body px={isConfirmationShown ? 0 : "xl"}>
+ {isConfirmationShown && checkData != null ? (
+ <PLUGIN_DEPENDENCIES.CheckDependenciesForm
+ checkData={checkData}
+ onSave={handleSaveAfterConfirmation}
+ onCancel={handleCloseConfirmation}
+ />
+ ) : (
+ <SnippetForm
+ snippet={snippet}
+ isEditing={isEditing}
+ isDirty={checkData != null}
+ onSubmit={handleSubmit}
+ onArchive={handleArchive}
+ onCancel={onClose}
+ />
+ )}
+ </Modal.Body>
+ </Modal.Content>
+ </Modal.Root>
);
}
-// eslint-disable-next-line import/no-default-export -- deprecated usage
-export default SnippetFormModal;
+function isSavedSnippet(
+ snippet: NativeQuerySnippet | Partial<Omit<NativeQuerySnippet, "id">>,
+): snippet is NativeQuerySnippet {
+ return "id" in snippet;
+}
diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsx b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsx
index 2df4da027ed..a8c9c1fafba 100644
--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsx
+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsx
@@ -13,7 +13,7 @@ import {
createMockNativeQuerySnippet,
} from "metabase-types/api/mocks";
-import SnippetFormModal from "./SnippetFormModal";
+import { SnippetFormModal } from "./SnippetFormModal";
const TOP_SNIPPETS_FOLDER = {
id: "root",
@@ -22,15 +22,15 @@ const TOP_SNIPPETS_FOLDER = {
};
type SetupOpts = {
- snippet?: Partial<NativeQuerySnippet>;
- onClose?: null | (() => void);
+ snippet?:
+ | NativeQuerySnippet
+ | (Omit<Partial<NativeQuerySnippet>, "id"> & { id?: undefined });
withDefaultFoldersList?: boolean;
};
async function setup({
snippet = {},
withDefaultFoldersList = true,
- onClose = jest.fn(),
}: SetupOpts = {}) {
fetchMock.get({
url: "path:/api/collection/root",
@@ -63,8 +63,16 @@ async function setup({
);
}
+ const onCreate = jest.fn();
+ const onUpdate = jest.fn();
+ const onClose = jest.fn();
renderWithProviders(
- <SnippetFormModal snippet={snippet} onClose={onClose || undefined} />,
+ <SnippetFormModal
+ snippet={snippet}
+ onCreate={onCreate}
+ onUpdate={onUpdate}
+ onClose={onClose}
+ />,
);
await waitForLoaderToBeRemoved();
@@ -159,13 +167,6 @@ describe("SnippetFormModal", () => {
});
});
- it("doesn't show cancel button if onClose props is not set", async () => {
- await setup({ onClose: null });
- expect(
- screen.queryByRole("button", { name: "Cancel" }),
- ).not.toBeInTheDocument();
- });
-
it("calls onClose when cancel button is clicked", async () => {
const { onClose } = await setup();
await userEvent.click(screen.getByRole("button", { name: "Cancel" }));
@@ -262,13 +263,6 @@ describe("SnippetFormModal", () => {
});
});
- it("doesn't show cancel button if onClose props is not set", async () => {
- await setupEditing({ onClose: null });
- expect(
- screen.queryByRole("button", { name: "Cancel" }),
- ).not.toBeInTheDocument();
- });
-
it("calls onClose when cancel button is clicked", async () => {
const { onClose } = await setupEditing();
await userEvent.click(screen.getByRole("button", { name: "Cancel" }));
diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.ts b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.ts
index d719f2e77bf..4ab22ced876 100644
--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.ts
+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.ts
@@ -1,2 +1 @@
-// eslint-disable-next-line import/no-default-export -- deprecated usage
-export { default } from "./SnippetFormModal";
+export * from "./SnippetFormModal";
diff --git a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsx b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsx
index 807963b3246..667db30f687 100644
--- a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsx
+++ b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsx
@@ -2,6 +2,7 @@ import { forwardRef, useState } from "react";
import { LeaveConfirmModal } from "metabase/common/components/LeaveConfirmModal";
import { SaveQuestionModal } from "metabase/common/components/SaveQuestionModal";
+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";
import { Flex } from "metabase/ui";
import * as Lib from "metabase-lib";
import type Question from "metabase-lib/v1/Question";
@@ -29,7 +30,7 @@ type MetricEditorProps = {
};
export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(
- function MetricEditorInner(
+ function MetricEditor(
{
question,
result,
@@ -49,6 +50,18 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(
) {
const [modalType, setModalType] = useState<MetricModalType>();
const isRunnable = Lib.canRun(question.query(), "metric");
+ const {
+ checkData,
+ isConfirmationShown,
+ handleInitialSave,
+ handleSaveAfterConfirmation,
+ handleCloseConfirmation,
+ } = PLUGIN_DEPENDENCIES.useCheckCardDependencies({
+ onSave,
+ onError: (error) => {
+ throw error;
+ },
+ });
const handleCreate = (question: Question) => {
return onCreate(question.setDefaultDisplay());
@@ -59,7 +72,7 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(
};
const handleSave = async (question: Question) => {
- await onSave(question.setDefaultDisplay());
+ await handleInitialSave(question.setDefaultDisplay());
};
const handleConfirmCancel = () => {
@@ -84,6 +97,7 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(
question={question}
isDirty={isDirty}
isRunnable={isRunnable}
+ isConfirmationShown={isConfirmationShown}
onCreate={handleCreateStart}
onSave={handleSave}
onCancel={handleCancelStart}
@@ -124,6 +138,14 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(
onClose={handleModalClose}
/>
)}
+ {isConfirmationShown && checkData != null && (
+ <PLUGIN_DEPENDENCIES.CheckDependenciesModal
+ checkData={checkData}
+ opened
+ onSave={handleSaveAfterConfirmation}
+ onClose={handleCloseConfirmation}
+ />
+ )}
</Flex>
);
},
diff --git a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsx b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsx
index 8c907a71927..8581c38239d 100644
--- a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsx
+++ b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsx
@@ -1,7 +1,10 @@
import cx from "classnames";
+import { useLayoutEffect, useRef } from "react";
import { t } from "ttag";
-import ActionButton from "metabase/common/components/ActionButton";
+import ActionButton, {
+ type ActionButtonHandle,
+} from "metabase/common/components/ActionButton";
import Button from "metabase/common/components/Button";
import EditBar from "metabase/common/components/EditBar";
import ButtonsS from "metabase/css/components/buttons.module.css";
@@ -13,6 +16,7 @@ type MetricEditorHeaderProps = {
question: Question;
isDirty: boolean;
isRunnable: boolean;
+ isConfirmationShown: boolean;
onCreate: (question: Question) => void;
onSave: (question: Question) => Promise<void>;
onCancel: () => void;
@@ -22,13 +26,19 @@ export function MetricEditorHeader({
question,
isDirty,
isRunnable,
+ isConfirmationShown,
onCreate,
onSave,
onCancel,
}: MetricEditorHeaderProps) {
+ const saveButtonRef = useRef<ActionButtonHandle>(null);
const handleCreate = () => onCreate(question);
const handleSave = () => onSave(question);
+ useLayoutEffect(() => {
+ saveButtonRef.current?.resetState();
+ }, [isConfirmationShown]);
+
return (
<EditBar
className={S.root}
@@ -41,6 +51,7 @@ export function MetricEditorHeader({
</Button>
) : (
<ActionButton
+ ref={saveButtonRef}
key="save"
actionFn={handleSave}
disabled={!isRunnable || !isDirty}
diff --git a/frontend/test/__support__/server-mocks/dependencies.ts b/frontend/test/__support__/server-mocks/dependencies.ts
new file mode 100644
index 00000000000..5c7696ec4e3
--- /dev/null
+++ b/frontend/test/__support__/server-mocks/dependencies.ts
@@ -0,0 +1,21 @@
+import fetchMock from "fetch-mock";
+
+import type { CheckDependenciesResponse } from "metabase-types/api";
+
+export function setupCheckCardDependenciesEndpoint(
+ response: CheckDependenciesResponse,
+) {
+ fetchMock.post("path:/api/ee/dependencies/check_card", response);
+}
+
+export function setupCheckSnippetDependenciesEndpoint(
+ response: CheckDependenciesResponse,
+) {
+ fetchMock.post("path:/api/ee/dependencies/check_snippet", response);
+}
+
+export function setupCheckTransformDependenciesEndpoint(
+ response: CheckDependenciesResponse,
+) {
+ fetchMock.post("path:/api/ee/dependencies/check_transform", response);
+}
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index d01238d7bb5..01585a6e792 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -15,6 +15,7 @@
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
+ [metabase.driver.sql.normalize :as driver.sql.normalize]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.util :as sql.u]
[metabase.driver.sync :as driver.s]
@@ -998,7 +999,7 @@
transforms (t2/select [:model/Transform :id :target])]
(into #{} (comp
(map :component)
- (map #(assoc % :table (driver.sql/normalize-name driver (:table %))))
+ (map #(assoc % :table (driver.sql.normalize/normalize-name driver (:table %))))
(map #(let [parts (str/split (:table %) #"\.")]
{:schema (first parts) :table (second parts)}))
(keep #(driver.sql/find-table-or-transform driver db-tables transforms %)))
diff --git a/resources/migrations/057_update_migrations.yaml b/resources/migrations/057_update_migrations.yaml
index 6ea57204974..eed31a70b25 100644
--- a/resources/migrations/057_update_migrations.yaml
+++ b/resources/migrations/057_update_migrations.yaml
@@ -1352,6 +1352,187 @@ databaseChangeLog:
- column:
name: comment_id
rollback: # will be removed along the table
+ - changeSet:
+ id: v57.2025-09-08T17:50:00
+ author: metamben
+ comment: Drop old dependency table
+ preConditions:
+ - onFail: MARK_RAN
+ - tableExists:
+ tableName: dependency
+ changes:
+ - dropTable:
+ tableName: dependency
+ rollback:
+ - createTable:
+ tableName: dependency
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: model
+ type: varchar(32)
+ constraints:
+ nullable: false
+ - column:
+ name: model_id
+ type: int
+ constraints:
+ nullable: false
+ - column:
+ name: dependent_on_model
+ type: varchar(32)
+ constraints:
+ nullable: false
+ - column:
+ name: dependent_on_id
+ type: int
+ constraints:
+ nullable: false
+ - column:
+ name: created_at
+ type: DATETIME
+ constraints:
+ nullable: false
+ - createIndex:
+ tableName: dependency
+ indexName: idx_dependency_model
+ columns:
+ column:
+ name: model
+ - createIndex:
+ tableName: dependency
+ indexName: idx_dependency_model_id
+ columns:
+ column:
+ name: model_id
+ - createIndex:
+ tableName: dependency
+ indexName: idx_dependency_dependent_on_model
+ columns:
+ column:
+ name: dependent_on_model
+ - createIndex:
+ tableName: dependency
+ indexName: idx_dependency_dependent_on_id
+ columns:
+ column:
+ name: dependent_on_id
+ # - modifySql: # doesn't work with liquibase 4.33.0
+ # dbms: postgresql
+ # replace:
+ # replace: WITHOUT
+ # with: WITH
+
+ - changeSet:
+ id: v57.2025-09-08T18:00:00
+ author: metamben
+ comment: Create dependency table to track dependencies in a general way
+
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - tableExists:
+ tableName: dependency
+ changes:
+ - createTable:
+ tableName: dependency
+ remarks: A table to track dependencies between Metabase entities
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: from_entity_type
+ type: varchar(20)
+ remarks: The type of the dependent entity
+ constraints:
+ nullable: false
+ - column:
+ name: from_entity_id
+ type: int
+ remarks: The ID of the dependent entity
+ constraints:
+ nullable: false
+ - column:
+ name: to_entity_type
+ type: varchar(20)
+ remarks: The type of the entity depended on
+ constraints:
+ nullable: false
+ - column:
+ name: to_entity_id
+ type: int
+ remarks: The ID of the entity depended on
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v57.2025-09-08T18:10:00
+ author: metamben
+ comment: Unique constraint to prevent duplicate dependencies
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - uniqueConstraintExists:
+ tableName: dependency
+ constraintName: idx_unique_dependency
+ changes:
+ - addUniqueConstraint:
+ tableName: dependency
+ columnNames: from_entity_type, from_entity_id, to_entity_type, to_entity_id
+ constraintName: idx_unique_dependency
+
+ - changeSet:
+ id: v57.2025-09-08T18:20:00
+ author: metamben
+ comment: Index for finding dependencies FROM a specific entity
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - indexExists:
+ tableName: dependency
+ indexName: idx_dependency_from
+ changes:
+ - createIndex:
+ tableName: dependency
+ indexName: idx_dependency_from
+ columns:
+ - column:
+ name: from_entity_type
+ - column:
+ name: from_entity_id
+
+ - changeSet:
+ id: v57.2025-09-08T18:30:00
+ author: metamben
+ comment: Index for finding dependencies TO a specific entity
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - indexExists:
+ tableName: dependency
+ indexName: idx_dependency_to
+ changes:
+ - createIndex:
+ tableName: dependency
+ indexName: idx_dependency_to
+ columns:
+ - column:
+ name: to_entity_type
+ - column:
+ name: to_entity_id
- changeSet:
id: v57.2025-09-11T10:00:00
@@ -1465,6 +1646,85 @@ databaseChangeLog:
constraints:
nullable: true
+ - changeSet:
+ id: v57.2025-09-18T10:00:00
+ author: metamben
+ comment: Add dependency_analysis_version to report_card
+ changes:
+ - addColumn:
+ tableName: report_card
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ remarks: Version of the dependency analysis for this entity.
+ constraints:
+ nullable: false
+ - changeSet:
+ id: v57.2025-09-18T10:01:00
+ author: metamben
+ comment: Add index for dependency_analysis_version to report_card
+ changes:
+ - createIndex:
+ tableName: report_card
+ indexName: idx_report_card_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+ - changeSet:
+ id: v57.2025-09-18T10:02:00
+ author: metamben
+ comment: Add dependency_analysis_version to transform
+ changes:
+ - addColumn:
+ tableName: transform
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ remarks: Version of the dependency analysis for this entity.
+ constraints:
+ nullable: false
+ - changeSet:
+ id: v57.2025-09-18T10:03:00
+ author: metamben
+ comment: Add index for dependency_analysis_version to transform
+ changes:
+ - createIndex:
+ tableName: transform
+ indexName: idx_transform_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+ - changeSet:
+ id: v57.2025-09-18T10:04:00
+ author: metamben
+ comment: Add dependency_analysis_version to native_query_snippet
+ changes:
+ - addColumn:
+ tableName: native_query_snippet
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ remarks: Version of the dependency analysis for this entity.
+ constraints:
+ nullable: false
+ - changeSet:
+ id: v57.2025-09-18T10:05:00
+ author: metamben
+ comment: Add index for dependency_analysis_version to native_query_snippet
+ changes:
+ - createIndex:
+ tableName: native_query_snippet
+ indexName: idx_native_query_snippet_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+
- changeSet:
id: v57.2025-09-19T11:30:20
author: ranquild
diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.clj
index 24ffd86dd74..8cc8067cffb 100644
--- a/src/metabase/analytics/stats.clj
+++ b/src/metabase/analytics/stats.clj
@@ -887,7 +887,10 @@
:enabled (premium-features/enable-transforms?)}
{:name :transforms-python
:available (premium-features/enable-python-transforms?)
- :enabled (premium-features/enable-python-transforms?)}])
+ :enabled (premium-features/enable-python-transforms?)}
+ {:name :dependencies
+ :available (premium-features/enable-dependencies?)
+ :enabled (premium-features/enable-dependencies?)}])
(defn- snowplow-features
[]
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 72a26c449e3..6a955cf82b5 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -745,6 +745,9 @@
;; Does this driver support executing python transforms?
:transforms/python
+ ;; Does this driver support calculating dependencies of native queries?
+ :dependencies/native
+
;; Does this driver properly support the table-exists? method for checking table existence?
:metadata/table-existence-check
@@ -1195,7 +1198,20 @@
"Gets the table dependencies of a given sql string (or equivalent).
Drivers that support any of the `:transforms/...` features must implement this method."
- {:added "0.57.0" :arglists '([driver query])}
+ {:added "0.57.0" :arglists '([driver query] [driver query metadata-provider transforms])}
+ dispatch-on-initialized-driver
+ :hierarchy #'hierarchy)
+
+(defmulti native-result-metadata
+ "Gets the result-metadata for a native query using static analysis (i.e., without actually
+ going to the database)."
+ {:added "0.57.0" :arglists '([driver metadata-provider native-query])}
+ dispatch-on-initialized-driver
+ :hierarchy #'hierarchy)
+
+(defmulti validate-native-query-fields
+ "Validates a native query and returns a list of all 'bad' field references."
+ {:added "0.57.0" :arglists '([driver metadata-provider native-query])}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj
index d438b5bbf40..b9e418a4129 100644
--- a/src/metabase/driver/h2.clj
+++ b/src/metabase/driver/h2.clj
@@ -9,11 +9,13 @@
[metabase.driver.common :as driver.common]
[metabase.driver.h2.actions :as h2.actions]
[metabase.driver.settings :as driver.settings]
+ [metabase.driver.sql :as sql]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
+ [metabase.driver.sql.normalize :as sql.normalize]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
@@ -668,3 +670,11 @@
(defmethod sql-jdbc/impl-table-known-to-not-exist? :h2
[_ e]
(#{"42S02" "42S03" "42S04"} (sql-jdbc/get-sql-state e)))
+
+(defmethod sql.normalize/normalize-unquoted-name :h2
+ [_ name-str]
+ (u/upper-case-en name-str))
+
+(defmethod sql/default-schema :h2
+ [_]
+ "PUBLIC")
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index 96011f887e3..a735196d350 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -3,22 +3,21 @@
(:refer-clojure :exclude [some])
(:require
[clojure.set :as set]
- [clojure.string :as str]
[macaw.core :as macaw]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.common.parameters.parse :as params.parse]
[metabase.driver.common.parameters.values :as params.values]
+ [metabase.driver.sql.normalize :as sql.normalize]
[metabase.driver.sql.parameters.substitute :as sql.params.substitute]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sql.references :as sql.references]
[metabase.driver.sql.util :as sql.u]
- [metabase.util :as u]
+ [metabase.util.humanization :as u.humanization]
[metabase.util.malli :as mu]
[metabase.util.performance :refer [some]]
- [potemkin :as p]
- ^{:clj-kondo/ignore [:discouraged-namespace]}
- [toucan2.core :as t2]))
+ [potemkin :as p]))
(comment sql.params.substitution/keep-me) ; this is so `cljr-clean-ns` and the linter don't remove the `:require`
@@ -48,7 +47,8 @@
:expressions/text
:expressions/today
:distinct-where
- :database-routing]]
+ :database-routing
+ :dependencies/native]]
(defmethod driver/database-supports? [:sql feature] [_driver _feature _db] true))
(defmethod driver/database-supports? [:sql :persist-models-enabled]
@@ -138,21 +138,6 @@
;; honeysql, and accepts a keyword too. This way we delegate proper escaping and qualification to honeysql.
(driver/drop-table! driver (:id database) (qualified-name target)))
-(defn normalize-name
- "Normalizes the (primarily table/column) name passed in.
- Should return a value that matches the name listed in the appdb."
- [driver name-str]
- (let [quote-style (sql.qp/quote-style driver)
- quote-char (if (= quote-style :mysql) \` \")]
- (if (and (= (first name-str) quote-char)
- (= (last name-str) quote-char))
- (let [quote-quote (str quote-char quote-char)
- quote (str quote-char)]
- (-> name-str
- (subs 1 (dec (count name-str)))
- (str/replace quote-quote quote)))
- (u/lower-case-en name-str))))
-
(defmulti default-schema
"Returns the default schema for a given database driver.
@@ -168,13 +153,12 @@
(defn find-table-or-transform
"Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.
It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."
- [driver tables transforms {:keys [table schema]}]
- (let [normalized-table (normalize-name driver table)
- normalized-schema (or (some->> schema (normalize-name driver))
- (default-schema driver))
+ [driver tables transforms {search-table :table raw-schema :schema}]
+ (let [search-schema (or raw-schema
+ (default-schema driver))
matches? (fn [db-table db-schema]
- (and (= normalized-table db-table)
- (= normalized-schema db-schema)))]
+ (and (= search-table db-table)
+ (= search-schema db-schema)))]
(or (some (fn [{:keys [name schema id]}]
(when (matches? name schema)
{:table id}))
@@ -184,16 +168,131 @@
{:transform id}))
transforms))))
+(defn- normalize-table-spec
+ [driver {:keys [table schema]}]
+ {:table (sql.normalize/normalize-name driver table)
+ :schema (some->> schema (sql.normalize/normalize-name driver))})
+
(defmethod driver/native-query-deps :sql
- [driver query]
- (let [db-tables (driver-api/tables (driver-api/metadata-provider))
- transforms (t2/select [:model/Transform :id :target])]
- (->> query
- macaw/parsed-query
- macaw/query->components
- :tables
- (map :component)
- (into #{} (keep #(find-table-or-transform driver db-tables transforms %))))))
+ ([driver query]
+ (driver/native-query-deps driver query
+ (driver-api/metadata-provider)))
+ ([driver query metadata-provider]
+ (let [db-tables (driver-api/tables metadata-provider)
+ db-transforms (driver-api/transforms metadata-provider)]
+ (->> query
+ macaw/parsed-query
+ macaw/query->components
+ :tables
+ (map :component)
+ (into #{} (keep #(->> (normalize-table-spec driver %)
+ (find-table-or-transform driver db-tables db-transforms))))))))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Dependencies |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(defmulti resolve-field
+ "Resolves a field reference to one or more actual database fields.
+
+ This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field
+ references returned by sql.references/field-references.
+
+ Note: this currently sets :lib/desired-column-alias but no other :lib/* fields, because the callers of this function
+ don't need the other fields. If we care about other :lib/* fields in the future, we can add them then."
+ {:added "0.57.0" :arglists '([driver metadata-provider col-spec])}
+ (fn [_driver _metadata-provider col-spec]
+ (:type col-spec)))
+
+(defmethod resolve-field :all-columns
+ [driver metadata-provider col-spec]
+ (or (some->> (:table col-spec)
+ (find-table-or-transform driver (driver-api/tables metadata-provider) (driver-api/transforms metadata-provider))
+ :table
+ (driver-api/active-fields metadata-provider)
+ (map #(assoc % :lib/desired-column-alias (:name %))))
+ [(assoc col-spec ::bad-reference true)]))
+
+(defmethod resolve-field :single-column
+ [driver metadata-provider {:keys [alias] :as col-spec}]
+ [(if-let [{:keys [name] :as found}
+ (->> (:source-columns col-spec)
+ (some (fn [source-col-set]
+ ;; in cases like `select (select blah from ...) from ...`, if blah refers to a
+ ;; column in both the inner query and the outer query, the column from the inner
+ ;; query will be preferred. However, if blah doesn't refer to something in the
+ ;; inner query, it can also refer to something in the outer query.
+ ;; sql.references/field-references organizes source-cols into a list of lists
+ ;; to account for this.
+ (->> (mapcat (partial resolve-field driver metadata-provider) source-col-set)
+ (some #(when (= (:name %) (:column col-spec))
+ %))))))]
+ (assoc found :lib/desired-column-alias (or alias name))
+ (assoc col-spec ::bad-reference true))])
+
+(defn- get-name [m]
+ (or (:alias m) (str (gensym "new-col"))))
+
+(defn- get-display-name [m]
+ (->> (get-name m)
+ (u.humanization/name->human-readable-name :simple)))
+
+(defmethod resolve-field :custom-field
+ [_driver _metadata-provider col-spec]
+ [{:base-type :type/*
+ :name (get-name col-spec)
+ :lib/desired-column-alias (get-name col-spec)
+ :display-name (get-display-name col-spec)
+ :effective-type :type/*
+ :semantic-type :Semantic/*}])
+
+(defmethod resolve-field :invalid-table-wildcard
+ [_driver _metadata-provider col-spec]
+ [(assoc col-spec ::bad-reference true)])
+
+(defn- lca [default-type & types]
+ (let [ancestor-sets (for [t types
+ :when t]
+ (conj (ancestors t) t))
+ common-ancestors (when (seq ancestor-sets)
+ (apply set/intersection ancestor-sets))]
+ (if (seq common-ancestors)
+ (apply (partial max-key (comp count ancestors)) common-ancestors)
+ default-type)))
+
+(defmethod resolve-field :composite-field
+ [driver metadata-provider col-spec]
+ (let [member-fields (mapcat (partial resolve-field driver metadata-provider)
+ (:member-fields col-spec))]
+ [{:name (get-name col-spec)
+ :lib/desired-column-alias (get-name col-spec)
+ :display-name (get-display-name col-spec)
+ :base-type (apply lca :type/* (map :base-type member-fields))
+ :effective-type (apply lca :type/* (map :effective-type member-fields))
+ :semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}]))
+
+(defmethod driver/native-result-metadata :sql
+ [driver metadata-provider native-query]
+ (let [{:keys [returned-fields]} (->> (macaw/parsed-query native-query)
+ macaw/->ast
+ (sql.references/field-references driver))]
+ (->> (mapcat (partial resolve-field driver metadata-provider) returned-fields)
+ (remove ::bad-reference))))
+
+(defmethod driver/validate-native-query-fields :sql
+ [driver metadata-provider native-query]
+ (let [{:keys [used-fields returned-fields bad-sql]} (->> (macaw/parsed-query native-query)
+ macaw/->ast
+ (sql.references/field-references driver))
+ check-fields #(mapcat (fn [col-spec]
+ (->> (resolve-field driver metadata-provider col-spec)
+ (filter ::bad-reference)))
+ %)]
+ (-> (concat (when bad-sql
+ [{:error :bad-sql}])
+ (check-fields used-fields)
+ (check-fields returned-fields))
+ distinct)))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Convenience Imports |
diff --git a/src/metabase/driver/sql/normalize.clj b/src/metabase/driver/sql/normalize.clj
new file mode 100644
index 00000000000..591b989c2aa
--- /dev/null
+++ b/src/metabase/driver/sql/normalize.clj
@@ -0,0 +1,52 @@
+(ns metabase.driver.sql.normalize
+ (:require
+ [clojure.string :as str]
+ [metabase.driver :as driver]
+ [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.util :as u]))
+
+(defmulti normalize-unquoted-name
+ "Normalize an unquoted table/column name according to the database's rules."
+ {:added "0.57.0", :arglists '([driver name-str])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod normalize-unquoted-name :sql
+ [_ name-str]
+ (u/lower-case-en name-str))
+
+(defn normalize-name
+ "Normalizes the (primarily table/column) name passed in.
+ Should return a value that matches the name listed in the appdb."
+ [driver name-str]
+ (let [quote-style (sql.qp/quote-style driver)
+ quote-char (if (= quote-style :mysql) \` \")]
+ (if (and (= (first name-str) quote-char)
+ (= (last name-str) quote-char))
+ (let [quote-quote (str quote-char quote-char)
+ quote (str quote-char)]
+ (-> name-str
+ (subs 1 (dec (count name-str)))
+ (str/replace quote-quote quote)))
+ (normalize-unquoted-name driver name-str))))
+
+(defmulti reserved-literal
+ "Checks whether a particular name is actually a literal value in a given sql dialect.
+
+ For example, true and false are usually booleans, not normal names."
+ {:added "0.57.0", :arglists '([driver name])}
+ (fn [driver name]
+ [(driver/dispatch-on-initialized-driver driver) name])
+ :hierarchy #'driver/hierarchy)
+
+(defmethod reserved-literal :default
+ [_driver _name]
+ nil)
+
+(defmethod reserved-literal [:sql "true"]
+ [_driver _name]
+ {:value true})
+
+(defmethod reserved-literal [:sql "false"]
+ [_driver _name]
+ {:value false})
diff --git a/src/metabase/driver/sql/references.clj b/src/metabase/driver/sql/references.clj
new file mode 100644
index 00000000000..4c79ddc1820
--- /dev/null
+++ b/src/metabase/driver/sql/references.clj
@@ -0,0 +1,316 @@
+(ns metabase.driver.sql.references
+ (:require
+ [macaw.ast-types :as macaw.ast-types]
+ [metabase.driver :as driver]
+ [metabase.driver.sql.normalize :as sql.normalize]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.performance :refer [every? mapv select-keys some]]))
+
+(mr/def ::single-column
+ [:map
+ [:type [:= :single-column]]
+ [:column :string]
+ [:source-columns [:sequential [:sequential [:ref ::col-spec]]]]
+ [:alias [:maybe :string]]])
+
+(mr/def ::all-columns
+ [:map
+ [:type [:= :all-columns]]
+ [:table [:map
+ [:table :string]
+ [:schema {:optional true} :string]
+ [:database {:optional true} :string]
+ [:table-alias {:optional true} :string]]]])
+
+(mr/def ::custom-field
+ [:map
+ [:type [:= :custom-field]]
+ [:alias [:maybe :string]]
+ [:used-fields [:set [:ref ::col-spec]]]])
+
+(mr/def ::composite-field
+ [:map
+ [:type [:= :composite-field]]
+ [:alias [:maybe :string]]
+ [:member-fields [:sequential ::col-spec]]])
+
+(mr/def ::invalid-table-wildcard
+ [:map
+ [:type [:= :invalid-table-wildcard]]
+ [:table :string]
+ [:schema {:optional true} :string]
+ [:database {:optional true} :string]
+ [:table-alias {:optional true} :string]])
+
+(mr/def ::col-spec
+ [:multi {:dispatch :type}
+ [:single-column [:ref ::single-column]]
+ [:all-columns [:ref ::all-columns]]
+ [:custom-field [:ref ::custom-field]]
+ [:composite-field [:ref ::composite-field]]
+ [:invalid-table-wildcard [:ref ::invalid-table-wildcard]]])
+
+(mr/def ::field-references
+ [:map
+ [:used-fields [:set [:ref ::col-spec]]]
+ [:returned-fields [:sequential [:ref ::col-spec]]]
+ [:bad-sql {:optional true} :boolean]])
+
+(defn- normalize-fields [driver m]
+ (update-vals m
+ #(if (string? %)
+ (sql.normalize/normalize-name driver %)
+ %)))
+
+(defn- col-fields [driver m]
+ (->> (select-keys m [:type :column :table :schema :database :alias])
+ (normalize-fields driver)))
+
+(defn- table-matches? [search table]
+ (or (when (and (not (:schema search))
+ (not (:database search)))
+ (= (:table search) (:table-alias table)))
+ (->> (select-keys search [:table :schema :database])
+ keys
+ (every? #(= (% search) (% table))))))
+
+(defn- find-source [search sources]
+ (some (fn [sublist]
+ (some #(when (table-matches? search (:names %))
+ %)
+ sublist))
+ sources))
+
+(defn- get-column [driver sources raw-col]
+ (if (and (nil? (:table raw-col))
+ (nil? (:schema raw-col))
+ (nil? (:database raw-col))
+ (sql.normalize/reserved-literal driver (:column raw-col)))
+ []
+ (let [{:keys [alias column table] :as expr} (col-fields driver raw-col)
+ valid-sources (if table
+ [[(find-source expr sources)]]
+ sources)
+ source-columns (into []
+ (map #(into []
+ (comp (mapcat :returned-fields)
+ (filter (fn [{field-alias :alias
+ field-type :type
+ field-column :column}]
+ (or (= field-type :all-columns)
+ (= (or field-alias field-column) column)))))
+ %))
+ valid-sources)
+ source-column (some #(when (= column (or (:alias %) (:column %)))
+ %)
+ (first source-columns))]
+ [(if source-column
+ (cond-> source-column
+ alias (assoc :alias alias))
+ (-> {:column column
+ :alias alias
+ :type :single-column}
+ (assoc :source-columns source-columns)))])))
+
+(defmulti find-used-fields
+ "Finds the fields used in a given sql expression."
+ {:added "0.57.0", :arglists '([driver sources withs expr])}
+ (fn [driver _sources _withs expr]
+ [(driver/dispatch-on-initialized-driver driver) (:type expr)])
+ :hierarchy #'driver/hierarchy)
+
+(defmethod find-used-fields :default
+ [_driver _sources _withs _expr]
+ nil)
+
+(defmethod find-used-fields [:sql :macaw.ast/column]
+ [driver sources _withs expr]
+ (get-column driver sources expr))
+
+(defmethod find-used-fields [:sql :macaw.ast/unary-expression]
+ [driver sources withs expr]
+ (find-used-fields driver sources withs (:expression expr)))
+
+(defmethod find-used-fields [:sql :macaw.ast/binary-expression]
+ [driver sources withs expr]
+ (concat (find-used-fields driver sources withs (:left expr))
+ (find-used-fields driver sources withs (:right expr))))
+
+(defmethod find-used-fields [:sql :macaw.ast/expression-list]
+ [driver sources withs expr]
+ (mapcat (partial find-used-fields driver sources withs) (:expressions expr)))
+
+(defmethod find-used-fields [:sql :macaw.ast/join]
+ [driver sources withs expr]
+ (mapcat (partial find-used-fields driver sources withs) (:condition expr)))
+
+(defmethod find-used-fields [:sql :macaw.ast/function]
+ [driver sources withs expr]
+ (mapcat (partial find-used-fields driver sources withs) (:params expr)))
+
+(defmethod find-used-fields [:sql :macaw.ast/case]
+ [driver sources withs expr]
+ (concat (find-used-fields driver sources withs (:switch expr))
+ (find-used-fields driver sources withs (:else expr))
+ (mapcat (fn [{when-clause :when then-clause :then}]
+ (concat (find-used-fields driver sources withs when-clause)
+ (find-used-fields driver sources withs then-clause)))
+ (:when-clauses expr))))
+
+(defmethod find-used-fields [:sql :macaw.ast/between]
+ [driver sources withs expr]
+ (concat (find-used-fields driver sources withs (:expression expr))
+ (find-used-fields driver sources withs (:start expr))
+ (find-used-fields driver sources withs (:end expr))))
+
+(defmethod find-used-fields [:sql :macaw.ast/set-operation]
+ [driver sources withs expr]
+ (mapcat (partial find-used-fields driver sources withs)
+ (:selects expr)))
+
+(defmethod find-used-fields [:sql :macaw.ast/analytic-expression]
+ [driver sources withs expr]
+ (let [rec (partial find-used-fields driver sources withs)]
+ (concat (rec (:expression expr))
+ (rec (:offset expr))
+ (rec (:window expr))
+ (mapcat rec (:partition-by expr))
+ (mapcat rec (:order-by expr)))))
+
+(defmulti find-returned-fields
+ "Finds the fields returned by a given sql query/expression."
+ {:added "0.57.0", :arglists '([driver sources withs expr])}
+ (fn [driver _sources _withs expr]
+ [(driver/dispatch-on-initialized-driver driver) (:type expr)])
+ :hierarchy #'driver/hierarchy)
+
+(defmethod find-returned-fields :default
+ [driver sources withs {:keys [alias] :as expr}]
+ [{:alias (when alias
+ (sql.normalize/normalize-name driver alias))
+ :type :custom-field
+ :used-fields (into #{} (find-used-fields driver sources withs expr))}])
+
+(defmethod find-returned-fields [:sql :macaw.ast/column]
+ [driver sources _withs expr]
+ (get-column driver sources expr))
+
+(defmethod find-returned-fields [:sql :macaw.ast/wildcard]
+ [_driver sources _withs _expr]
+ (into []
+ (mapcat :returned-fields)
+ (first sources)))
+
+(defmethod find-returned-fields [:sql :macaw.ast/table-wildcard]
+ [driver sources _withs expr]
+ (or (some-> (find-source (col-fields driver expr) sources)
+ :returned-fields)
+ [(assoc expr
+ :type :invalid-table-wildcard)]))
+
+(defmethod find-returned-fields [:sql :macaw.ast/set-operation]
+ [driver sources withs expr]
+ (->> (map (partial find-returned-fields driver sources withs)
+ (:selects expr))
+ (apply map (fn [& fields]
+ {:alias (or (:alias (first fields))
+ (:column (first fields)))
+ :type :composite-field
+ :member-fields (into [] fields)}))))
+
+(defmulti field-references-impl
+ "Implementation of field-references."
+ {:added "0.57.0", :arglists '([driver sources withs expr])}
+ (fn [driver _outside-sources _withs expr]
+ [(driver/dispatch-on-initialized-driver driver) (:type expr)])
+ :hierarchy #'driver/hierarchy)
+
+(defmethod field-references-impl [:sql :macaw.ast/table]
+ [driver _outside-sources withs expr]
+ (or (when-let [source (find-source expr [withs])]
+ (update source :names assoc :table-alias (:table-alias expr)))
+ {:used-fields #{}
+ :returned-fields [{:type :all-columns
+ :table (->> (select-keys expr [:table :schema :database :table-alias])
+ (normalize-fields driver))}]
+ :names (normalize-fields driver expr)}))
+
+(defn- get-select-sources
+ [driver outside-sources withs expr]
+ (cond-> (mapv (partial field-references-impl driver outside-sources withs) (:join expr))
+ (:from expr) (conj (field-references-impl driver outside-sources withs (:from expr)))))
+
+(defmethod find-used-fields [:sql :macaw.ast/select]
+ [driver outside-sources withs expr]
+ (let [local-sources (get-select-sources driver outside-sources withs expr)
+ sources (cons local-sources outside-sources)
+ rec (partial find-used-fields driver sources withs)]
+ (-> (into #{}
+ (mapcat rec)
+ (:select expr))
+ (into (rec (:where expr)))
+ (into (mapcat rec)
+ (:join expr))
+ (into (mapcat rec)
+ (:group-by expr))
+ (into (mapcat rec)
+ (:order-by expr))
+ (into (mapcat :used-fields)
+ local-sources))))
+
+(defmethod find-returned-fields [:sql :macaw.ast/select]
+ [driver outside-sources withs expr]
+ (let [sources (cons (get-select-sources driver outside-sources withs expr)
+ outside-sources)
+ returned-fields (mapcat (partial find-returned-fields driver sources withs) (:select expr))]
+ (if (:alias expr)
+ ;; :alias is a column alias, so this is presumably something like `select (select * from ...)`
+ ;; there should only be one field returned, and that field should have the appropriate alias
+ (do (assert (= 1 (count returned-fields)))
+ (map #(assoc % :alias (sql.normalize/normalize-name driver (:alias expr)))
+ returned-fields))
+ returned-fields)))
+
+(defmethod field-references-impl [:sql :macaw.ast/select]
+ [driver outside-sources outside-withs expr]
+ (let [local-withs (map (partial field-references-impl driver outside-sources outside-withs)
+ (:with expr))
+ withs (into outside-withs local-withs)]
+ {:used-fields (into (find-used-fields driver outside-sources withs expr)
+ (mapcat :used-fields)
+ local-withs)
+ :returned-fields (into []
+ (find-returned-fields driver outside-sources withs expr))
+ :names (when-let [alias (:table-alias expr)]
+ {:table-alias (sql.normalize/normalize-name driver alias)})}))
+
+(defmethod field-references-impl [:sql :macaw.ast/join]
+ [driver outside-sources withs expr]
+ (field-references-impl driver outside-sources withs (:source expr)))
+
+(defmethod field-references-impl [:sql :macaw.ast/set-operation]
+ [driver outside-sources withs expr]
+ {:used-fields (->> (find-used-fields driver outside-sources withs expr)
+ (into #{}))
+ :returned-fields (->> (find-returned-fields driver outside-sources withs expr)
+ (into []))
+ :names (when-let [alias (:table-alias expr)]
+ {:table-alias (sql.normalize/normalize-name driver alias)})})
+
+(defmethod field-references-impl :default
+ [_driver _outside-sources _outside-withs _expr]
+ {:used-fields #{}
+ :returned-fields []
+ :names nil
+ :bad-sql true})
+
+(mu/defn field-references :- [:ref ::field-references]
+ "Takes a sql query in the macaw ast format and returns the fields referenced by a query.
+
+ Specifically, this returns a set of the fields used and a list of the fields returned, along with a boolean that
+ denotes when this is given an invalid query of some sort."
+ [driver :- :keyword
+ expr :- macaw.ast-types/ast]
+ (-> (field-references-impl driver nil #{} expr)
+ (dissoc :names)))
diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.clj
index e44bc5593a6..2ec9e1d06b4 100644
--- a/src/metabase/driver_api/core.clj
+++ b/src/metabase/driver_api/core.clj
@@ -104,8 +104,10 @@
lib.metadata/database
lib.metadata/field
lib.metadata/fields
+ lib.metadata/active-fields
lib.metadata/table
lib.metadata/tables
+ lib.metadata/transforms
lib.schema.common/instance-of-class
lib.schema.temporal-bucketing/date-bucketing-units
lib.types.isa/temporal?
diff --git a/src/metabase/events/schema.clj b/src/metabase/events/schema.clj
index 5ac3685496b..3fb897006ed 100644
--- a/src/metabase/events/schema.clj
+++ b/src/metabase/events/schema.clj
@@ -161,3 +161,13 @@
[:map {:closed true}
[:user-id [:maybe pos-int?]]
[:model [:or :keyword :string]]])
+
+;; snippet events
+
+(mr/def ::snippet
+ [:map {:closed true}
+ [:object [:fn #(t2/instance-of? :model/NativeQuerySnippet %)]]])
+
+(mr/def :event/snippet-create ::snippet)
+(mr/def :event/snippet-update ::snippet)
+(mr/def :event/snippet-delete ::snippet)
diff --git a/src/metabase/graph/core.cljc b/src/metabase/graph/core.cljc
new file mode 100644
index 00000000000..e389054aee1
--- /dev/null
+++ b/src/metabase/graph/core.cljc
@@ -0,0 +1,91 @@
+(ns metabase.graph.core
+ "Interface to a library of generic graph walking functionality.
+
+ Used for eg. walking the `enterprise/dependencies` graph, or similar directed graphs.
+
+ Implementation-agnostic! The graph representation is wrapped by a protocol exported from this module, which can be
+ used to write graph searches which can run in memory, against AppDB, on the filesystem, etc.
+
+ Each graph defines an arbitrary **key**. This can be any hash map key, such as a number, a `[type id]` pair, etc."
+ (:require
+ #?@(:cljs ([flatland.ordered.set :as oset]))
+ [metabase.util.malli.registry :as mr]
+ [potemkin :as p])
+ #?@(:clj ((:import java.util.LinkedHashSet))))
+
+#?(:clj (set! *warn-on-reflection* true))
+
+(p/defprotocol+ Graph
+ (children-of [graph key-seq]
+ "Given a graph and a seq of keys, returns a map from each input key to the set of keys **directly reachable** from
+ that key.
+
+ If you want a *transitive* search, call [[transitive]].
+
+ Contract details:
+ - If a key is not known in the graph, it should be missing from the output map.
+ - If a key is found in the graph, and it has no children, its value is `#{}`.
+ - If the input `key-seq` is empty, return `{}`."))
+
+(defn- stable-iteration-set []
+ ;; Both of these sets explicitly have stable iteration order: values are returned in the order of (first) insertion.
+ #?(:clj (let [s (LinkedHashSet.)]
+ {:insert-many! #(.addAll s %)
+ :seen? #(.contains s %)
+ :->iterable (constantly s)})
+ :cljs (let [s (volatile! (oset/ordered-set))]
+ {:insert-many! #(vswap! s into %)
+ :seen? #(contains? @s %)
+ :->iterable #(deref s)})))
+
+(defn transitive
+ "Given a graph and `key-seq`, returns a seq of all transitive children of those starting keys.
+
+ The returned seq:
+ - Excludes the input keys, even if they're reachable from the input keys.
+ - Contains each key only once, even if it can be reached by multiple paths.
+ - Is **topologically sorted**: all keys reachable in `n` steps from the starters appear before any keys reachable
+ in `n+1` steps, etc.
+ - Makes no guarantee about the order of keys at the same \"level\".
+ - That is, if two keys `k1` and `k2` are reachable in no fewer than `n` steps from the starters, then `k1` might
+ precede or follow `k2` in the output seq."
+ [graph key-seq]
+ (let [{:keys [insert-many! seen? ->iterable]} (stable-iteration-set)]
+ (insert-many! key-seq)
+ (loop [new-keys key-seq]
+ (if (seq new-keys)
+ (let [k->children (children-of graph new-keys)
+ new-children (into #{} (comp cat (remove seen?)) (vals k->children))]
+ (insert-many! new-children)
+ (recur new-children))
+ (drop (count key-seq) (->iterable))))))
+
+(defn graph?
+ "Whether `x` is a valid `Graph`."
+ [x]
+ #?(:clj (extends? Graph (class x))
+ :cljs (satisfies? Graph x)))
+
+(mr/def ::graph
+ "Schema for anything that satisfies the [[Graph]] protocol."
+ [:fn
+ {:error/message "Valid Graph instance"}
+ #'graph?])
+
+;; ## In-memory Graphs
+(p/deftype+ InMemoryGraph [adjacency-map]
+ Graph
+ (children-of [_this key-seq]
+ (select-keys adjacency-map key-seq)))
+
+(defn in-memory
+ "Given a map of which keys are reachable from each key, `{key #{key...}, ...}`, returns an instance of [[Graph]]
+ backed by this in-memory graph.
+
+ Note that this map is in the same form as [[children-of]] returns."
+ [adjacency-map]
+ (->InMemoryGraph adjacency-map))
+
+;; TODO: (Braden, 09/19/2025) We may find use for some extra helpers. For example, a wrapper that implements the
+;; multi-key [[children-of]] on top of a single-key `(children graph key) => #{keys...}` function.
+;; But YAGNI, so since I don't have a use-case in mind right now, I'm just leaving a note.
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index 6597fcf4da9..0f290f2f049 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -55,6 +55,7 @@
[metabase.lib.template-tags :as lib.template-tags]
[metabase.lib.temporal-bucket :as lib.temporal-bucket]
[metabase.lib.util :as lib.util]
+ [metabase.lib.validate :as lib.validate]
[metabase.lib.walk.util]
[metabase.util.namespaces :as shared.ns]))
@@ -360,6 +361,7 @@
metadata-provider?
metadata-providerable?]
[lib.native
+ add-parameters-for-template-tags
engine
extract-template-tags
has-template-tag-variables?
@@ -458,6 +460,8 @@
source-table-id
source-card-id
update-query-stage]
+ [lib.validate
+ find-bad-refs]
[metabase.lib.walk.util
all-field-ids
all-source-card-ids
diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljc
index 374ed0d50b6..7fdb4631c7c 100644
--- a/src/metabase/lib/metadata.cljc
+++ b/src/metabase/lib/metadata.cljc
@@ -120,6 +120,18 @@
(when-let [remap-field-id (get-in column [:lib/external-remap :field-id])]
(field metadata-providerable remap-field-id))))
+;; TODO: Better schemas for transforms coming out of the metadata provider.
+(mu/defn transform :- [:maybe [:map]]
+ "Gets a Transform by ID, or nil if it does not exist."
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ transform-id :- :int]
+ (lib.metadata.protocols/transform (->metadata-provider metadata-providerable) transform-id))
+
+(mu/defn transforms :- [:maybe [:sequential [:map]]]
+ "Gets all Transforms"
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable]
+ (lib.metadata.protocols/transforms (->metadata-provider metadata-providerable)))
+
(mu/defn setting :- any?
"Get the value of a Metabase setting for the instance we're querying."
([metadata-providerable :- ::lib.schema.metadata/metadata-providerable
diff --git a/src/metabase/lib/metadata/cached_provider.cljc b/src/metabase/lib/metadata/cached_provider.cljc
index 55d8e2a9c73..e0fcb481b2c 100644
--- a/src/metabase/lib/metadata/cached_provider.cljc
+++ b/src/metabase/lib/metadata/cached_provider.cljc
@@ -30,16 +30,17 @@
id :- pos-int?
metadata :- [:multi
{:dispatch :lib/type}
- [:metadata/database ::lib.schema.metadata/database]
- [:metadata/table ::lib.schema.metadata/table]
- [:metadata/column ::lib.schema.metadata/column]
- [:metadata/card ::lib.schema.metadata/card]
- [:metadata/metric ::lib.schema.metadata/metric]
- [:metadata/segment ::lib.schema.metadata/segment]]]
+ [:metadata/database ::lib.schema.metadata/database]
+ [:metadata/table ::lib.schema.metadata/table]
+ [:metadata/column ::lib.schema.metadata/column]
+ [:metadata/card ::lib.schema.metadata/card]
+ [:metadata/metric ::lib.schema.metadata/metric]
+ [:metadata/segment ::lib.schema.metadata/segment]
+ [:metadata/native-query-snippet ::lib.schema.metadata/native-query-snippet]]]
(let [metadata (-> metadata
(update-keys u/->kebab-case-en)
(assoc :lib/type metadata-type))]
- (store-in-cache! cache [metadata-type id] metadata))
+ (store-in-cache! cache [metadata-type :id id] metadata))
true)
(defn- get-in-cache-or-fetch [cache ks fetch-thunk]
diff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljc
index fc15b25f32f..c9eb3c34248 100644
--- a/src/metabase/lib/metadata/protocols.cljc
+++ b/src/metabase/lib/metadata/protocols.cljc
@@ -10,7 +10,7 @@
(mr/def ::metadata-type-excluding-database
"Database metadata is stored separately/in a special way. These are the types of metadata that are stored with the
other non-Database methods."
- [:enum :metadata/table :metadata/column :metadata/card :metadata/metric :metadata/segment :metadata/native-query-snippet])
+ [:enum :metadata/table :metadata/column :metadata/card :metadata/metric :metadata/segment :metadata/native-query-snippet :metadata/transform])
(mr/def ::metadata-spec
"Spec for fetching objects from a metadata provider. `:lib/type` is the type of the object to fetch, and the other
@@ -42,8 +42,8 @@
(or (not (:card-id spec))
(#{:metadata/metric} (:lib/type spec))))]
[:fn
- {:error/message "All metadata types except for :metadata/table must include at least one filter"}
- (some-fn :id :name :table-id :card-id #(= (:lib/type %) :metadata/table))]])
+ {:error/message "All metadata types except for :metadata/table and :metadata/transform must include at least one filter"}
+ (some-fn :id :name :table-id :card-id #(= (:lib/type %) :metadata/table) #(= (:lib/type %) :metadata/transform))]])
(mu/defn default-spec-filter-xform
"Create a `filter` transducer to a sequence of objects according to `metadata-spec`. Assumes objects are all the
@@ -239,6 +239,18 @@
card-id :- ::lib.schema.id/card]
(metadata-by-id metadata-provider :metadata/card card-id))
+;; TODO: Better schemas for transforms.
+(mu/defn transform :- [:maybe [:map]]
+ "Return information about a specific Transform. Nil if it does not exist."
+ [metadata-provider :- ::metadata-provider
+ card-id :- ::lib.schema.id/card]
+ (metadata-by-id metadata-provider :metadata/transform card-id))
+
+(mu/defn transforms :- [:maybe [:sequential [:map]]]
+ "Return information about all Transforms."
+ [metadata-provider :- ::metadata-provider]
+ (metadatas metadata-provider {:lib/type :metadata/transform}))
+
(mu/defn native-query-snippet :- [:maybe ::lib.schema.metadata/native-query-snippet]
"Get metadata for a NativeQuerySnippet with `snippet-id` if it can be found."
[metadata-provider :- ::metadata-provider
diff --git a/src/metabase/lib/native.cljc b/src/metabase/lib/native.cljc
index 55f88aa6ceb..017579dd241 100644
--- a/src/metabase/lib/native.cljc
+++ b/src/metabase/lib/native.cljc
@@ -7,6 +7,7 @@
[clojure.string :as str]
[medley.core :as m]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
[metabase.lib.parse :as lib.parse]
[metabase.lib.query :as lib.query]
[metabase.lib.schema :as lib.schema]
@@ -364,3 +365,54 @@
[query :- ::lib.schema/query]
(assert-native-query (lib.util/query-stage query 0))
(:engine (lib.metadata/database query)))
+
+(defn- get-parameter-value
+ [tag-name {:keys [id dimension], param-type :type}]
+ ;; note that the actual values chosen are completely arbitrary. We just need to provide some
+ ;; value so that the query will compile.
+ (case param-type
+ :text {:id id,
+ :type :string/=,
+ :value ["foo"],
+ :target ["variable" ["template-tag" tag-name]]}
+ :number {:id id,
+ :type :number/=,
+ :value ["0"],
+ :target ["variable" ["template-tag" tag-name]]}
+ :date {:id id,
+ :type :date/single,
+ :value "1970-01-01",
+ :target ["variable" ["template-tag" tag-name]]}
+ :boolean {:id id,
+ :type :boolean/=,
+ :value [false],
+ :target ["variable" ["template-tag" tag-name]]}
+ :dimension (merge {:id id,
+ :type :string/=,
+ :value ["foo"],
+ :target ["dimension" ["template-tag" tag-name]]}
+ (when (isa? (-> dimension lib.options/options :effective-type) :type/Number)
+ {:type :number/=,
+ :value ["0"]}))
+ :temporal-unit {:id id,
+ :type :temporal-unit,
+ :value "week",
+ :target ["dimension" ["template-tag" tag-name]]}
+ nil))
+
+(defn add-parameters-for-template-tags
+ "Adds dummy values for parameters that don't have one.
+ This is so that the resulting native query can be parsed. It's not expected to be executable."
+ [query]
+ (let [ttags (-> (lib.util/query-stage query 0)
+ :template-tags)
+ parameters (:parameters query)
+ params-by-id (m/index-by :id parameters)
+ new-parameters (into []
+ (keep (fn [[tag-name {:keys [id] :as tag}]]
+ (or (params-by-id id)
+ (get-parameter-value tag-name tag))))
+
+ ttags)]
+ (cond-> query
+ (seq new-parameters) (assoc :parameters new-parameters))))
diff --git a/src/metabase/lib/schema/id.cljc b/src/metabase/lib/schema/id.cljc
index 68d076e4a47..02899dfaa98 100644
--- a/src/metabase/lib/schema/id.cljc
+++ b/src/metabase/lib/schema/id.cljc
@@ -79,6 +79,9 @@
"Valid Native Query Snippet ID"
pos-int?)
+(mr/def ::transform
+ [:schema {:doc/title "Valid Transform ID"} pos-int?])
+
(mr/def ::collection
"Valid Collection ID"
pos-int?)
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index a02683e3c70..46fe497a94e 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -566,7 +566,7 @@
separate table in the app DB, but now that they're a subtype of Card it's probably not important anymore, we can
probably just use `:metadata/card` here."
[:enum :metadata/database :metadata/table :metadata/column :metadata/card :metadata/metric
- :metadata/segment])
+ :metadata/segment :metadata/native-query-snippet])
(mr/def ::lib-or-legacy-column
"Schema for the maps in card `:result-metadata` and similar. These can be either
diff --git a/src/metabase/lib/util.cljc b/src/metabase/lib/util.cljc
index 447445e66fc..8b328e88957 100644
--- a/src/metabase/lib/util.cljc
+++ b/src/metabase/lib/util.cljc
@@ -745,3 +745,31 @@
(concat (collect-source-tables source-query) from-joins)
(cond->> from-joins
(:source-table legacy-query) (cons (:source-table legacy-query))))))
+
+(defn- split-tables-and-legacy-card-refs [source-ids]
+ (-> (reduce (fn [m src]
+ (if-let [card-id (legacy-string-table-id->card-id src)]
+ (update m :card conj! card-id)
+ (update m :table conj! src)))
+ {:card (transient #{})
+ :table (transient #{})}
+ source-ids)
+ (update-vals persistent!)))
+
+(mu/defn source-tables-and-cards :- [:map
+ [:card {:optional true} [:set ::lib.schema.id/card]]
+ [:table {:optional true} [:set ::lib.schema.id/table]]]
+ "Returns all the table and card IDs that this list of **legacy, MBQL 4 queries** depend on."
+ [legacy-queries]
+ (let [metric-ids (-> (lib.util.match/match legacy-queries
+ [:metric (id :guard pos-int?) & _]
+ id)
+ set)
+ source-ids (-> #{}
+ (into (map #(str "card__" %)) metric-ids)
+ (into
+ ;; existing usage -- do not use in new code
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (mapcat #(collect-source-tables (:query %)))
+ legacy-queries))]
+ (split-tables-and-legacy-card-refs source-ids)))
diff --git a/src/metabase/lib/validate.cljc b/src/metabase/lib/validate.cljc
new file mode 100644
index 00000000000..f057b4b0ef5
--- /dev/null
+++ b/src/metabase/lib/validate.cljc
@@ -0,0 +1,41 @@
+(ns metabase.lib.validate
+ "Checks and validation for queries."
+ (:require
+ [metabase.lib.field.resolution :as lib.field.resolution]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+ [metabase.lib.walk :as lib.walk]
+ [metabase.util.malli :as mu]))
+
+(mu/defn find-bad-refs :- [:maybe [:sequential ::lib.schema.mbql-clause/clause]]
+ "Returns a list of bad `:field` refs on this query.
+
+ Returns nil if all refs on the query are sound, that is if they can be resolved to a column from some source."
+ [query :- ::lib.schema/query]
+ (let [bad-fields (volatile! [])]
+ (lib.walk/walk-clauses
+ query (fn [query path-type path clause]
+ (when (and (= path-type :lib.walk/stage)
+ (vector? clause)
+ (= (first clause) :field))
+ (let [column (lib.walk/apply-f-for-stage-at-path
+ lib.field.resolution/resolve-field-ref
+ query path clause)]
+ (when (or (not column)
+ (::lib.field.resolution/fallback-metadata? column)
+ (not (:active column true)))
+ (vswap! bad-fields conj clause))))
+ nil))
+ (not-empty @bad-fields)))
+
+(comment
+ (require '[metabase.lib.core :as lib])
+ (require '[metabase.lib.test-metadata :as meta])
+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/breakout (meta/field-metadata :orders :quantity))
+ (lib/aggregate (lib/count))
+ lib/append-stage)
+ [_category cnt] (lib/visible-columns base)
+ query (lib/filter base (lib/> cnt 100))]
+ {:valid-query (find-bad-refs query)
+ :bad-query (find-bad-refs (assoc-in query [:stages 1 :filters 0 2 2] "n"))}))
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index ade2af7fce6..a5ba12c99d3 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -348,6 +348,28 @@
[snippet]
(instance->metadata snippet :metadata/native-query-snippet))
+;;;
+;;; Transforms
+;;;
+
+(derive :metadata/transform :model/Transform)
+
+(methodical/defmethod t2.model/resolve-model :metadata/transform
+ [model]
+ (t2/resolve-model :model/Transform) ; for side-effects
+ model)
+
+(methodical/defmethod t2.pipeline/build [#_query-type :toucan.query-type/select.*
+ #_model :metadata/transform
+ #_resolved-query clojure.lang.IPersistentMap]
+ [query-type model parsed-args honeysql]
+ (merge (next-method query-type model parsed-args honeysql)
+ {:select [:id :name :source :target]}))
+
+(t2/define-after-select :metadata/transform
+ [snippet]
+ (instance->metadata snippet :metadata/transform))
+
;;;
;;; MetadataProvider
;;;
@@ -367,7 +389,8 @@
:metadata/card :card/database_id
:metadata/metric :database_id
:metadata/segment :table/db_id
- :metadata/native-query-snippet nil))
+ :metadata/native-query-snippet nil
+ :metadata/transform nil))
(defn- id-key [metadata-type]
(case metadata-type
@@ -376,7 +399,8 @@
:metadata/card :card/id
:metadata/metric :id
:metadata/segment :segment/id
- :metadata/native-query-snippet :id))
+ :metadata/native-query-snippet :id
+ :metadata/transform :id))
(defn- name-key [metadata-type]
(case metadata-type
@@ -385,7 +409,8 @@
:metadata/card :card/name
:metadata/metric :name
:metadata/segment :segment/name
- :metadata/native-query-snippet :name))
+ :metadata/native-query-snippet :name
+ :metadata/transform :name))
(defn- table-id-key [metadata-type]
;; types not in the case statement do not support Table ID
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index fde952e936e..15d96a69fc9 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -230,6 +230,13 @@
(and (map? query) (seq query))
normalize)))
+(defn maybe-normalize-transform-source
+ "Normalizes the `source` of a transform."
+ [in-or-out raw-source]
+ (let [{source-type :type :as source} (m/map-keys keyword raw-source)]
+ (cond-> source
+ (= (keyword source-type) :query) (update :query (partial maybe-normalize-query in-or-out)))))
+
(defn catch-normalization-exceptions
"Wraps normalization fn `f` and returns a version that gracefully handles Exceptions during normalization. When
invalid queries (etc.) come out of the Database, it's best we handle normalization failures gracefully rather than
@@ -264,6 +271,11 @@
{:in (comp json-in (partial maybe-normalize-query :in))
:out (comp (catch-normalization-exceptions (partial maybe-normalize-query :out)) json-out-without-keywordization)})
+(def transform-transform-source
+ "Transform for transform source fields."
+ {:in (comp json-in (partial maybe-normalize-transform-source :in))
+ :out (comp (catch-normalization-exceptions (partial maybe-normalize-transform-source :out)) json-out-without-keywordization)})
+
(def transform-parameters-list
"Transform for parameters list."
{:in (comp json-in normalize-parameters-list)
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index 47afe26daee..ef9a0f2e1a5 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -38,6 +38,7 @@
:model/DataPermissions metabase.permissions.models.data-permissions
:model/Database metabase.warehouses.models.database
:model/DatabaseRouter metabase-enterprise.database-routing.model
+ :model/Dependency metabase-enterprise.dependencies.models.dependency
:model/Dimension metabase.warehouse-schema.models.dimension
:model/Document metabase-enterprise.documents.models.document
:model/DocumentBookmark metabase-enterprise.bookmarks.models.bookmark
diff --git a/src/metabase/native_query_snippets/core.clj b/src/metabase/native_query_snippets/core.clj
index 4ecdf3cd545..0bc73e177e6 100644
--- a/src/metabase/native_query_snippets/core.clj
+++ b/src/metabase/native_query_snippets/core.clj
@@ -1,10 +1,16 @@
(ns metabase.native-query-snippets.core
(:require
+ [metabase.native-query-snippets.models.native-query-snippet]
[metabase.native-query-snippets.models.native-query-snippet.permissions]
[potemkin :as p]))
-(comment metabase.native-query-snippets.models.native-query-snippet.permissions/keep-me)
+(comment
+ metabase.native-query-snippets.models.native-query-snippet.permissions/keep-me
+ metabase.native-query-snippets.models.native-query-snippet/keep-me)
(p/import-vars
+ [metabase.native-query-snippets.models.native-query-snippet
+ add-template-tags
+ NativeQuerySnippetName]
[metabase.native-query-snippets.models.native-query-snippet.permissions
has-any-native-permissions?])
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index abb821a52fe..e4b8d6f60b7 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -2,6 +2,7 @@
(:require
[honey.sql.helpers :as sql.helpers]
[metabase.collections.models.collection :as collection]
+ [metabase.events.core :as events]
[metabase.lib.core :as lib]
[metabase.lib.normalize :as lib.normalize]
[metabase.models.interface :as mi]
@@ -11,7 +12,8 @@
[metabase.util.i18n :refer [deferred-tru tru]]
[metabase.util.malli :as mu]
[methodical.core :as methodical]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2]
+ [toucan2.realize :as t2.realize]))
;;; ----------------------------------------------- Entity & Lifecycle -----------------------------------------------
@@ -32,7 +34,13 @@
[_]
#{:snippets})
-(defn- add-template-tags [{old-tags :template_tags :as snippet}]
+(derive ::event :metabase/event)
+(doseq [e [:event/snippet-create :event/snippet-update :event/snippet-delete]]
+ (derive e ::event))
+
+(defn add-template-tags
+ "Update the template tags based on the new contents."
+ [{old-tags :template_tags :as snippet}]
;; Parse the snippet content to identify all template tags (like {{snippet: FilterA}} or {{var}}).
;; For snippet references, we need to resolve them to snippet IDs while preserving reference stability.
;;
@@ -67,6 +75,11 @@
(u/prog1 (add-template-tags snippet)
(collection/check-collection-namespace :model/NativeQuerySnippet (:collection_id snippet))))
+(t2/define-after-insert :model/NativeQuerySnippet
+ [snippet]
+ (u/prog1 (t2.realize/realize snippet)
+ (events/publish-event! :event/snippet-create {:object <>})))
+
(t2/define-before-update :model/NativeQuerySnippet
[snippet]
(u/prog1 (cond-> snippet
@@ -76,6 +89,16 @@
(throw (UnsupportedOperationException. (tru "You cannot update the creator_id of a NativeQuerySnippet."))))
(collection/check-collection-namespace :model/NativeQuerySnippet (:collection_id snippet))))
+(t2/define-after-update :model/NativeQuerySnippet
+ [snippet]
+ (u/prog1 (t2.realize/realize snippet)
+ (events/publish-event! :event/snippet-update {:object <>})))
+
+(t2/define-before-delete :model/NativeQuerySnippet
+ [snippet]
+ (u/prog1 snippet
+ (events/publish-event! :event/snippet-delete {:object <>})))
+
(defmethod serdes/hash-fields :model/NativeQuerySnippet
[_snippet]
[:name (serdes/hydrated-hash :collection) :created_at])
@@ -123,6 +146,7 @@
(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]
{:copy [:archived :content :description :entity_id :name :template_tags]
+ :skip [:dependency_analysis_version]
:transform {:created_at (serdes/date)
:collection_id (serdes/fk :model/Collection)
:creator_id (serdes/fk :model/User)}})
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index 3b69a16f5ca..4c1ce9af441 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -52,6 +52,7 @@
enable-dashboard-subscription-filters?
enable-database-auth-providers?
enable-database-routing?
+ enable-dependencies?
enable-documents?
enable-email-allow-list?
enable-email-restrict-recipients?
diff --git a/src/metabase/premium_features/settings.clj b/src/metabase/premium_features/settings.clj
index da840c557c2..a1388135c04 100644
--- a/src/metabase/premium_features/settings.clj
+++ b/src/metabase/premium_features/settings.clj
@@ -299,6 +299,10 @@
"Should we allow users to use Python transforms?"
:transforms-python)
+(define-premium-feature ^{:added "0.57.0"} enable-dependencies?
+ "Should we allow users to use dependency tracking?"
+ :dependencies)
+
(defn- -token-features []
{:advanced_permissions (enable-advanced-permissions?)
:ai_sql_fixer (enable-ai-sql-fixer?)
@@ -316,6 +320,7 @@
:dashboard_subscription_filters (enable-dashboard-subscription-filters?)
:database_auth_providers (enable-database-auth-providers?)
:database_routing (enable-database-routing?)
+ :dependencies (enable-dependencies?)
:development_mode (development-mode?)
:disable_password_login (can-disable-password-login?)
:documents (enable-documents?)
diff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.clj
index 998d3399e6f..c2caa525a87 100644
--- a/src/metabase/queries/metadata.clj
+++ b/src/metabase/queries/metadata.clj
@@ -49,16 +49,6 @@
(t2/select-fn-set :table_id :model/Field :id [:in field-ids])
#{}))
-(defn- split-tables-and-legacy-card-refs [source-ids]
- (-> (reduce (fn [m src]
- (if-let [card-id (lib.util/legacy-string-table-id->card-id src)]
- (update m :cards conj! card-id)
- (update m :tables conj! src)))
- {:cards (transient #{})
- :tables (transient #{})}
- source-ids)
- (update-vals persistent!)))
-
(defn- query->template-tags
[query]
(-> query :native :template-tags vals))
@@ -124,13 +114,8 @@
(defn- batch-fetch-query-metadata*
"Fetch dependent metadata for ad-hoc queries."
[queries]
- (let [source-ids (into #{}
- ;; existing usage -- do not use in new code
- #_{:clj-kondo/ignore [:deprecated-var]}
- (mapcat #(lib.util/collect-source-tables (:query %)))
- queries)
- {source-table-ids :tables
- source-card-ids :cards} (split-tables-and-legacy-card-refs source-ids)
+ (let [{source-table-ids :table
+ source-card-ids :card} (lib.util/source-tables-and-cards queries)
source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids)
(schema.table/batch-fetch-card-query-metadatas source-card-ids
{:include-database? false}))
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 9ce2cd0e21d..1f8a46a08ed 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1189,7 +1189,9 @@
;; this is data migration column
:dataset_query_metrics_v2_migration_backup
;; this column is not used anymore
- :cache_ttl]
+ :cache_ttl
+ ;; dependencies aren't serialized, so the version of dependency analysis done shouldn't be serialized
+ :dependency_analysis_version]
:transform
{:created_at (serdes/date)
:database_id (serdes/fk :model/Database :name)
diff --git a/src/metabase/query_processor.clj b/src/metabase/query_processor.clj
index b505b5d7298..db160ec0279 100644
--- a/src/metabase/query_processor.clj
+++ b/src/metabase/query_processor.clj
@@ -7,6 +7,7 @@
Various REST API endpoints, such as `POST /api/dataset`, return the results of queries; they usually
use [[userland-query]] or [[userland-query-with-default-constraints]] (see below)."
(:require
+ [metabase.lib.core :as lib]
[metabase.lib.schema.info :as lib.schema.info]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.debug :as qp.debug]
@@ -80,6 +81,13 @@
(let [rff (or rff qp.reducible/default-rff)]
(process-query* query rff)))))
+(mu/defn compile-query-with-metadata-provider
+ "Compiles a given query with the provided metadata provider."
+ [metadata-providerable :- :metabase.lib.schema.metadata/metadata-providerable
+ query :- ::qp.schema/any-query]
+ (-> (lib/query metadata-providerable query)
+ qp.compile/compile))
+
(mu/defn userland-query :- ::qp.schema/any-query
"Add middleware options and `:info` to a `query` so it is ran as a 'userland' query, which slightly changes the QP
behavior:
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index 67862862661..de89abb37d9 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -32,6 +32,7 @@
:model/ContentTranslation
:model/DashboardFavorite
:model/DatabaseRouter
+ :model/Dependency
:model/PythonLibrary
:model/Query
:model/QueryCache
diff --git a/test/metabase/driver/sql/references_test.clj b/test/metabase/driver/sql/references_test.clj
new file mode 100644
index 00000000000..0423877ac83
--- /dev/null
+++ b/test/metabase/driver/sql/references_test.clj
@@ -0,0 +1,2172 @@
+(ns metabase.driver.sql.references-test
+ (:require
+ [clojure.test :refer :all]
+ [macaw.core :as macaw]
+ [metabase.driver.sql.references :as sql.references]))
+
+(defn- ->references [query]
+ (->> query macaw/parsed-query macaw/->ast (sql.references/field-references :sql)))
+
+(deftest garbage-test
+ (is (= {:used-fields #{}
+ :returned-fields []
+ :bad-sql true}
+ (->references "nothing"))))
+
+(deftest basic-select-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]}
+ (->references "select * from (select a, b from products)"))))
+
+(deftest basic-join-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:type :all-columns, :table {:table "products"}}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]}
+ (->references "select products.*, orders.id from products inner join orders on products.id = orders.product_id"))))
+
+(deftest indeterminite-join-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "orders"}}
+ {:type :all-columns, :table {:table "products"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "orders"}}
+ {:type :all-columns, :table {:table "products"}}]]}]}
+ (->references "select id from products inner join orders on products.id = orders.product_id"))))
+
+(deftest table-wildcard-join-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:type :all-columns, :table {:table "orders"}}]}
+ (->references "select orders.* from products inner join orders on products.id = orders.product_id"))))
+
+(deftest wildcard-join-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:type :all-columns, :table {:table "orders"}}
+ {:type :all-columns, :table {:table "products"}}]}
+ (->references "select * from products inner join orders on products.id = orders.product_id"))))
+
+(deftest basic-alias-test
+ (is (= {:used-fields
+ #{{:column "c",
+ :alias "d",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}
+ {:column "a",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}
+ {:column "c",
+ :alias "d",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}]}
+ (->references "select p.a as b, p.c as d from products p"))))
+
+(deftest basic-nested-query-test
+ (is (= {:used-fields
+ #{{:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}]}
+ (->references "select a, b from (select a, b from products)"))))
+
+(deftest renamed-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "a",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}]}
+ (->references "select b as c from (select a as b from products)"))))
+
+(deftest broken-nested-query-test
+ (is (= {:used-fields
+ #{{:column "b",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[]]}]}
+ (->references "select a from (select b from products)"))))
+
+(deftest different-case-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}]}
+ (->references "select A from (select a from orders)"))))
+
+(deftest wildcard-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}]}
+ (->references "select * from (select a from orders)"))))
+
+(deftest table-wildcard-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}]}
+ (->references "select o.* from (select a from orders) o"))))
+
+(deftest bad-table-wildcard-nested-query-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:type :invalid-table-wildcard, :table "foo"}]}
+ (->references "select foo.* from (select a from orders)"))))
+
+(deftest bad-table-name-test
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[]]}]}
+ (->references "select bad.a from products"))))
+
+(deftest basic-where-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]}
+ (->references "select * from products where category = 'hello'"))))
+
+(deftest basic-aggregation-test
+ (is (=? {:used-fields #{},
+ :returned-fields [{:alias nil, :type :custom-field, :used-fields #{}}]}
+ (->references "select count(*) from products"))))
+
+(deftest named-aggregation-test
+ (is (=? {:used-fields #{},
+ :returned-fields [{:alias "count", :type :custom-field, :used-fields #{}}]}
+ (->references "select count(*) as count from products"))))
+
+(deftest extra-names-test
+ (is (= {:used-fields
+ #{{:column "col",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:database "db",
+ :schema "schema",
+ :table "table"}}]]}},
+ :returned-fields
+ [{:column "col",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:database "db",
+ :schema "schema",
+ :table "table"}}]]}]}
+ (->references "select db.schema.table.col from db.schema.table"))))
+
+(deftest basic-grouping-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:alias "sum",
+ :type :custom-field,
+ :used-fields
+ #{{:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]}
+ (->references "select sum(total) as sum from orders group by category"))))
+
+(deftest basic-arg-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]}
+ (->references "select * from products where category = ?"))))
+
+(deftest basic-case-test
+ (is (=? {:used-fields
+ #{{:column "tax",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "subtotal",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "tax",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "total",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "subtotal",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}}}]}
+ (->references "select case when total < 0 then -subtotal else tax end from orders"))))
+
+(deftest switch-case-test
+ (is (=? {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields
+ [{:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}}}]}
+ (->references "select case category when 'Gizmo' then 'is gizmo' else 'is not gizmo' end from products"))))
+
+(deftest basic-select-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]}
+ (->references "select (select category from products where products.id = orders.product_id), * from orders"))))
+
+(deftest named-select-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias "category2",
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]}
+ (->references "select (select category from products where products.id = orders.product_id) as category2, * from orders"))))
+
+(deftest nested-select-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ []
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ []
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]}
+ (->references "select (select (select category from products where products.id = orders.product_id)), * from orders"))))
+
+(deftest select-subquery-with-normal-subquery-test
+ (is (= {:used-fields
+ #{{:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]
+ [{:type :all-columns,
+ :table {:table "orders"}}]]}
+ {:type :all-columns, :table {:table "orders"}}]}
+ (->references "select (select * from (select category from products where products.id = orders.product_id) sub), * from orders"))))
+
+(deftest nested-select-subquery-with-reference-to-middle-select-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[]
+ [{:type :all-columns, :table {:table "products"}}]
+ [{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields
+ [{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[]
+ [{:type :all-columns, :table {:table "products"}}]
+ [{:type :all-columns, :table {:table "orders"}}]]}]}
+ (->references "select (select (select category) from products where products.id = orders.product_id) from orders"))))
+
+(deftest nested-select-subquery-with-direct-match-in-outer-query
+ (is (= {:used-fields
+ #{{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}
+ {:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "t1"}}]
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}},
+ :returned-fields
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table "t1"}}]
+ [{:column "a",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}]}
+ (->references "select (select a from t1) from (select a from t2)"))))
+
+(deftest basic-exists-test
+ (is (= {:used-fields
+ #{{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}
+ {:column "user_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "o", :table "orders"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}
+ {:column "email",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}},
+ :returned-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}
+ {:column "email",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table-alias "u", :table "users"}}]]}]}
+ (->references "SELECT u.name, u.email
+FROM users u
+WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
+
+(deftest basic-not-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]}
+ (->references "select * from products where not (category = 'Gizmo')"))))
+
+(deftest basic-is-null-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]}
+ (->references "select * from products where category is null"))))
+
+(deftest negated-is-null-test
+ (is (= {:used-fields
+ #{{:column "category",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "products"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]}
+ (->references "select * from products where category is not null"))))
+
+(deftest basic-between-test
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "left",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}
+ {:column "right",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},
+ :returned-fields [{:type :all-columns, :table {:table "orders"}}]}
+ (->references "select * from orders where created_at between left and right"))))
+
+(deftest basic-cte-test
+ (is (= {:used-fields
+ #{{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "active",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}},
+ :returned-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}]}
+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)
+SELECT * FROM active_users"))))
+
+(deftest unused-cte-test
+ (is (= {:used-fields
+ #{{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "active",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}},
+ :returned-fields [{:type :all-columns,
+ :table {:table "products"}}]}
+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)
+SELECT * FROM products"))))
+
+(deftest shadowed-cte-test
+ (is (= {:used-fields
+ #{{:column "y",
+ :alias "x",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "y",
+ :alias "a",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "z",
+ :alias "y",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "x",
+ :alias "z",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "z",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "x",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}},
+ :returned-fields
+ [{:column "y",
+ :alias "a",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "z",
+ :alias "b",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}
+ {:column "x",
+ :alias "c",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}]}
+ (->references "WITH c AS (
+ WITH b AS (
+ SELECT
+ x as z,
+ y as x,
+ z as y
+ FROM a
+ )
+ SELECT
+ x as a,
+ y as b,
+ z as c
+ FROM b
+)
+SELECT
+ a,
+ b,
+ c
+FROM c;"))))
+
+(deftest recursive-cte-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "h", :table "emp_hierarchy"}}]]}
+ {:alias "name",
+ :type :composite-field,
+ :member-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}]}
+ {:column "manager_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}
+ {:column "level",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "h",
+ :table "emp_hierarchy"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}
+ {:alias "level",
+ :type :composite-field,
+ :member-fields
+ [{:alias "level", :type :custom-field, :used-fields #{}}
+ {:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "level",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "h",
+ :table "emp_hierarchy"}}]]}}}]}
+ {:column "manager_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}},
+ :returned-fields
+ [{:alias "name",
+ :type :composite-field,
+ :member-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "e",
+ :table "employees"}}]]}]}
+ {:alias "level",
+ :type :composite-field,
+ :member-fields
+ [{:alias "level", :type :custom-field, :used-fields #{}}
+ {:alias nil,
+ :type :custom-field,
+ :used-fields
+ #{{:column "level",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "h",
+ :table "emp_hierarchy"}}]]}}}]}]}
+ (->references "WITH RECURSIVE emp_hierarchy AS (
+ SELECT id, name, manager_id, 0 AS level
+ FROM employees
+ WHERE manager_id IS NULL
+ UNION ALL
+ SELECT e.id, e.name, e.manager_id, h.level + 1
+ FROM employees e
+ JOIN emp_hierarchy h ON e.manager_id = h.id
+)
+SELECT name, level FROM emp_hierarchy"))))
+
+(deftest basic-union-test
+ (is (= {:used-fields
+ #{{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}},
+ :returned-fields
+ [{:alias "id",
+ :type :composite-field,
+ :member-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}]}
+ {:alias "name",
+ :type :composite-field,
+ :member-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "users"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "archived_users"}}]]}]}]}
+ (->references "SELECT id, name FROM users
+UNION
+SELECT id, name FROM archived_users"))))
+
+(deftest row-number-test
+ (is (= {:used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}},
+ :returned-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}
+ {:alias "rank",
+ :type :custom-field,
+ :used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns,
+ :table {:table "employees"}}]]}}}]}
+ (->references "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) AS rank
+FROM employees"))))
+
+(deftest partition-by-test
+ (is (= {:used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "department",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}},
+ :returned-fields
+ [{:column "name",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:alias "dept_rank",
+ :type :custom-field,
+ :used-fields
+ #{{:column "salary",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}
+ {:column "department",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}}}]}
+ (->references "SELECT name,
+ RANK() OVER (PARTITION BY department ORDER BY salary DESC) AS dept_rank
+FROM employees"))))
+
+(deftest week-test
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}},
+ :returned-fields
+ [{:alias "created_at",
+ :type :custom-field,
+ :used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "count", :type :custom-field, :used-fields #{}}]}
+ (->references "SELECT
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ ) AS \"created_at\",
+ COUNT(*) AS \"count\"
+FROM
+ \"public\".\"orders\"
+GROUP BY
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ORDER BY
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ ) ASC"))))
+
+(deftest week-of-year-test
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}},
+ :returned-fields
+ [{:alias "created_at",
+ :type :custom-field,
+ :used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "count", :type :custom-field, :used-fields #{}}]}
+ (->references "SELECT
+ CEIL(
+ (
+ CAST(
+ extract(
+ doy
+ from
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ ) AS integer
+ ) / 7.0
+ )
+ ) AS \"created_at\",
+ COUNT(*) AS \"count\"
+FROM
+ \"public\".\"orders\"
+GROUP BY
+ CEIL(
+ (
+ CAST(
+ extract(
+ doy
+ from
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ ) AS integer
+ ) / 7.0
+ )
+ )
+ORDER BY
+ CEIL(
+ (
+ CAST(
+ extract(
+ doy
+ from
+ (
+ DATE_TRUNC(
+ 'week',
+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')
+ ) + INTERVAL '-1 day'
+ )
+ ) AS integer
+ ) / 7.0
+ )
+ ) ASC"))))
+
+;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the
+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.
+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
+(deftest complicated-test-1
+ (is (= {:used-fields
+ #{{:column "title",
+ :alias "title",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "discount",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "rating",
+ :alias "rating",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:alias "upper_category",
+ :type :custom-field,
+ :used-fields
+ #{{:column "category",
+ :alias "category",
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}}}
+ {:column "ean",
+ :alias "ean",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "created_at",
+ :alias "created_at",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "product_id",
+ :alias "product_id",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "category",
+ :alias "Products__category",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "product_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "category",
+ :alias "category",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "created_at",
+ :alias "created_at",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "discount",
+ :alias "discount",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "vendor",
+ :alias "vendor",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "total",
+ :alias "total",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}
+ {:column "id",
+ :alias "Products__id",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "id",
+ :alias "id",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}
+ {:column "price",
+ :alias "price",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}},
+ :returned-fields
+ [{:alias "created_at",
+ :type :custom-field,
+ :used-fields
+ #{{:column "created_at",
+ :alias "created_at",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "upper_category",
+ :type :custom-field,
+ :used-fields
+ #{{:column "category",
+ :alias "category",
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:schema "public",
+ :table "products"}}]]}}}
+ {:alias "sum",
+ :type :custom-field,
+ :used-fields
+ #{{:column "total",
+ :alias "total",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "sum_2",
+ :type :custom-field,
+ :used-fields
+ #{{:column "discount",
+ :alias "discount",
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:schema "public",
+ :table "orders"}}]]}}}
+ {:alias "count", :type :custom-field, :used-fields #{}}]}
+ (->references "SELECT
+ DATE_TRUNC('month', \"source\".\"created_at\") AS \"created_at\",
+ \"source\".\"upper_category\" AS \"upper_category\",
+ SUM(\"source\".\"total\") AS \"sum\",
+ SUM(\"source\".\"discount\") AS \"sum_2\",
+ COUNT(*) AS \"count\"
+FROM
+ (
+ SELECT
+ \"public\".\"orders\".\"product_id\" AS \"product_id\",
+ \"public\".\"orders\".\"total\" AS \"total\",
+ \"public\".\"orders\".\"discount\" AS \"discount\",
+ \"public\".\"orders\".\"created_at\" AS \"created_at\",
+ UPPER(\"Products\".\"category\") AS \"upper_category\",
+ \"Products\".\"id\" AS \"Products__id\",
+ \"Products\".\"category\" AS \"Products__category\"
+ FROM
+ \"public\".\"orders\"
+
+LEFT JOIN (
+ SELECT
+ \"public\".\"products\".\"id\" AS \"id\",
+ \"public\".\"products\".\"ean\" AS \"ean\",
+ \"public\".\"products\".\"title\" AS \"title\",
+ \"public\".\"products\".\"category\" AS \"category\",
+ \"public\".\"products\".\"vendor\" AS \"vendor\",
+ \"public\".\"products\".\"price\" AS \"price\",
+ \"public\".\"products\".\"rating\" AS \"rating\",
+ \"public\".\"products\".\"created_at\" AS \"created_at\"
+ FROM
+ \"public\".\"products\"
+ ) AS \"Products\" ON \"public\".\"orders\".\"product_id\" = \"Products\".\"id\"
+
+WHERE
+ \"public\".\"orders\".\"discount\" > 0
+ ) AS \"source\"
+GROUP BY
+ DATE_TRUNC('month', \"source\".\"created_at\"),
+ \"source\".\"upper_category\"
+ORDER BY
+ DATE_TRUNC('month', \"source\".\"created_at\") ASC,
+ \"source\".\"upper_category\" ASC"))))
+
+;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the
+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.
+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
+(deftest complicated-test-2
+ (is (= {:used-fields
+ #{{:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "is_converted",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "annual_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "base_fee",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "account_id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_alias",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_paid_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_active_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_active_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_paid_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_trialing_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "is_active_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:alias "is_active_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_active_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_trialing_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:column "customer_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_trialing",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:column "is_active",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:alias "is_paid_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_paid_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_paid_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}
+ {:column "valid_to",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table
+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "valid_from",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table
+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}},
+ :returned-fields
+ [{:column "id",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "customer_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "created_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "is_converted",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "is_active",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_alias",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "base_fee",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "annual_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}
+ {:alias "is_trialing",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_trialing_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_active_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_cloud",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_trialing_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_active_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_self_hosted",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "deployment",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_active_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_enterprise",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_active_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_starter",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}
+ {:alias "is_active_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "subscription_status",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}
+ {:alias "is_paid_accounts_pro",
+ :type :custom-field,
+ :used-fields
+ #{{:column "trial_ended_at",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "lifetime_value",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}
+ {:column "plan_name",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns,
+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}
+ {:column "month",
+ :alias nil,
+ :type :single-column,
+ :source-columns
+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}]}
+ (->references "with months as (
+ select
+ generate_series(date_trunc('month', '2024-01-01'::date),
+ date_trunc('month', current_date) - interval '1 month',
+ '1 month')::date as month
+
+)
+, monthly_account_subscription_status as (
+ select
+ a.id,
+ m.month,
+
+ max(case when ah.subscription_status is not null and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing,
+
+ -- cloud
+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_cloud,
+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' then 1 else 0 end) as is_active_accounts_cloud,
+ max(case when a.lifetime_value > 0 and a.deployment = 'cloud' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_cloud,
+
+ -- self-hosted
+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_self_hosted,
+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' then 1 else 0 end) as is_active_accounts_self_hosted,
+ max(case when a.lifetime_value > 0 and a.deployment = 'self-hosted' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_self_hosted,
+
+ -- enterprise
+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'enterprise' then 1 else 0 end) as is_active_accounts_enterprise,
+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'enterprise' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_enterprise,
+
+ -- starter
+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'starter' then 1 else 0 end) as is_active_accounts_starter,
+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'starter' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_starter,
+
+ -- pro
+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'pro' then 1 else 0 end) as is_active_accounts_pro,
+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'pro' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_pro
+ from dbt_models.account a
+ left join dbt_models.account_history ah
+ on a.id = ah.account_id
+ left join months m
+ on m.month between date_trunc('month', ah.valid_from) and date_trunc('month', ah.valid_to)
+ group by 1,2
+)
+select
+ a.id,
+ a.customer_name,
+ a.created_at,
+ a.is_converted,
+ a.lifetime_value,
+ a.is_active,
+ a.subscription_status,
+ a.deployment,
+ a.plan_name,
+ a.plan_alias,
+ a.base_fee,
+ a.annual_value,
+
+ -- monthly account status
+ mas.month,
+ mas.is_trialing,
+ mas.is_trialing_accounts_cloud,
+ mas.is_active_accounts_cloud,
+ mas.is_paid_accounts_cloud,
+ mas.is_trialing_accounts_self_hosted,
+ mas.is_active_accounts_self_hosted,
+ mas.is_paid_accounts_self_hosted,
+ mas.is_active_accounts_enterprise,
+ mas.is_paid_accounts_enterprise,
+ mas.is_active_accounts_starter,
+ mas.is_paid_accounts_starter,
+ mas.is_active_accounts_pro,
+ mas.is_paid_accounts_pro
+
+from dbt_models.account a
+left join monthly_account_subscription_status mas
+ on a.id = mas.id
+;"))))
diff --git a/test/metabase/lib/test_util/metadata_providers/mock.cljc b/test/metabase/lib/test_util/metadata_providers/mock.cljc
index d4a64fe88c0..60a32f69723 100644
--- a/test/metabase/lib/test_util/metadata_providers/mock.cljc
+++ b/test/metabase/lib/test_util/metadata_providers/mock.cljc
@@ -39,6 +39,8 @@
[:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/segment :metadata/segment)]]]
[:native-query-snippets {:optional true}
[:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/native-query-snippet :metadata/native-query-snippet)]]]
+ [:transforms {:optional true}
+ [:maybe [:sequential (with-optional-lib-type :map :metadata/transform)]]]
[:settings {:optional true}
[:maybe [:map-of :keyword any?]]]])
@@ -54,7 +56,8 @@
:metadata/card :cards
:metadata/segment :segments
:metadata/native-query-snippet :native-query-snippets
- :metadata/metric :cards)]
+ :metadata/metric :cards
+ :metadata/transform :transforms)]
(into []
(comp (metadata.protocols/default-spec-filter-xform metadata-spec)
(map #(assoc % :lib/type metadata-type)))
diff --git a/test/metabase/revisions/impl/card_test.clj b/test/metabase/revisions/impl/card_test.clj
index 0109d9c2c9e..29ea1bfb0f2 100644
--- a/test/metabase/revisions/impl/card_test.clj
+++ b/test/metabase/revisions/impl/card_test.clj
@@ -137,7 +137,10 @@
:card_schema
;; we don't expect a description for this column because it should never change
;; once created by the migration
- :dataset_query_metrics_v2_migration_backup} col)
+ :dataset_query_metrics_v2_migration_backup
+ ;; `dependency_analysis_version` is an internal bookkeeping field. It doesn't affect the
+ ;; actual card itself, so no description is necessary.
+ :dependency_analysis_version} col)
(testing (format "we should have a revision description for %s" col)
(let [diff-strings (revision/diff-strings
;; TODO -- huh? Shouldn't this be testing against `:model/Card` here???