The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Dependencies tracking (#63248)
William85fc7cb3fd99
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex d320ccc7291..5438d7b3c31 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -428,6 +428,11 @@:api #{metabase.database-routing.core}:uses #{premium-features}}+ dependencies+ {:team "Querying"+ :api #{metabase.premium-features.core}+ :uses #{premium-features}}+driver{:team "Drivers":api #{metabase.driver@@ -556,6 +561,11 @@modelsutil}}+ graph+ {:team "Querying"+ :api #{metabase.graph.core}+ :uses #{util}}+indexed-entities{:team "UX West":api #{metabase.indexed-entities.api@@ -725,6 +735,7 @@metabase.native-query-snippets.core}:uses #{apicollections+ eventslibmodelspermissions@@ -1716,6 +1727,29 @@warehouse-schemawarehouses}}+ enterprise/dependencies+ {:team "Querying"+ :api #{metabase-enterprise.dependencies.api+ metabase-enterprise.dependencies.core+ metabase-enterprise.dependencies.init}+ :uses #{analyze+ api+ app-db+ collections+ config+ driver+ events+ graph+ lib+ lib-be+ models+ native-query-snippets+ premium-features+ queries+ query-processor+ task+ util}}+enterprise/documents{:team "UX West":api #{metabase-enterprise.documents.apidiff --git a/deps.edn b/deps.ednindex b82eabaf744..6577a696963 100644--- a/deps.edn+++ b/deps.edn@@ -90,7 +90,7 @@:git/sha "4998a1fdd9d3713d7034d04509e9212204773bf2":git/url "https://github.com/eerohele/pp"};; TODO: replace back to normal artifact- io.github.metabase/macaw {:mvn/version "0.2.27"} ; Parse native SQL queries+ io.github.metabase/macaw {:mvn/version "0.2.28"} ; Parse native SQL queriesio.github.resilience4j/resilience4j-retry ^:antq/exclude ; 2.x do not support Java 8{:mvn/version "1.7.1"} ; Support for retrying operationsio.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collectordiff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 36bd741b38a..cc78d390ed8 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -29,6 +29,8 @@ title: Driver interface changelog- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabasetype (from the type hierarchy) as a HoneySQL spec. This method handles general Metabase base types.+- Added driver multimethods driver/native-result-metadata, driver/validate-native-query-fields, driver.sql/resolve-field, driver.sql.normalize-unquoted-name, driver.sql.normalize/reserved-literal, driver.sql.references/find-used-fields, driver.sql.references/find-returned-fields, and driver.sql.references/field-references-impl for use with the :dependencies/native feature.+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sourcesinto existing tables. This multimethod dispatches on both the driver and the data source type(`:rows` or `:jsonl-file`). It allows drivers to optimize based on the data source type and returns the numberdiff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.tsnew file mode 100644index 00000000000..ee0beb942c5--- /dev/null+++ b/e2e/support/helpers/e2e-transform-helpers.ts@@ -0,0 +1,37 @@+import type {+ ListTransformRunsResponse,+ TransformId,+ TransformRun,+} from "metabase-types/api";++export function visitTransform(transformId: TransformId) {+ cy.visit(`/admin/transforms/${transformId}`);+}++const WAIT_TIMEOUT = 10000;+const WAIT_INTERVAL = 100;++export function waitForTransformRuns(+ filter: (runs: TransformRun[]) => boolean,+ timeout = WAIT_TIMEOUT,+): Cypress.Chainable {+ return cy+ .request<ListTransformRunsResponse>("GET", "/api/ee/transform/run")+ .then((response) => {+ if (filter(response.body.data)) {+ return cy.wrap(response);+ } else if (timeout > 0) {+ cy.wait(WAIT_INTERVAL);+ return waitForTransformRuns(filter, timeout - WAIT_INTERVAL);+ } else {+ throw new Error("Run retry timeout");+ }+ });+}++export function waitForSucceededTransformRuns() {+ waitForTransformRuns(+ (runs) =>+ runs.length > 0 && runs.every((run) => run.status === "succeeded"),+ );+}diff --git a/e2e/support/helpers/index.ts b/e2e/support/helpers/index.tsindex 36e00b8db54..6bea5ad97a7 100644--- a/e2e/support/helpers/index.ts+++ b/e2e/support/helpers/index.ts@@ -47,6 +47,7 @@ export * from "./e2e-slack-helpers";export * from "./e2e-snowplow-helpers";export * from "./e2e-table-metadata-helpers";export * from "./e2e-token-helpers";+export * from "./e2e-transform-helpers";export * from "./e2e-ui-elements-helpers";export * from "./e2e-ui-elements-overflow-helpers";export * from "./e2e-upload-helpers";diff --git a/e2e/test/scenarios/admin/transforms.cy.spec.ts b/e2e/test/scenarios/admin/transforms.cy.spec.tsindex 60ef560d58f..1c3fbe438b8 100644--- a/e2e/test/scenarios/admin/transforms.cy.spec.ts+++ b/e2e/test/scenarios/admin/transforms.cy.spec.ts@@ -6,7 +6,6 @@ import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";import type {CardType,- ListTransformRunsResponse,PythonTransformTableAliases,TransformTagId,} from "metabase-types/api";@@ -1944,7 +1943,7 @@ describe("scenarios > admin > transforms > jobs", () => {tag_ids: [tag.id],});});- waitForRuns();+ H.waitForSucceededTransformRuns();visitRunListPage();getContentTable().within(() => {cy.findAllByText("MBQL transform").should("have.length.gte", 1);@@ -2961,24 +2960,6 @@ function visitTableQuestion({);}-const WAIT_TIMEOUT = 10000;-const WAIT_INTERVAL = 100;--function waitForRuns(timeout = WAIT_TIMEOUT): Cypress.Chainable {- return cy- .request<ListTransformRunsResponse>("GET", "/api/ee/transform/run")- .then((response) => {- if (response.body.data.length > 0) {- return cy.wrap(response);- } else if (timeout > 0) {- cy.wait(WAIT_INTERVAL);- return waitForRuns(timeout - WAIT_INTERVAL);- } else {- throw new Error("Run retry timeout");- }- });-}-function assertTableDoesNotExistError({targetTable = TARGET_TABLE,targetSchema = TARGET_SCHEMA,diff --git a/e2e/test/scenarios/dependencies/dependencies.cy.spec.ts b/e2e/test/scenarios/dependencies/dependencies.cy.spec.tsnew file mode 100644index 00000000000..8bc92b89c54--- /dev/null+++ b/e2e/test/scenarios/dependencies/dependencies.cy.spec.ts@@ -0,0 +1,678 @@+const { H } = cy;++import { WRITABLE_DB_ID } from "e2e/support/cypress_data";+import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";+import { ORDERS_DASHBOARD_ID } from "e2e/support/cypress_sample_instance_data";++const { PRODUCTS_ID } = SAMPLE_DATABASE;++H.describeWithSnowplowEE("documents", () => {+ beforeEach(() => {+ H.restore("postgres-writable");+ H.resetTestTable({ type: "postgres", table: "many_schemas" });+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: "Animals" });+ H.resetSnowplow();++ cy.intercept("PUT", "/api/card/*").as("updateCard");+ cy.intercept("PUT", "/api/native-query-snippet/*").as("updateSnippet");+ cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");+ });++ afterEach(() => {+ H.expectNoBadSnowplowEvents();+ });++ describe("questions", () => {+ it("should be able to confirm or cancel breaking changes to a MBQL question", () => {+ createMbqlQuestionWithDependentMbqlQuestions();++ cy.log("make breaking changes");+ H.visitQuestion("@questionId");+ H.openNotebook();+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();++ cy.log("cancel breaking changes");+ H.queryBuilderHeader().button("Save").click();+ H.modal().button("Save").click();+ H.modal().within(() => {+ cy.findByText("Question with fields").should("be.visible");+ cy.findByText("Question without fields").should("not.exist");+ cy.findByText("Base question").should("not.exist");+ cy.button("Cancel").click();+ });+ cy.get("@updateCard.all").should("have.length", 0);++ cy.log("confirm breaking changes");+ H.queryBuilderHeader().button("Save").click();+ H.modal().button("Save").click();+ H.modal().button("Save anyway").click();+ cy.wait("@updateCard");+ });++ it("should be able to navigate to affected questions or their collection", () => {+ createMbqlQuestionWithDependentMbqlQuestions();++ cy.log("check that we can navigate to the broken question");+ H.visitQuestion("@questionId");+ H.openNotebook();+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();+ H.queryBuilderHeader().button("Save").click();+ H.modal().button("Save").click();+ H.modal().findByText("Question with fields").click();+ confirmDiscardChanges();+ H.queryBuilderHeader()+ .findByDisplayValue("Question with fields")+ .should("be.visible");++ cy.log("check that we can navigate to the collection of that question");+ cy.go("back");+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();+ H.queryBuilderHeader().button("Save").click();+ H.modal().button("Save").click();+ H.modal().within(() => {+ cy.findByText("Question with fields").should("be.visible");+ cy.findByText("Our analytics").click();+ });+ confirmDiscardChanges();+ H.collectionTable().should("be.visible");++ cy.log("check that we can navigate to the dashboard of that question");+ cy.go("back");+ H.getNotebookStep("expression").findByText("Expr").icon("close").click();+ H.queryBuilderHeader().button("Save").click();+ H.modal().button("Save").click();+ H.modal().within(() => {+ cy.findByText("Question with fields").should("be.visible");+ cy.findByText("Orders in a dashboard").click();+ });+ confirmDiscardChanges();+ H.dashboardHeader().should("be.visible");+ });++ it("should not show a confirmation if there are no breaking changes when updating a MBQL question", () => {+ createMbqlQuestionWithDependentMbqlQuestions();+ H.visitQuestion("@questionId");+ H.openNotebook();+ H.getNotebookStep("expression").findByText("Expr").click();+ H.CustomExpressionEditor.clear().type("2 + 2");+ H.popover().button("Update").click();+ H.queryBuilderHeader().button("Save").click();+ H.modal().button("Save").click();+ cy.wait("@updateCard");+ });+ });++ describe("models", () => {+ it("should be able to confirm or cancel breaking changes to a SQL model", () => {+ createSqlModelWithDependentSqlQuestions();++ cy.log("make breaking changes");+ cy.get<number>("@modelId").then(H.visitModel);+ H.openQuestionActions("Edit query definition");+ H.NativeEditor.clear().type("SELECT ID, TITLE FROM PRODUCTS");+ H.runNativeQuery();++ cy.log("cancel breaking changes");+ H.datasetEditBar().button("Save changes").click();+ H.modal().within(() => {+ cy.findByText("Question with fields").should("be.visible");+ cy.findByText("Question without fields").should("not.exist");+ cy.findByText("Base model").should("not.exist");+ cy.button("Cancel").click();+ });+ cy.get("@updateCard.all").should("have.length", 0);++ cy.log("confirm breaking changes");+ H.datasetEditBar().button("Save changes").click();+ H.modal().button("Save anyway").click();+ cy.wait("@updateCard");+ });++ it("should not show a confirmation if there are no breaking changes when updating a SQL model", () => {+ createSqlModelWithDependentSqlQuestions();+ cy.get<number>("@modelId").then(H.visitModel);+ H.openQuestionActions("Edit query definition");+ H.NativeEditor.clear().type("SELECT ID, CATEGORY FROM PRODUCTS");+ H.runNativeQuery();+ H.datasetEditBar().button("Save changes").click();+ cy.wait("@updateCard");+ });+ });++ describe("metrics", () => {+ it("should be able to confirm or cancel breaking changes to a metric", () => {+ createMetricWithDependentMbqlQuestionsAndTransforms();++ cy.log("make breaking changes");+ cy.get<number>("@metricId").then(H.visitMetric);+ H.openQuestionActions("Edit metric definition");+ H.getNotebookStep("summarize").findByText("Min of Score").click();+ H.popover().within(() => {+ cy.icon("chevronleft").click();+ cy.findByText("Maximum of ...").click();+ cy.findByText("Score").click();+ });++ cy.log("cancel breaking changes");+ cy.findByTestId("edit-bar").button("Save changes").click();+ H.modal().within(() => {+ cy.findByText("Question with 2 stages").should("be.visible");+ cy.findByText("Transform with 2 stages").should("be.visible");+ cy.findByText("Question with 1 stage").should("not.exist");+ cy.findByText("Transform with 1 stage").should("not.exist");+ cy.findByText("Base metric").should("not.exist");+ cy.button("Cancel").click();+ });+ cy.get("@updateCard.all").should("have.length", 0);++ cy.log("confirm breaking changes");+ cy.findByTestId("edit-bar").button("Save changes").click();+ H.modal().button("Save anyway").click();+ cy.wait("@updateCard");+ });++ it("should not show a warning when a change to a metric is backward-compatible with existing content", () => {+ createMetricWithDependentMbqlQuestionsAndTransforms();+ cy.get<number>("@metricId").then(H.visitMetric);+ H.openQuestionActions("Edit metric definition");+ H.getNotebookStep("summarize").findByText("Min of Score").click();+ H.popover().findByText("Name").click();+ cy.findByTestId("edit-bar").button("Save changes").click();+ cy.wait("@updateCard");+ });+ });++ describe("transforms", () => {+ it("should be able to confirm or cancel breaking changes to a SQL transform after it was run", () => {+ createSqlTransformWithDependentMbqlQuestions();++ cy.log("make breaking changes");+ cy.get<number>("@transformId").then(H.visitTransform);+ cy.findByTestId("transform-page").findByText("Edit query").click();+ H.NativeEditor.clear().type('SELECT name FROM "Schema A"."Animals"');++ cy.log("cancel breaking changes");+ cy.findByTestId("transform-query-editor").button("Save changes").click();+ H.modal().within(() => {+ cy.findByText("Question with fields").should("be.visible");+ cy.findByText("Question without fields").should("not.exist");+ cy.findByText("Base transform").should("not.exist");+ cy.button("Cancel").click();+ });+ cy.get("@updateTransform.all").should("have.length", 0);++ cy.log("confirm breaking changes");+ cy.findByTestId("transform-query-editor").button("Save changes").click();+ H.modal().within(() => {+ cy.findByText("Question with fields").should("be.visible");+ cy.findByText("Question without fields").should("not.exist");+ cy.findByText("Base transform").should("not.exist");+ cy.button("Save anyway").click();+ });+ cy.wait("@updateTransform");+ });++ it("should not show a confirmation if there are no breaking changes when updating a SQL transform after it was run", () => {+ createSqlTransformWithDependentMbqlQuestions();+ cy.get<number>("@transformId").then(H.visitTransform);+ cy.findByTestId("transform-page").findByText("Edit query").click();+ H.NativeEditor.clear().type(+ 'SELECT score, name FROM "Schema A"."Animals"',+ );+ cy.findByTestId("transform-query-editor").button("Save changes").click();+ cy.wait("@updateTransform");+ });++ it("should be able to confirm or cancel breaking changes to a MBQL transform before it was run", () => {+ createMbqlTransformWithDependentSqlTransforms();++ cy.log("make breaking changes");+ cy.get<number>("@transformId").then(H.visitTransform);+ cy.findByTestId("transform-page").findByText("Edit query").click();+ H.getNotebookStep("data").findByLabelText("Pick columns").click();+ H.popover().findByLabelText("Score").click();++ cy.log("cancel breaking changes");+ cy.findByTestId("transform-query-editor").button("Save changes").click();+ H.modal().within(() => {+ cy.findByText("Score transform").should("be.visible");+ cy.findByText("Name transform").should("not.exist");+ cy.findByText("Base transform").should("not.exist");+ cy.button("Cancel").click();+ });+ cy.get("@updateTransform.all").should("have.length", 0);++ cy.log("confirm breaking changes");+ cy.findByTestId("transform-query-editor").button("Save changes").click();+ H.modal().within(() => {+ cy.findByText("Score transform").should("be.visible");+ cy.findByText("Name transform").should("not.exist");+ cy.button("Save anyway").click();+ });+ cy.wait("@updateTransform");+ });++ it("should not show a confirmation if there are no breaking changes when updating a MBQL transform before it was run", () => {+ createMbqlTransformWithDependentSqlTransforms();+ cy.get<number>("@transformId").then(H.visitTransform);+ cy.findByTestId("transform-page").findByText("Edit query").click();+ H.getNotebookStep("data").button("Sort").click();+ H.popover().findByText("Score").click();+ cy.findByTestId("transform-query-editor").button("Save changes").click();+ cy.wait("@updateTransform");+ });+ });++ describe("snippets", () => {+ it("should be able to confirm or cancel breaking changes to a snippet", () => {+ createSnippetWithDependentQuestionsAndTransforms();++ cy.log("make breaking changes");+ startEditingSnippet();+ H.modal()+ .findByLabelText("Enter some SQL here so you can reuse it later")+ .clear()+ .type("price = 1");++ cy.log("cancel breaking changes");+ H.modal().within(() => {+ cy.button("Save").click();+ cy.findByText("Question with snippet").should("be.visible");+ cy.findByText("Transform with snippet").should("be.visible");+ cy.findByText("ScoreSnippet").should("not.exist");+ cy.button("Cancel").click();+ });+ cy.get("@updateSnippet.all").should("have.length", 0);++ cy.log("confirm breaking changes");+ H.modal().within(() => {+ cy.button("Save").click();+ cy.findByText("Question with snippet").should("be.visible");+ cy.findByText("Transform with snippet").should("be.visible");+ cy.findByText("ScoreSnippet").should("not.exist");+ cy.button("Save anyway").click();+ });+ cy.wait("@updateSnippet");+ });++ it("should not warn if changes to a snippet are backward compatible", () => {+ createSnippetWithDependentQuestionsAndTransforms();+ startEditingSnippet();+ H.modal().within(() => {+ cy.findByLabelText("Enter some SQL here so you can reuse it later")+ .clear()+ .type("score = 2");+ cy.button("Save").click();+ });+ cy.wait("@updateSnippet");+ });+ });+});++function createMbqlQuestionWithDependentMbqlQuestions() {+ H.createQuestion({+ name: "Base question",+ query: {+ "source-table": PRODUCTS_ID,+ expressions: {+ Expr: ["+", 1, 1],+ },+ },+ }).then(({ body: card }) => {+ cy.wrap(card.id).as("questionId");++ H.createQuestion({+ name: "Question with fields",+ query: {+ "source-table": `card__${card.id}`,+ filter: [">", ["field", "Expr", { "base-type": "type/Integer" }], 1],+ },+ dashboard_id: ORDERS_DASHBOARD_ID,+ });++ H.createQuestion({+ name: "Question without fields",+ query: {+ "source-table": `card__${card.id}`,+ },+ });+ });+}++function createSqlModelWithDependentSqlQuestions() {+ H.createNativeQuestion({+ name: "Base model",+ type: "model",+ native: {+ query: "SELECT ID, TITLE, CATEGORY FROM PRODUCTS",+ "template-tags": {},+ },+ }).then(({ body: card }) => {+ cy.wrap(card.id).as("modelId");+ cy.request("POST", `/api/card/${card.id}/query`);++ const tagName = `#${card.id}`;+ H.createNativeQuestion({+ name: "Question with fields",+ native: {+ query: `SELECT ID, CATEGORY FROM {{#${card.id}}}`,+ "template-tags": {+ [tagName]: {+ id: "10422a0f-292d-10a3-fd90-407cc9e3e20e",+ name: tagName,+ "display-name": tagName,+ type: "card",+ "card-id": card.id,+ },+ },+ },+ });++ H.createNativeQuestion({+ name: "Question without fields",+ native: {+ query: `SELECT COUNT(*) FROM {{#${card.id}}}`,+ "template-tags": {+ [tagName]: {+ id: "10422a0f-292d-10a3-fd90-407cc9e3e20f",+ name: tagName,+ "display-name": tagName,+ type: "card",+ "card-id": card.id,+ },+ },+ },+ });+ });+}++function createMetricWithDependentMbqlQuestionsAndTransforms() {+ H.getTableId({ databaseId: WRITABLE_DB_ID, name: "Animals" }).then(+ (tableId) => {+ H.getFieldId({ tableId, name: "score" }).then((fieldId) => {+ H.createQuestion({+ name: "Base metric",+ type: "metric",+ database: WRITABLE_DB_ID,+ query: {+ "source-table": tableId,+ aggregation: [["min", ["field", fieldId, null]]],+ },+ }).then(({ body: metric }) => {+ cy.wrap(metric.id).as("metricId");++ H.createQuestion({+ name: "Question with 1 stage",+ database: WRITABLE_DB_ID,+ query: {+ "source-table": tableId,+ aggregation: [["metric", metric.id]],+ },+ });++ H.createQuestion({+ name: "Question with 2 stages",+ database: WRITABLE_DB_ID,+ query: {+ "source-query": {+ "source-table": tableId,+ aggregation: [["metric", metric.id]],+ breakout: [["field", fieldId, null]],+ },+ aggregation: [+ ["avg", ["field", "min", { "base-type": "type/Integer" }]],+ ],+ },+ });++ H.createTransform({+ name: "Transform with 1 stage",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "query",+ query: {+ "source-table": tableId,+ aggregation: [["metric", metric.id]],+ },+ },+ },+ target: {+ type: "table",+ name: "transform_1_stage",+ schema: "public",+ database: WRITABLE_DB_ID,+ },+ });++ H.createTransform({+ name: "Transform with 2 stages",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "query",+ query: {+ "source-query": {+ "source-table": tableId,+ aggregation: [["metric", metric.id]],+ breakout: [["field", fieldId, null]],+ },+ aggregation: [+ ["avg", ["field", "min", { "base-type": "type/Integer" }]],+ ],+ },+ },+ },+ target: {+ type: "table",+ name: "transform_2_stages",+ schema: "public",+ database: WRITABLE_DB_ID,+ },+ });+ });+ });+ },+ );+}++function createSqlTransformWithDependentMbqlQuestions() {+ const transformTableName = "base_transform";++ H.createTransform({+ name: "Base transform",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "native",+ native: {+ query: 'SELECT name, score FROM "Schema A"."Animals"',+ "template-tags": {},+ },+ },+ },+ target: {+ type: "table",+ name: transformTableName,+ schema: "public",+ database: WRITABLE_DB_ID,+ },+ }).then(({ body: transform }) => {+ cy.wrap(transform.id).as("transformId");+ cy.request("POST", `/api/ee/transform/${transform.id}/run`);+ H.waitForSucceededTransformRuns();+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: transformTableName });++ H.getTableId({ databaseId: WRITABLE_DB_ID, name: transformTableName }).then(+ (tableId) => {+ H.createQuestion({+ name: "Question with fields",+ database: WRITABLE_DB_ID,+ query: {+ "source-table": tableId,+ filter: [+ ">",+ ["field", "score", { "base-type": "type/Integer" }],+ 1,+ ],+ },+ });+ H.createQuestion({+ name: "Question without fields",+ database: WRITABLE_DB_ID,+ query: {+ "source-table": tableId,+ },+ });+ },+ );+ });+}++function createMbqlTransformWithDependentSqlTransforms() {+ H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(+ (tableId) => {+ H.createTransform(+ {+ name: "Base transform",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "query",+ query: {+ "source-table": tableId,+ },+ },+ },+ target: {+ type: "table",+ name: "base_transform",+ schema: "public",+ database: WRITABLE_DB_ID,+ },+ },+ { wrapId: true },+ );++ H.createTransform({+ name: "Name transform",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "native",+ native: {+ query: "SELECT name FROM base_transform",+ "template-tags": {},+ },+ },+ },+ target: {+ type: "table",+ name: "name_transform",+ schema: "public",+ database: WRITABLE_DB_ID,+ },+ });++ H.createTransform({+ name: "Score transform",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "native",+ native: {+ query: "SELECT score FROM base_transform",+ "template-tags": {},+ },+ },+ },+ target: {+ type: "table",+ name: "score_transform",+ schema: "public",+ database: WRITABLE_DB_ID,+ },+ });+ },+ );+}++function createSnippetWithDependentQuestionsAndTransforms() {+ H.createSnippet({+ name: "ScoreSnippet",+ content: "score = 1",+ }).then(({ body: snippet }) => {+ cy.wrap(snippet.id).as("snippetId");++ H.createNativeQuestion({+ name: "Question with snippet",+ database: WRITABLE_DB_ID,+ native: {+ query:+ 'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',+ "template-tags": {+ "snippet:ScoreSnippet": {+ id: "4b77cc1f-ea70-4ef6-84db-58432fce6928",+ name: "snippet:ScoreSnippet",+ "display-name": "snippet:ScoreSnippet",+ type: "snippet",+ "snippet-id": snippet.id,+ "snippet-name": snippet.name,+ },+ },+ },+ });++ H.createTransform({+ name: "Transform with snippet",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "native",+ native: {+ query:+ 'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',+ "template-tags": {+ "snippet:ScoreSnippet": {+ id: "4b77cc1f-ea70-4ef6-84db-58432fce6929",+ name: "snippet:ScoreSnippet",+ "display-name": "snippet:ScoreSnippet",+ type: "snippet",+ "snippet-id": snippet.id,+ "snippet-name": snippet.name,+ },+ },+ },+ },+ },+ target: {+ type: "table",+ name: "transform_with_snippet",+ schema: "public",+ database: WRITABLE_DB_ID,+ },+ });+ });+}++function startEditingSnippet() {+ H.startNewNativeQuestion();+ cy.findByTestId("native-query-editor-action-buttons").icon("snippet").click();+ cy.findByTestId("sidebar-content").within(() => {+ cy.icon("chevrondown").click({ force: true });+ cy.button(/Edit/).click();+ });+}++function confirmDiscardChanges() {+ H.modal().should("have.length", 2).last().button("Discard changes").click();+}diff --git a/enterprise/backend/src/metabase_enterprise/api/routes.clj b/enterprise/backend/src/metabase_enterprise/api/routes.cljindex 18a010c7264..b5a056623f0 100644--- a/enterprise/backend/src/metabase_enterprise/api/routes.clj+++ b/enterprise/backend/src/metabase_enterprise/api/routes.clj@@ -20,6 +20,7 @@[metabase-enterprise.content-verification.api.routes][metabase-enterprise.database-replication.api :as database-replication.api][metabase-enterprise.database-routing.api]+ [metabase-enterprise.dependencies.api][metabase-enterprise.documents.api][metabase-enterprise.email.api][metabase-enterprise.embedding-hub.api]@@ -52,6 +53,7 @@:ai-entity-analysis (deferred-tru "AI Entity Analysis"):collection-cleanup (deferred-tru "Collection Cleanup"):content-translation (deferred-tru "Content translation")+ :dependencies (deferred-tru "Dependency Tracking"):documents (deferred-tru "Documents"):embedding (deferred-tru "Embedding"):etl-connections (deferred-tru "ETL Connections")@@ -104,6 +106,7 @@(premium-handler :etl-connections)(premium-handler :etl-connections-pg))"/database-routing" (premium-handler metabase-enterprise.database-routing.api/routes :database-routing)+ "/dependencies" (premium-handler metabase-enterprise.dependencies.api/routes :dependencies)"/document" (premium-handler metabase-enterprise.documents.api/routes :documents)"/email" (premium-handler metabase-enterprise.email.api/routes :cloud-custom-smtp)"/embedding-hub" (premium-handler metabase-enterprise.embedding-hub.api/routes :embedding)diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex ac1b618e720..db80c8986fc 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -12,6 +12,7 @@[metabase-enterprise.cache.init][metabase-enterprise.comments.init][metabase-enterprise.database-replication.init]+ [metabase-enterprise.dependencies.init][metabase-enterprise.documents.init][metabase-enterprise.gsheets.init][metabase-enterprise.llm.init]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljnew file mode 100644index 00000000000..4ddf2874033--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -0,0 +1,124 @@+(ns metabase-enterprise.dependencies.api+ (:require+ [medley.core :as m]+ [metabase-enterprise.dependencies.core :as dependencies]+ [metabase.analyze.core :as analyze]+ [metabase.api.macros :as api.macros]+ [metabase.api.routes.common :refer [+auth]]+ [metabase.api.util.handlers :as handlers]+ [metabase.collections.models.collection.root :as collection.root]+ [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.models.interface :as mi]+ [metabase.native-query-snippets.core :as native-query-snippets]+ [metabase.queries.schema :as queries.schema]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.log :as log]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]+ [toucan2.core :as t2]))++(mr/def ::card-body+ [:map+ [:id {:optional false} ms/PositiveInt]+ [:dataset_query {:optional true} [:maybe ms/Map]]+ [:type {:optional true} [:maybe ::queries.schema/card-type]]+ [:result_metadata {:optional true} [:maybe analyze/ResultsMetadata]]])++(defn- broken-cards-response+ [{:keys [card transform]}]+ (let [broken-card-ids (keys card)+ broken-cards (when (seq broken-card-ids)+ (-> (t2/select :model/Card :id [:in broken-card-ids])+ (t2/hydrate [:collection :effective_ancestors] :dashboard)))+ broken-transform-ids (keys transform)+ broken-transforms (when (seq broken-transform-ids)+ (t2/select :model/Transform :id [:in broken-transform-ids]))]+ {:success (and (empty? broken-card-ids)+ (empty? broken-transform-ids))+ :bad_cards (into [] (comp (filter (fn [card]+ (if (mi/can-read? card)+ card+ (do (log/warnf "Eliding broken card %d - not readable by the user" (:id card))+ nil))))+ (map (fn [card]+ (-> card+ collection.root/hydrate-root-collection+ (update :dashboard #(some-> % (select-keys [:id :name])))))))+ broken-cards)+ :bad_transforms (into [] broken-transforms)}))++(api.macros/defendpoint :post "/check_card"+ "Check a proposed edit to a card, and return the card IDs for those cards this edit will break."+ [_route-params+ _query-params+ body :- ::card-body]+ (let [database-id (-> body :dataset_query :database)+ base-provider (lib-be.metadata.jvm/application-database-metadata-provider database-id)+ original (lib.metadata/card base-provider (:id body))+ card (-> original+ (assoc :dataset-query (:dataset_query body)+ :type (:type body (:type original)))+ ;; Remove the old `:result-metadata` from the card, it's likely wrong now.+ (dissoc :result-metadata)+ ;; But if the request includes `:result_metadata`, use that. It may be from a native card+ ;; that's been run before saving the card.+ (cond-> #_card+ (:result_metadata body) (assoc :result-metadata (:result_metadata body))))+ edits {:card [card]}+ breakages (dependencies/errors-from-proposed-edits base-provider edits)]+ (broken-cards-response breakages)))++(mr/def ::transform-body+ [:map+ [:id {:optional false} ms/PositiveInt]+ [:name {:optional true} :string]+ [:source {:optional true} [:maybe ms/Map]]+ [:target {:optional true} [:maybe ms/Map]]])++(api.macros/defendpoint :post "/check_transform"+ "Check a proposed edit to a transform, and return the card, transform, etc. IDs for things that will break."+ [_route-params+ _query-params+ {:keys [id source target] :as _body} :- ::transform-body]+ (if (= (keyword (:type source))+ :query)+ (let [database-id (-> source :query :database)+ base-provider (lib-be.metadata.jvm/application-database-metadata-provider database-id)+ original (lib.metadata/transform base-provider id)+ transform (-> original+ (cond-> #_transform source (assoc :source source))+ (cond-> #_transform target (assoc :target target)))+ edits {:transform [transform]}+ breakages (dependencies/errors-from-proposed-edits base-provider edits)]+ (broken-cards-response breakages))+ ;; if this isn't a sql query, just claim it works+ {:success true}))++(api.macros/defendpoint :post "/check_snippet"+ "Check a proposed edit to a native snippet, and return the cards, etc. which will be broken."+ [_route-params+ _query-params+ {:keys [id content], snippet-name :name}+ :- [:map+ [:id {:optional false} ms/PositiveInt]+ [:name {:optional true} native-query-snippets/NativeQuerySnippetName]+ [:content {:optional true} :string]]]+ (let [original (t2/select-one :model/NativeQuerySnippet id)+ _ (when (and snippet-name+ (not= snippet-name (:name original))+ (t2/exists? :model/NativeQuerySnippet :name snippet-name))+ (throw (ex-info (tru "A snippet with that name already exists. Please pick a different name.")+ {:status-code 400})))+ snippet (cond-> (m/assoc-some original+ :lib/type :metadata/native-query-snippet+ :name snippet-name+ :content content)+ content native-query-snippets/add-template-tags)+ breakages (dependencies/errors-from-proposed-edits {:snippet [snippet]})]+ (broken-cards-response breakages)))++(def ^{:arglists '([request respond raise])} routes+ "`/api/ee/dependencies` routes."+ (handlers/routes+ (api.macros/ns-handler *ns* +auth)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljnew file mode 100644index 00000000000..085f9936c03--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -0,0 +1,66 @@+(ns metabase-enterprise.dependencies.calculation+ (:require+ [metabase-enterprise.dependencies.native-validation :as deps.native]+ [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.util :as lib.util]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++(defn- upstream-deps:mbql-query [legacy-query]+ (lib.util/source-tables-and-cards [legacy-query]))++(defn- upstream-deps:native-query [metadata-provider query]+ (let [engine (:engine (lib.metadata/database metadata-provider))+ deps (deps.native/native-query-deps engine metadata-provider query)]+ ;; The deps are in #{{:table 7} ...} form and need conversion to ::upstream-deps form.+ (u/group-by ffirst (comp second first) conj #{} deps)))++(mr/def ::upstream-deps+ [:map+ [:card {:optional true} [:set ::lib.schema.id/card]]+ [:table {:optional true} [:set ::lib.schema.id/table]]+ [:snippet {:optional true} [:set ::lib.schema.id/snippet]]+ [:transform {:optional true} [:set ::lib.schema.id/transform]]])++(defn- upstream-deps:query [metadata-provider legacy-query]+ (case (:type legacy-query)+ (:query "query") (upstream-deps:mbql-query legacy-query)+ (:native "native") (upstream-deps:native-query metadata-provider legacy-query)+ (throw (ex-info "Unhandled kind of query" {:query legacy-query}))))++(mu/defn upstream-deps:card :- ::upstream-deps+ "Given a Toucan `:model/Card`, return its upstream dependencies as a map from the kind to a set of IDs."+ ([card] (upstream-deps:card (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))+ card))+ ([metadata-provider :- ::lib.schema.metadata/metadata-provider+ {query :dataset_query :as _toucan-card}]+ (upstream-deps:query metadata-provider query)))++(mu/defn upstream-deps:transform :- ::upstream-deps+ "Given a Transform (in Toucan form), return its upstream dependencies."+ ([transform] (upstream-deps:transform nil transform))+ ([metadata-provider :- [:maybe ::lib.schema.metadata/metadata-provider]+ {{:keys [query type]} :source :as transform}]+ (if (#{"query" :query} type)+ (upstream-deps:query (or metadata-provider+ (lib-be.metadata.jvm/application-database-metadata-provider (:database query)))+ query)+ (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) type)+ {}))))++(mu/defn upstream-deps:snippet :- ::upstream-deps+ "Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."+ [{:keys [template_tags] :as _snippet}]+ (let [type->id-key {:card :card-id, :snippet :snippet-id}+ dependencies (keep (fn [tag]+ (let [entity-type (:type tag)]+ (when-let [id-key (type->id-key entity-type)]+ (when-let [entity-id (id-key tag)]+ [entity-type entity-id]))))+ (vals template_tags))]+ (u/group-by first second conj #{} dependencies)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj b/enterprise/backend/src/metabase_enterprise/dependencies/core.cljnew file mode 100644index 00000000000..de034ba06fd--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj@@ -0,0 +1,205 @@+(ns metabase-enterprise.dependencies.core+ "API namespace for the `metabase-enterprise.dependencies` module.++ Call [[errors-from-proposed-edits]] to find out what things will break downstream of a set of new/updated entities."+ (:require+ [metabase-enterprise.dependencies.metadata-provider :as deps.provider]+ [metabase-enterprise.dependencies.models.dependency :as deps.graph]+ [metabase-enterprise.dependencies.native-validation :as deps.native]+ [metabase.graph.core :as graph]+ [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2]))++(mr/def ::entity-type+ [:enum :card :transform :snippet :table])++(mr/def ::updates-map+ ;; TODO: Make this more specific.+ [:map-of ::entity-type [:sequential [:map [:id {:optional true} :int]]]])++(mu/defn- metadata-provider :- ::lib.schema.metadata/metadata-provider+ "Constructs a `MetadataProvider` with some pending edits applied.++ The edits are *transitive*! Since a card's output columns can depend on its (possibly updated) inputs, all dependents+ of editing things are considered edited too.++ Note that if `:result-metadata` is present on an updated Card, it **will be used!** The only case where that is+ actually useful is with a native query which has been executed, so the caller has driver metadata to give us. Any+ old, pre-update `:result-metadata` should be dropped from any other cards in `updated-entities`."+ ([base-provider :- ::lib.schema.metadata/metadata-provider+ updated-entities :- ::updates-map+ & {:keys [graph dependents]}]+ (let [dependents (or dependents (deps.graph/transitive-dependents graph updated-entities))]+ (deps.provider/override-metadata-provider base-provider updated-entities dependents))))++(defmulti check-query-inner+ "Given a `MetadataProvider` and a lib query, find any bad refs in that query."+ {:arglists '([metadata-provider driver query])}+ (fn [_mp _driver query]+ (-> (lib/query-stage query 0)+ :lib/type)))++(defmethod check-query-inner :mbql.stage/mbql+ [_mp _driver query]+ (lib/find-bad-refs query))++(defmethod check-query-inner :mbql.stage/native+ [mp driver query]+ (deps.native/validate-native-query driver mp query))++(defn- check-query+ "Given a `MetadataProvider` and a query, find any bad refs in that query."+ [mp driver query]+ (check-query-inner mp driver (lib/query mp query)))++(defmulti check-entity+ "Given a `MetadataProvider`, and entity type, and an entity id, find any bad refs in that entity."+ {:arglists '([metadata-provider entity-type entity-id])}+ (fn [_mp entity-type _entity-id]+ entity-type))++(defmethod check-entity :default+ [_mp _entity-type _entity-id]+ nil)++(defmethod check-entity :card+ [mp _entity-type entity-id]+ (let [query (:dataset-query (lib.metadata/card mp entity-id))+ driver (:engine (lib.metadata/database mp))]+ (check-query mp driver query)))++(defmethod check-entity :transform+ [mp _entity-type entity-id]+ (let [{{target-schema :schema target-name :name} :target+ {:keys [query]} :source+ :as _transform} (lib.metadata/transform mp entity-id)+ driver (:engine (lib.metadata/database mp))+ output-table (some #(when (and (= (:schema %) target-schema)+ (= (:name %) target-name))+ %)+ (lib.metadata/tables mp))+ output-fields (lib.metadata/active-fields mp (:id output-table))+ {:keys [duplicate-fields]} (reduce (fn [{:keys [seen duplicate-fields]}+ {name :name :as field}]+ (if (seen name)+ {:seen seen+ :duplicate-fields (conj duplicate-fields field)}+ {:seen (conj seen name)+ :duplicate-fields duplicate-fields}))+ {:seen #{}}+ output-fields)]+ (into duplicate-fields (check-query mp driver query))))++(mu/defn- check-query-soundness ;; :- [:map-of ::lib.schema.id/card [:sequential ::lib.schema.mbql-clause/clause]]+ "Given a `MetadataProvider` as returned by [[metadata-provider]], scan all its updated entities and their dependents+ to check that everything is still sound.++ In particular, we're looking for bad clauses with refs to columns which no longer exist, since those queries fail+ to compile.++ May return false positives: if a card already has a bad ref in it, then it will be returned even if that bad ref has+ nothing to do with the updates in the `provider`.++ Returns a map `{:card {card-id [bad-ref ...]}, :transform {...}}`. It will be empty, if there are no bad refs+ detected."+ [provider :- ::lib.schema.metadata/metadata-provider]+ (let [overrides (deps.provider/all-overrides provider)+ errors (volatile! {})]+ (doseq [[entity-type ids] overrides+ id ids+ :let [bad-refs (check-entity provider entity-type id)]]+ (when (seq bad-refs)+ (vswap! errors assoc-in [entity-type id] bad-refs)))+ @errors))++(defn- group-by-db [deps]+ (let [by-db (volatile! {})]+ (when (seq (:card deps))+ (doseq [[db-id card-ids] (->> (t2/select [:model/Card :id :database_id :card_schema] :id [:in (:card deps)])+ (u/group-by :database_id :id conj #{}))]+ (vswap! by-db assoc-in [db-id :card] card-ids)))+ (when (seq (:table deps))+ (doseq [[db-id table-ids] (->> (t2/select [:model/Table :id :db_id] :id [:in (:table deps)])+ (u/group-by :db_id :id conj #{}))]+ (vswap! by-db assoc-in [db-id :table] table-ids)))+ (when (seq (:transform deps))+ (doseq [[db-id transform-ids] (->> (t2/select [:model/Transform :id :source] :id [:in (:transform deps)])+ (u/group-by #(get-in % [:source :query :database]) :id conj #{}))]+ (if db-id+ (vswap! by-db assoc-in [db-id :transform] transform-ids)+ (log/warnf "Unable to infer database from transforms %s" transform-ids))))+ (when (seq (:snippet deps))+ ;; Copy any snippet deps to each database, since they span them all.+ (vswap! by-db update-vals #(assoc % :snippet (:snippet deps))))++ @by-db))++;; TODO: (Braden 09/22/2025) More precise schemas for the errors this function returns. Currently they're pretty+;; opaque, since the consumers of this function really care about "working/broken" and not the details of what's wrong+;; with any particular card.+(mu/defn errors-from-proposed-edits :- [:map-of ::entity-type [:map-of :int [:or :boolean [:sequential :any]]]]+ "Given a regular `MetadataProvider`, and a map of entity types (`:card`, `:transform`, `:snippet`) to lists of+ updated entities, this returns a map of `{entity-type {entity-id [bad-ref ...]}}`.++ The 1-arity groups all the dependents by which Database they are part of, and runs the analysis for each of them.++ The output is a map: `{entity-type {id [errors...]}}`; an empty map is returned when there are no errors+ detected."+ ([edits :- ::updates-map]+ (let [all-deps (deps.graph/transitive-dependents edits)+ by-db (group-by-db all-deps)]+ (reduce (fn [errors [db-id deps]]+ (-> (lib-be.metadata.jvm/application-database-metadata-provider db-id)+ (metadata-provider edits :dependents deps)+ check-query-soundness+ (merge errors)))+ {} by-db)))++ ([base-provider :- ::lib.schema.metadata/metadata-provider+ edits :- ::updates-map]+ (errors-from-proposed-edits base-provider nil edits))++ ([base-provider :- ::lib.schema.metadata/metadata-provider+ graph :- [:maybe ::graph/graph]+ edits :- ::updates-map]+ (-> base-provider+ (metadata-provider edits :graph graph)+ check-query-soundness)))++#_{:clj-kondo/ignore [:unresolved-namespace]}+(comment+ ;; This should work on any fresh-ish Metabase instance; these are the built-in example questions.+ (let [base-mp (lib-be.metadata.jvm/application-database-metadata-provider 39)+ transform (lib.metadata/transform base-mp 1)+ transform (-> transform+ (update-in [:source :query :query :expressions]+ ;; Replacing the expression Age with Duration - this breaks downstream uses!+ update-keys (constantly "Fair Die"))+ #_(dissoc :result-metadata))+ mp (metadata-provider base-mp {:transform [transform]})]+ #_(deps.provider/all-overrides mp)+ (check-query-soundness mp))++ ;; Checking that we can properly group everything by database. Surprisingly fast even fetching everything.+ (let [deps {:card (t2/select-fn-set :id :model/Card)+ :transform (t2/select-fn-set :id :model/Transform)+ :table (t2/select-fn-set :id :model/Table)}]+ (group-by-db deps))++ (t2/select-fn-vec (juxt :id :name :active :table_id) :model/Field :id [:in [1065 1066 1067 1068 1069]])++ (t2/select-one :model/Card :id 124)+ (deps.graph/transitive-dependents {:transform [{:id 1}]})+ (deps.graph/transitive-dependents {:table [{:id 155}]})+ (metabase.premium-features.core/token-features)++ (let [card (t2/select-one :model/Card :id 121)+ mp (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))]+ (upstream-deps:card mp card)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj b/enterprise/backend/src/metabase_enterprise/dependencies/init.cljnew file mode 100644index 00000000000..51eb03b6482--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj@@ -0,0 +1,4 @@+(ns metabase-enterprise.dependencies.init+ (:require+ [metabase-enterprise.dependencies.models.dependency]+ [metabase-enterprise.dependencies.task.backfill]))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.cljnew file mode 100644index 00000000000..4454ade9055--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj@@ -0,0 +1,271 @@+(ns metabase-enterprise.dependencies.metadata-provider+ "A `MetadataProvider` wrapper that understands how to update entities, and to lazily compute the new metadata for+ updated entities on demand. This is useful when analyzing a tree of dependents, which might depend on each other in+ complex ways.++ Just to illustrate, imagine I change a card A which is used by a transform B whose output table C is used by a card+ D. If I remove a column from card A, and transform B selects all columns, that field needs to disappear from C+ before we can correctly analyze card D.++ By representing the updated entities as lazy `delay`s, this `MetadataProvider` can support analysis of the+ downstream entities in any order."+ (:require+ [medley.core :as m]+ [metabase-enterprise.dependencies.native-validation :as deps.native]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [pretty.core :as pretty]))++(set! *warn-on-reflection* true)++(defn- metadatas [delegate overrides {metadata-type :lib/type+ search-name :name+ :keys [id card-id table-id]+ :as metadata-spec}]+ (let [type-overrides (get overrides metadata-type)+ metadata-keys (set (or id search-name))]+ (log/tracef "OverridingMetadataProvider request for %s" metadata-spec)+ (cond+ ;; For tables, cards, snippets, transforms, and columns, return overrides if present and looking for specific+ ;; keys and delegate if not.+ (and (#{:metadata/table :metadata/card :metadata/transform :metadata/native-query-snippet :metadata/column} metadata-type)+ (seq metadata-keys))+ (let [overrides (if (seq id)+ (select-keys type-overrides id)++ (into {} (keep #(when-let [x (deref %)]+ (when (search-name (:name x))+ [(:name x) %])))+ (vals type-overrides)))+ override-key (if (seq id) :id :name)+ overridden-keys (set (keys overrides))+ missing-keys (remove overridden-keys metadata-keys)]+ (log/tracef "Overridden keys %s; missing keys %s" (sort overridden-keys) (sort missing-keys))+ (cond-> (map deref (vals overrides))+ (seq missing-keys)+ (concat (lib.metadata.protocols/metadatas delegate {:lib/type metadata-type+ override-key (set missing-keys)}))))++ ;; if fetching all tables or transforms, get every override and everything from the delegate and then dedupe+ (and (empty? metadata-keys)+ (#{:metadata/table :metadata/transform} metadata-type))+ (as-> (vals type-overrides) entities+ (into [] (map force) entities)+ (into entities (lib.metadata.protocols/metadatas delegate metadata-spec))+ (m/distinct-by :id entities))++ ;; For columns, return overrides if they're present.+ (= metadata-type :metadata/column)+ (if-let [overridden-columns (cond+ card-id (get-in overrides [::card-columns card-id])+ table-id (get-in overrides [::table-columns table-id]))]+ @overridden-columns+ (lib.metadata.protocols/metadatas delegate metadata-spec))++ :else (throw (ex-info "Unknown :lib/metadata type for OverridingMetadataProvider" metadata-spec)))))++(declare all-overrides setup-transform! setup-transforms!)++(deftype OverridingMetadataProvider [delegate *overrides]+ lib.metadata.protocols/MetadataProvider+ (database [_this] (lib.metadata.protocols/database delegate))+ (metadatas [this metadata-spec]+ (setup-transforms! this)+ (metadatas delegate @*overrides metadata-spec))+ (setting [_this setting-key]+ (lib.metadata.protocols/setting delegate setting-key))++ ;; Cached things are direct delegation.+ lib.metadata.protocols/CachedMetadataProvider+ (cached-metadatas [_this metadata-type metadata-ids]+ (lib.metadata.protocols/cached-metadatas delegate metadata-type metadata-ids))+ (store-metadata! [_this a-metadata]+ (lib.metadata.protocols/store-metadata! delegate a-metadata))+ (cached-value [_this k not-found]+ (lib.metadata.protocols/cached-value delegate k not-found))+ (cache-value! [_this k v]+ (lib.metadata.protocols/cache-value! delegate k v))+ (has-cache? [_this]+ (lib.metadata.protocols/has-cache? delegate))++ pretty/PrettyPrintable+ (pretty [this]+ (list `overriding-metadata-provider (all-overrides this) delegate)))++(defmulti add-override+ "Given an `OverridingMetadataProvider`, an `entity-type` and that entity, add this entity as an override."+ {:arglists '([metadata-provider entity-type id updates-or-nil])}+ (fn [_mp entity-type _id _updates]+ entity-type))++(defn- inner-mp [^OverridingMetadataProvider mp]+ (.delegate mp))++(defn- with-overrides [^OverridingMetadataProvider mp kvs]+ (swap! (.*overrides mp)+ #(reduce (fn [m [ks v]]+ (assoc-in m ks v))+ % kvs)))++(defn- get-returned-columns [mp queryable]+ (let [query (lib/query mp queryable)]+ (if (= (:lib/type (lib/query-stage query 0))+ :mbql.stage/mbql)+ (lib/returned-columns query)+ (deps.native/native-result-metadata (:engine (lib.metadata/database mp))+ mp query))))++(defmethod add-override :card [^OverridingMetadataProvider mp _entity-type id updates]+ (with-overrides mp+ ;; If the `updates` contain `:result-metadata`, we want to use that. However, any `:result-metadata` from the+ ;; inner-mp should be ignored.+ {[:metadata/card id] (delay (merge (when id+ (-> (inner-mp mp)+ (lib.metadata/card id)+ (dissoc :result-metadata)))+ updates))+ ;; This uses the outer OMP and so the overrides are visible!+ [::card-columns id] (delay (get-returned-columns mp (lib.metadata/card mp id)))}))++(defonce ^:private last-fake-id (atom 2000000000))++(defn- fake-id []+ (swap! last-fake-id inc))++(defn- setup-transform!+ [^OverridingMetadataProvider mp s+n->table {:keys [source target] :as _transform}]+ ;; See if the output table exists in the underlying MetadataProvider.+ ;; - If the table doesn't exist, invent an ID for it.+ ;; - Always add an override for the table, even if it already existed.+ ;; - Compare the transform's output columns to the fields that exist in the metadata.+ ;; - For any newly added fields, invent an ID and add an override for the field.+ ;; - For any removed fields that used to exist, add an override for their ID that is nil!+ ;; - Add an internal [::table-columns table-id] with the list of output columns.+ ;; Note that this assumes that all other overrides are already loaded!+ (let [existing-table (get s+n->table [(:schema target) (:name target)])+ output-table (or existing-table+ (merge (select-keys target [:schema :name])+ {:lib/type :metadata/table+ :id (fake-id)+ :db-id (:id (lib.metadata/database (inner-mp mp)))+ :display-name (:name target)+ :visibility-type nil}))+ existing-cols (when existing-table+ (lib/returned-columns (lib/query (inner-mp mp) existing-table)))+ output-cols (delay+ ;; Note that this will analyze the query with any upstream changes included!+ (let [new-cols (get-returned-columns mp (:query source))+ by-name (m/index-by :lib/desired-column-alias existing-cols)]+ (into [] (for [col new-cols+ :let [old-col (by-name (:lib/desired-column-alias col))]]+ (merge (select-keys col [:display-name+ :base-type :effective-type :semantic-type])+ {:name (:lib/desired-column-alias col)+ :lib/type :metadata/column+ :id (or (:id old-col) (fake-id))+ :table-id (:id output-table)+ :active true})))))+ outputs-by-id (delay (m/index-by :id @output-cols))]+ ;; NOTE: Any newly added output columns can't be looked up directly by ID, with this version.+ ;; The `output-cols` code above can be adapted to add them to the `*overrides` atom, but YAGNI for now.+ (with-overrides mp+ (into {[:metadata/table (:id output-table)] (delay output-table)+ [::table-columns (:id output-table)] output-cols}+ ;; For each pre-existing output column, we need to add an override for a direct read by ID.+ ;; If the column does not exist in the output, return the original with :active false+ (map (fn [existing-col]+ [[:metadata/column (:id existing-col)]+ (delay (or (get @outputs-by-id (:id existing-col))+ (assoc existing-col :active false)))]))+ existing-cols))))++(defn- setup-transforms! [^OverridingMetadataProvider mp]+ (locking mp+ (when-not (-> (.*overrides mp) deref ::setup-complete)+ (swap! (.*overrides mp) assoc ::setup-complete true)+ (let [;; Fetching all the pre-existing tables.+ ;; TODO: Once Cam's new by-name logic in the `MetadataProvider` lands, we can use that here.+ tables (lib.metadata/tables (inner-mp mp))+ s+n->table (m/index-by (juxt :schema :name) tables)]+ (doseq [transform (-> (.*overrides mp) deref :metadata/transform vals)]+ (setup-transform! mp s+n->table (deref transform)))))))++(defmethod add-override :transform [^OverridingMetadataProvider mp _entity-type id updates]+ ;; Transforms are complicated:+ ;; 1. Treated as a card on the input side.+ ;; 2. The returned-columns of that card become the columns of the output table.+ ;; 3. The output table is constructed from the transform.+ ;; 4. Its fields are adapted from the transform's input card.+ ;; All of these can be wrapped in delays and therefore computed lazily on demand.+ ;;+ ;; The trouble is that a transform does not know the output table's ID, nor its field IDs.+ ;; A modified transform might produce a new table, or have new fields or removed fields.+ ;; The OverridingMetadataProvider has to account for all of this when it returns tables or fields by ID, or gets+ ;; the column list for a table.+ ;;+ ;; This is achieved by putting `[:metadata/transform id]` into overrides for all transforms, and then one-time+ ;; logic that runs before the first use of the OverridingMetadataProvider as a MetadataProvider will walk all the+ ;; transforms and make sure their tables etc. are overridden properly.+ (let [transform (merge (when id+ (lib.metadata/transform (inner-mp mp) id))+ updates)]+ (with-overrides mp+ {;; The transform itself, merged with the incoming updates.+ [:metadata/transform id] (delay transform)+ #_#_[::transform-columns id] (delay (->> (lib.metadata/transform mp id)+ :source+ :query+ (lib/query mp)+ lib/returned-columns))})))++(defmethod add-override :table [^OverridingMetadataProvider mp _entity-type id updates]+ (when (seq updates)+ (throw (ex-info "Updating a table directly is not supported" {:table-id id, :edited updates})))+ (with-overrides mp+ {[:metadata/table id] (delay (log/warnf "Can't happen: Table %d dep should be replaced by its transform" id)+ (lib.metadata/table (inner-mp mp) id))+ [::table-columns id] (delay+ (log/warnf "Can't happen: Table %d ::table-columns should be replaced by its transform" id)+ (lib.metadata/fields (inner-mp mp) id))}))++(defmethod add-override :snippet [^OverridingMetadataProvider mp _entity-type id updates]+ (with-overrides mp+ {[:metadata/native-query-snippet id] (delay (merge (when id+ (lib.metadata/native-query-snippet (inner-mp mp) id))+ (u/normalize-map updates)))}))++(defn all-overrides+ "Returns all the overrides by ID, in the same form as the map input to [[with-deps]]:+ `{:card [1 2 3], :transform [45 99]}."+ [^OverridingMetadataProvider override-metadata-provider]+ (-> (.*overrides override-metadata-provider)+ deref+ (dissoc ::setup-complete)+ (update-vals keys)+ (update-keys {:metadata/card :card+ :metadata/transform :transform+ :metadata/native-query-snippet :snippet})+ (select-keys [:card :transform :snippet])))++(defn override-metadata-provider+ "Given an underlying `MetadataProvider`, wraps it to support in-memory overrides of cards, fields, etc.++ Important note: all overrides must be added first, before this is used as a `MetadataProvider`.+ The easiest way to do that is to call the 3-arity, which provides the overrides up front."+ ([metadata-provider]+ (->OverridingMetadataProvider metadata-provider (atom {})))++ ([metadata-provider updated-entities dependent-ids]+ (let [^OverridingMetadataProvider omp (override-metadata-provider metadata-provider)]+ (doseq [[entity-type updates] updated-entities+ updated-entity updates]+ (add-override omp entity-type (:id updated-entity) updated-entity))+ (doseq [[entity-type dependents] dependent-ids+ :let [updated (into #{} (map :id) (get updated-entities entity-type))]+ id dependents+ :when (not (updated id))]+ (add-override omp entity-type id nil))+ omp)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljnew file mode 100644index 00000000000..d8320e6b6eb--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -0,0 +1,212 @@+(ns metabase-enterprise.dependencies.models.dependency+ (:require+ [clojure.set :as set]+ [metabase-enterprise.dependencies.calculation :as deps.calculation]+ [metabase.events.core :as events]+ [metabase.graph.core :as graph]+ [metabase.models.interface :as mi]+ [metabase.premium-features.core :as premium-features]+ [metabase.util :as u]+ [methodical.core :as methodical]+ [potemkin :as p]+ [toucan2.core :as t2]))++(def current-dependency-analysis-version+ "Current version of the dependency analysis logic.+ This should be incremented when the dependency analysis logic changes."+ 1)++(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)++(derive :model/Dependency :metabase/model)++(t2/deftransforms :model/Dependency+ {:from_entity_type mi/transform-keyword+ :to_entity_type mi/transform-keyword})++(defn- deps-children [src-type src-id dst-type dst-id key-seq]+ ;; Group all keys with the same type together, so we make O(types) indexed [[t2/select]] calls, not O(n).+ (transduce (map (fn [[entity-type entity-keys]]+ (let [deps (t2/select :model/Dependency+ src-type entity-type+ src-id [:in entity-keys])]+ (u/group-by (juxt src-type src-id)+ (juxt dst-type dst-id)+ conj #{}+ deps))))+ merge {}+ (u/group-by first second key-seq)))++(defn- key-dependents+ "Get the dependent entity keys for the entity keys in `entity-keys`.+ Entity keys are [entity-type, entity-id] pairs. See [[entity-type->model]]."+ [key-seq]+ (deps-children :to_entity_type :to_entity_id :from_entity_type :from_entity_id key-seq))++(p/deftype+ DependencyGraph [children-fn]+ graph/Graph+ (children-of [_this key-seq]+ (children-fn key-seq)))++;; NOTE: We can easily construct a graph of upstream dependencies too, if it's useful.+(defn- graph-dependents []+ (->DependencyGraph key-dependents))++(defn transitive-dependents+ "Given a map of updated entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependents+ as `{entity-type #{4 5 6}}` - that is, a map from downstream entity type to a set of IDs.++ Uses the provided `graph`, or defaults to the `:model/Dependency` table in AppDB.++ The inputs must be maps containing `:id`; anything without an `:id` is skipped. They could be Toucan entities,+ `MetadataProvider` entities, user input, etc.++ **Excludes** the input entities from the list of dependents!"+ ([updated-entities] (transitive-dependents nil updated-entities))+ ([graph updated-entities]+ (let [graph (or graph (graph-dependents))+ starters (for [[entity-type updates] updated-entities+ entity updates+ :when (:id entity)]+ [entity-type (:id entity)])]+ (->> (graph/transitive graph starters) ; This returns a flat list.+ (u/group-by first second conj #{})))))++(defn replace-dependencies+ "Replace the dependencies of the entity of type `entity-type` with id `entity-id` with+ the ones specified in `dependencies-by-type`. "+ [entity-type entity-id dependencies-by-type]+ (let [current-dependencies (t2/select [:model/Dependency :id :to_entity_type :to_entity_id]+ :from_entity_type entity-type+ :from_entity_id entity-id)+ to-remove (keep (fn [{:keys [id to_entity_type to_entity_id]}]+ (when-not (get-in dependencies-by-type [to_entity_type to_entity_id])+ id))+ current-dependencies)+ current-by-type (-> (group-by :to_entity_type current-dependencies)+ (update-vals #(into #{} (map :to_entity_id) %)))+ to-add (for [[to-entity-type ids] dependencies-by-type+ to-entity-id (set/difference ids (current-by-type to-entity-type))]+ {:from_entity_type entity-type+ :from_entity_id entity-id+ :to_entity_type to-entity-type+ :to_entity_id to-entity-id})]+ (t2/with-transaction [_conn]+ (when (seq to-remove)+ (t2/delete! :model/Dependency :id [:in to-remove]))+ (when (seq to-add)+ (t2/insert! :model/Dependency to-add)))))++;; ## Maintaining the dependency graph+;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the+;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created+;; or changed at that point.++;; ### Cards+(derive ::card-deps :metabase/event)+(derive :event/card-create ::card-deps)+(derive :event/card-update ::card-deps)++(methodical/defmethod events/publish-event! ::card-deps+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/with-transaction [_conn]+ (replace-dependencies :card (:id object) (deps.calculation/upstream-deps:card object))+ (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)+ (t2/update! :model/Card (:id object)+ {:dependency_analysis_version current-dependency-analysis-version})))))++(derive ::card-delete :metabase/event)+(derive :event/card-delete ::card-delete)++(methodical/defmethod events/publish-event! ::card-delete+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))))++;; ### Snippets+(derive ::snippet-deps :metabase/event)+(derive :event/snippet-create ::snippet-deps)+(derive :event/snippet-update ::snippet-deps)++(methodical/defmethod events/publish-event! ::snippet-deps+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/with-transaction [_conn]+ (replace-dependencies :snippet (:id object) (deps.calculation/upstream-deps:snippet object))+ (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)+ (t2/update! :model/NativeQuerySnippet (:id object)+ {:dependency_analysis_version current-dependency-analysis-version})))))++(derive ::snippet-delete :metabase/event)+(derive :event/snippet-delete ::snippet-delete)++(methodical/defmethod events/publish-event! ::snippet-delete+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))))++;; ### Transforms+(derive ::transform-deps :metabase/event)+(derive :event/create-transform ::transform-deps)+(derive :event/update-transform ::transform-deps)++;; On *saving* a transform, the upstream deps of its query are computed and saved.+(defn- drop-outdated-target-dep! [{:keys [id source target] :as _transform}]+ (let [db-id (some-> source :query :database)+ downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency+ :from_entity_type :table+ :to_entity_type :transform+ :to_entity_id id)+ downstream-tables (when (seq downstream-table-ids)+ (t2/select :model/Table :id [:in downstream-table-ids]))+ outdated-tables (remove (fn [table]+ (and (= (:schema table) (:schema target))+ (= (:name table) (:name target))+ (or (not db-id)+ (= db-id (:db_id table)))))+ downstream-tables)+ not-found-table-ids (remove (into #{} (map :id) downstream-tables)+ downstream-table-ids)]+ (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)+ (map :id) outdated-tables))]+ (t2/delete! :model/Dependency+ :from_entity_type :table+ :from_entity_id [:in outdated-downstream-table-ids]+ :to_entity_type :transform+ :to_entity_id id))))++(methodical/defmethod events/publish-event! ::transform-deps+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/with-transaction [_conn]+ (replace-dependencies :transform (:id object) (deps.calculation/upstream-deps:transform object))+ (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)+ (t2/update! :model/Transform (:id object) {:dependency_analysis_version current-dependency-analysis-version}))+ (drop-outdated-target-dep! object))))++(derive ::transform-delete :metabase/event)+(derive :event/delete-transform ::transform-delete)++(methodical/defmethod events/publish-event! ::transform-delete+ [_ {:keys [id]}]+ (when (premium-features/has-feature? :dependencies)+ ;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?+ (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)))++;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.+;; (And if the target has changed, the old table's dep on the transform is dropped.)+;; The upstream deps of the transform are not touched - those change only when the transform is edited.+(derive ::transform-run :metabase/event)+(derive :event/transform-run-complete ::transform-run)++(defn- transform-table-deps! [{:keys [db-id output-schema output-table transform-id] :as _details}]+ (let [;; output-table is a keyword like :my_schema/my_table+ table-name (name output-table)]+ (when-let [table-id (t2/select-one-fn :id :model/Table :db_id db-id :schema output-schema :name table-name)]+ (replace-dependencies :table table-id {:transform #{transform-id}}))))++(methodical/defmethod events/publish-event! ::transform-run+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (transform-table-deps! object)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.cljnew file mode 100644index 00000000000..51291a74390--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj@@ -0,0 +1,38 @@+(ns metabase-enterprise.dependencies.native-validation+ (:require+ [metabase.driver :as driver]+ [metabase.lib.core :as lib]+ [metabase.query-processor :as qp]))++(defn- compile-query [metadata-provider query]+ (->> (lib/query metadata-provider query)+ lib/add-parameters-for-template-tags+ (qp/compile-query-with-metadata-provider metadata-provider)+ :query))++(defn validate-native-query+ "Compiles a (native) query and validates that the fields and tables it refers to really exist.++ Returns either nil or a list of errors."+ [driver metadata-provider query]+ (->> (compile-query metadata-provider query)+ (driver/validate-native-query-fields driver metadata-provider)))++(defn native-result-metadata+ "Compiles a (native) query and calculates its result metadata"+ [driver metadata-provider query]+ (->> (compile-query metadata-provider query)+ (driver/native-result-metadata driver metadata-provider)))++(defn native-query-deps+ "Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs."+ [driver metadata-provider query]+ (let [compiled (compile-query metadata-provider query)]+ (into (driver/native-query-deps driver compiled metadata-provider)+ (keep #(case (:type %)+ "snippet" {:snippet (:snippet-id %)}+ :snippet {:snippet (:snippet-id %)}+ "card" {:card (:card-id %)}+ :card {:card (:card-id %)}+ nil))+ (-> query :native :template-tags vals))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.cljnew file mode 100644index 00000000000..09a19f3c038--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj@@ -0,0 +1,191 @@+(ns metabase-enterprise.dependencies.task.backfill+ "Implements a task that brings all entities with dependencies tracked in the dependency table+ (see [[metabase-enterprise.dependencies.models.dependency]]) up to date, that is, makes sure+ the dependency table contains fresh entries.++ This is done by finding entities whose dependency_analysis_version is less than+ [[metabase-enterprise.dependencies.models.dependency/current-dependency-analysis-version]] and+ updates this field. In most cases this triggers an event handler that calculates the dependencies+ and populates the dependency table. For cards, the event is emitted by the job handler itself,+ because the update of the record doesn't trigger the event handler. "+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.scheduler :as qs]+ [clojurewerkz.quartzite.triggers :as triggers]+ [environ.core :as env]+ [java-time.api :as t]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase.config.core :as config]+ [metabase.events.core :as events]+ [metabase.premium-features.core :as premium-features]+ [metabase.task.core :as task]+ [metabase.util.log :as log]+ [toucan2.core :as t2])+ (:import+ (java.util Map Set)+ (java.util.concurrent ConcurrentHashMap)))++(set! *warn-on-reflection* true)++(defn- current-millis+ "Returns the current epoch millis. Uses java-time so that clock settings can be used in tests."+ []+ (t/to-millis-from-epoch (t/instant)))++(def ^:private entities [:model/Card :model/Transform :model/NativeQuerySnippet])++;; In-memory state for tracking failed entities+;; Stores {:model/Type {id {:fail-count N :next-retry-timestamp M}}}+(def ^:private retry-state+ (zipmap entities (repeatedly #(ConcurrentHashMap.))))++;; Stores {:model/Type #{id1 id2 ...}} for entities that have exceeded MAX_RETRIES+(def ^:private terminally-broken+ (zipmap entities (repeatedly #(ConcurrentHashMap/newKeySet))))++(def ^:private MAX_RETRIES 5)++(defn- long-env-value+ [env-var-kw default]+ (or (parse-long (env/env env-var-kw "")) default))++(defn- get-batch-size []+ (long-env-value :mb-dependency-backfill-batch-size 20))++(defn- get-delay-minutes []+ (long-env-value :mb-dependency-backfill-delay-minutes 60))++(defn- get-variance-minutes []+ (long-env-value :mb-dependency-backfill-variance-minutes 10))++(defn- get-processable-ids [model-kw batch-size]+ (let [target-version models.dependency/current-dependency-analysis-version+ terminally-broken-ids ^Set (terminally-broken model-kw)+ retry-state-map ^Map (retry-state model-kw)+ current-time (current-millis)]+ (into []+ (comp+ (map :id)+ (filter (fn [id]+ (and (not (.contains terminally-broken-ids id))+ (let [entity-retry-info (.get retry-state-map id)]+ (or (nil? entity-retry-info)+ (>= current-time (:next-retry-timestamp entity-retry-info)))))))+ (take batch-size))+ (t2/reducible-select [model-kw :id] :dependency_analysis_version [:< target-version]))))++(defn- backfill-card!+ [id target-version]+ ;; We don't want to change the card at all, we just want to update the dependency data and+ ;; mark the card as processed for this dependency analysis version.+ (let [update-count (t2/update! :model/Card id :dependency_analysis_version [:< target-version]+ {:dependency_analysis_version target-version})]+ (when-let [card (and (pos? update-count)+ (t2/select-one :model/Card id))]+ (events/publish-event! :event/card-update {:object card :user-id config/internal-mb-user-id}))+ update-count))++(defn- backfill-entity!+ [model-kw id target-version]+ (t2/with-transaction [_]+ (case model-kw+ :model/Card (backfill-card! id target-version)+ (t2/update! model-kw id :dependency_analysis_version [:< target-version]+ {:dependency_analysis_version target-version}))))++(defn- backfill-entity-batch!+ [model-kw batch-size]+ (let [model-name (name model-kw)+ target-version models.dependency/current-dependency-analysis-version+ retry-state-map ^Map (retry-state model-kw)+ terminally-broken-set ^Set (terminally-broken model-kw)+ ids (get-processable-ids model-kw batch-size)] ; Use the new get-processable-ids+ (when (seq ids)+ (log/infof "Processing a batch of %s %s(s)..." (count ids) model-name))+ (reduce (fn [total id]+ (+ total+ (try+ ;; this should update the dependency table via a toucan2 update hook+ (let [update-count (backfill-entity! model-kw id target-version)]+ (.remove retry-state-map id)+ update-count)+ (catch Exception e+ (let [current-time (current-millis)+ entity-retry-info (.get retry-state-map id)+ failure-count (inc (:fail-count entity-retry-info 0))+ retry-minutes (* failure-count (get-delay-minutes))+ new-next-retry-timestamp (+ current-time (* retry-minutes 60 1000))]+ (if (> failure-count MAX_RETRIES)+ (do (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."+ model-name id failure-count)+ (.add terminally-broken-set id)+ (.remove retry-state-map id)) ; Remove from retry map+ (do (log/warnf e "Entity %s %s failed, failure count: %d, next retry no sooner than %d minutes."+ model-name id failure-count retry-minutes)+ (.put retry-state-map id {:fail-count failure-count+ :next-retry-timestamp new-next-retry-timestamp}))))+ 0))))+ 0+ ids)))++(defn- backfill-dependencies+ "Job to backfill dependencies for all entities.+ Returns true if a full batch has been selected, nil or false otherwise."+ []+ (when (premium-features/has-feature? :dependencies)+ (-> (reduce (fn [batch-size model-kw]+ (if (< batch-size 1)+ (reduced 0)+ (let [processed (backfill-entity-batch! model-kw batch-size)]+ (when (pos? processed)+ (log/info "Updated" processed "entities."))+ (- batch-size processed))))+ (get-batch-size)+ entities)+ (< 1))))++(defn- has-pending-retries? []+ (some (fn [^Map model-retry-state]+ (not (.isEmpty model-retry-state)))+ (vals retry-state)))++(declare schedule-next-run!)++(task/defjob+ ^{:doc "Backfill the dependency table."+ org.quartz.DisallowConcurrentExecution true}+ BackfillDependencies [ctx]+ (log/info "Executing BackfillDependencies job...")+ (let [full-batch-selected? (backfill-dependencies)+ retries? (has-pending-retries?)]+ (if (or full-batch-selected?+ retries?)+ (let [delay-seconds (* (get-delay-minutes) 60)+ variance-seconds (* (get-variance-minutes) 60)+ delay-in-seconds (max 0 (+ (- delay-seconds variance-seconds) (rand-int (* 2 variance-seconds))))]+ (schedule-next-run! delay-in-seconds (.getScheduler ctx)))+ (log/info "No more entities to backfill for, stopping."))))++(def ^:private job-key "metabase.task.dependency-backfill.job")+(def ^:private trigger-key "metabase.task.dependency-backfill.trigger")++(defn- schedule-next-run!+ ([delay-in-seconds] (schedule-next-run! delay-in-seconds nil))+ ([delay-in-seconds scheduler]+ (let [start-at (java.util.Date. (long (+ (current-millis) (* delay-in-seconds 1000))))+ trigger (triggers/build+ (triggers/with-identity (triggers/key (str trigger-key \. (random-uuid))))+ (triggers/for-job job-key)+ (triggers/start-at start-at))]+ (log/info "Scheduling next run at" start-at)+ (if scheduler+ ;; re-scheduling from the job+ (qs/add-trigger scheduler trigger)+ ;; first schedule+ (let [job (jobs/build (jobs/of-type BackfillDependencies) (jobs/with-identity job-key))]+ (task/schedule-task! job trigger))))))++(defmethod task/init! ::DependencyBackfill [_]+ (if (pos? (get-batch-size))+ (schedule-next-run! (rand-int (* (get-variance-minutes) 60)))+ (log/info "Not starting dependency backfill job because the batch size is not positive")))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex d7572128a94..802eb335af7 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -65,6 +65,7 @@"DashboardBookmark""DataPermissions""DatabaseRouter"+ "Dependency""DocumentBookmark""HTTPAction""ImplicitAction"diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.cljindex bfc527b5349..fa920c6bbae 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj@@ -4,6 +4,7 @@[metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver][metabase.driver.util :as driver.u]+ [metabase.events.core :as events][metabase.lib.schema.common :as schema.common][metabase.util.log :as log][metabase.util.malli.registry :as mr]@@ -35,6 +36,7 @@{driver :engine :as database} (t2/select-one :model/Database db)feature (transforms.util/required-database-feature transform)transform-details {:db-id db+ :transform-id id:transform-type (keyword (:type target)):conn-spec (driver/connection-spec driver database):query (transforms.util/compile-source source)@@ -56,7 +58,9 @@run-id driver transform-details(fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))(transforms.instrumentation/with-stage-timing [run-id :table-sync]- (transforms.util/sync-target! target database run-id))))+ (transforms.util/sync-target! target database run-id)+ ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.+ (events/publish-event! :event/transform-run-complete {:object transform-details}))))(catch Throwable t(log/error t "Error executing transform")(when start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.cljindex 8b5934873c8..7308dceed70 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj@@ -3,6 +3,7 @@[clojure.set :as set][medley.core :as m][metabase-enterprise.transforms.models.transform-run :as transform-run]+ [metabase.events.core :as events][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.util :as u]@@ -17,7 +18,7 @@(derive :model/Transform trait))(t2/deftransforms :model/Transform- {:source mi/transform-json+ {:source mi/transform-transform-source:target mi/transform-json:run_trigger mi/transform-keyword})@@ -61,6 +62,18 @@(for [transform transforms](assoc transform :tag_ids (vec (get transform-id->tag-ids (:id transform) [])))))))+(t2/define-after-insert :model/Transform [transform]+ (events/publish-event! :event/create-transform {:object transform})+ transform)++(t2/define-after-update :model/Transform [transform]+ (events/publish-event! :event/update-transform {:object transform})+ transform)++(t2/define-before-delete :model/Transform [transform]+ (events/publish-event! :event/delete-transform {:id (:id transform)})+ transform)+(defn update-transform-tags!"Update the tags associated with a transform using smart diff logic.Only modifies what has changed: deletes removed tags, updates positions for moved tags,@@ -134,7 +147,7 @@(defmethod serdes/make-spec "Transform"[_model-name opts]{:copy [:name :description :entity_id]- :skip []+ :skip [:dependency_analysis_version]:transform {:created_at (serdes/date):updated_at (serdes/date):source {:export serdes/export-mbql :import serdes/import-mbql}diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex 8b616b41756..301813f16da 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -176,8 +176,8 @@(defn compile-source"Compile the source query of a transform."[{query-type :type :as source}]- (case query-type- "query" (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))+ (case (keyword query-type)+ :query (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))(defn required-database-feature"Returns the database feature necessary to execute `transform`."diff --git a/enterprise/backend/test/metabase_enterprise/api/session_test.clj b/enterprise/backend/test/metabase_enterprise/api/session_test.cljindex e33028e8e31..344dfd0de24 100644--- a/enterprise/backend/test/metabase_enterprise/api/session_test.clj+++ b/enterprise/backend/test/metabase_enterprise/api/session_test.clj@@ -99,5 +99,6 @@:database_routing true:cloud_custom_smtp true:etl_connections false- :etl_connections_pg false}+ :etl_connections_pg false+ :dependencies false}(:token-features (mt/user-http-request :crowberto :get 200 "session/properties"))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.cljnew file mode 100644index 00000000000..d761133355a--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj@@ -0,0 +1,189 @@+(ns metabase-enterprise.dependencies.core-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [medley.core :as m]+ [metabase-enterprise.dependencies.core :as dependencies]+ [metabase-enterprise.dependencies.test-util :as deps.tu]+ [metabase.graph.core :as graph]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]))++(defn- testbed+ "A `MetadataProvider` with a chain of MBQL cards and transforms for testing."+ []+ (let [mp (deps.tu/default-metadata-provider)+ query1 (-> (lib/query mp (meta/table-metadata :orders))+ (lib/expression "Tax Rate" (lib// (meta/field-metadata :orders :tax)+ (meta/field-metadata :orders :subtotal))))+ mp (lib.tu/metadata-provider-with-card-from-query mp 101 query1)+ card1 (lib.metadata/card mp 101)+ card1-query (lib/query mp card1)+ card1-cols (lib/returned-columns card1-query)+ tax-rate (m/find-first #(= (:lib/desired-column-alias %) "Tax Rate") card1-cols)+ query2 (-> card1-query+ (lib/filter (lib/> tax-rate 0.06)))+ mp (lib.tu/metadata-provider-with-card-from-query mp 102 query2)++ ;; A transform that depends on card1+ tf1-query (-> (lib/query mp card1)+ lib/->legacy-MBQL)+ transform1 {:id 30+ :name "MBQL Transform"+ :source {:query tf1-query}+ :target {:schema "Transformed"+ :name "output_tf30"}}+ tf1-output (-> (meta/table-metadata :orders)+ (assoc :id 1234567+ :schema "Transformed"+ :name "output_tf30"+ :display-name "Transform 30 Output"))+ tf1-cols (map-indexed (fn [i col]+ (assoc col+ :id (+ 123456700 i)+ :table-id 1234567+ :lib/source :source/table-defaults))+ card1-cols)+ mp (lib.tu/mock-metadata-provider mp {:tables [tf1-output]+ :fields tf1-cols+ :transforms [transform1]})++ ;; An MBQL card that consumes the output table of transform1.+ ;; References some fields but does not change the output columns.+ tf1-consumer (-> (lib/query mp (lib.metadata/table mp 1234567))+ (lib/filter (lib/= (m/find-first #(= (:name %) "Tax Rate") tf1-cols)+ 101)))+ mp (lib.tu/metadata-provider-with-card-from-query mp 301 tf1-consumer)++ ;; SQL cards and snippets+ ;; Card 7 from [[deps.tu/default-metadata-provider]] uses the "outer" snippet (2).+ ;; This adds a SQL transform that consumes card 7, and a card that selects from that transform.+ tf2-query (lib/native-query mp "SELECT * FROM {{#7}}")+ transform2 {:id 31+ :name "SQL transform"+ :source {:query (lib/->legacy-MBQL tf2-query)}+ :target {:schema "Transformed"+ :name "output_tf31"}}+ tf2-output (-> (meta/table-metadata :orders)+ (assoc :id 1234568+ :schema "Transformed"+ :name "output_tf31"+ :display-name "Transform 31 Output"))+ ;; Output columns for transform2 are derived from card 7. That's "SELECT * FROM {{snippet: outer}}";+ ;; the outer snippet references {{snippet: inner}}; the inner snippet is {{#card-ref}} to card 1;+ ;; card 1 is a basic MBQL query of products.+ card7 (-> (lib.metadata/card mp 7)+ (assoc :result-metadata (:result-metadata (:products/native (lib.tu/mock-cards)))))+ mp (lib.tu/mock-metadata-provider mp {:cards [card7]})+ tf2-cols (->> card7+ (lib/query mp)+ lib/returned-columns+ (map-indexed (fn [i col]+ (assoc col+ :id (+ 123456800 i)+ :table-id 1234568+ :lib/source :source/table-defaults))))+ mp (lib.tu/mock-metadata-provider mp {:tables [tf2-output]+ :fields tf2-cols+ :transforms [transform2]})++ ;; MBQL card consuming tf2-output.+ query3 (-> (lib/query mp tf2-output)+ (lib/filter (lib/< (m/find-first #(= (:name %) "RATING") tf2-cols)+ 3)))+ ;; SQL card consuming tf2-output.+ query4 (lib/native-query mp "SELECT * FROM Transformed.output_tf31")+ mp (-> mp+ (lib.tu/metadata-provider-with-card-from-query 303 query3)+ (lib.tu/metadata-provider-with-card-from-query 304 query4))]+ {:provider mp+ ;; NOTE: This is a *downstream* graph! So it's a map of keys to those entities which depend on them.+ :graph (graph/in-memory {[:card 101] #{[:card 102]+ [:transform 30]}+ [:transform 30] #{[:table 1234567]}+ [:table 1234567] #{[:card 301]}++ [:snippet 1] #{[:snippet 2]}+ [:snippet 2] #{[:card 7]}+ [:card 7] #{[:transform 31]}+ [:transform 31] #{[:table 1234568]}+ [:table 1234568] #{[:card 303]+ [:card 304]}})+ :mbql-base card1+ :mbql-dependent (lib.metadata/card mp 102)+ :mbql-transform transform1+ :mbql-transform-output tf1-output+ :mbql-transform-cols (m/index-by :name tf1-cols)+ :mbql-transform-consumer (lib.metadata/card mp 301)++ :snippet-inner (lib.metadata/native-query-snippet mp 1)+ :snippet-outer (lib.metadata/native-query-snippet mp 2)+ :sql-base (lib.metadata/card mp 7)+ :sql-transform transform2+ :sql-transform-output tf2-output+ :sql-transform-cols (m/index-by :name tf2-cols)+ :sql-transform-mbql-consumer (lib.metadata/card mp 303)+ :sql-transform-sql-consumer (lib.metadata/card mp 304)}))++(comment+ *e+ (:mbql-dependent (testbed))+ (:mbql-transform-consumer (testbed)))++(deftest ^:parallel basic-mbql-card-test+ (testing "when changing an MBQL card with dependents"+ (let [{:keys [provider graph mbql-base]+ {tax-rate "Tax Rate"} :mbql-transform-cols+ {downstream-card-id :id} :mbql-dependent+ {transformed-card-id :id} :mbql-transform-consumer} (testbed)]+ (testing "a column that no longer exists will cause errors when referenced"+ (let [card' (-> mbql-base+ (update-in [:dataset-query :query :expressions]+ update-keys (constantly "Sales Taxes"))+ (dissoc :result-metadata))+ errors (dependencies/errors-from-proposed-edits provider graph {:card [card']})]+ (is (=? {:card {downstream-card-id [[:field {} "Tax Rate"]]+ transformed-card-id [[:field {} (:id tax-rate)]]}}+ errors))+ (is (= [:card] (keys errors)))+ (is (= #{downstream-card-id transformed-card-id} (set (keys (:card errors)))))))++ (testing "changing something unrelated will cause no errors"+ (let [card' (-> mbql-base+ (assoc-in [:dataset-query :query :filter]+ [:> [:field (meta/id :orders :quantity) nil] 100])+ (dissoc :result-metadata))]+ (is (= {} (dependencies/errors-from-proposed-edits provider graph {:card [card']}))))))))++(deftest ^:parallel sql-snippet->card->transform->cards-test+ (testing "changing a snippet correctly finds downstream errors"+ (let [{:keys [provider graph sql-transform snippet-inner sql-transform-cols]+ {direct-sql-card-id :id} :sql-base+ {transformed-sql-card-id :id} :sql-transform-sql-consumer+ {transformed-mbql-card-id :id} :sql-transform-mbql-consumer} (testbed)]+ (testing "when breaking the inner snippet with a nonexistent table"+ (let [snippet' (assoc snippet-inner+ :content "nonexistent_table"+ :template-tags {})+ rating (get sql-transform-cols "RATING")+ errors (dependencies/errors-from-proposed-edits provider graph {:snippet [snippet']})]+ ;; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL+ ;; queries that consume the transform's table.+ (is (=? {:card {direct-sql-card-id [{:table {:table "NONEXISTENT_TABLE"},+ :type :all-columns,+ :metabase.driver.sql/bad-reference true}]+ transformed-sql-card-id [{:table {:schema "TRANSFORMED", :table "OUTPUT_TF31"},+ :type :all-columns,+ :metabase.driver.sql/bad-reference true}]+ transformed-mbql-card-id [[:field {} (:id rating)]]}+ :transform {(:id sql-transform) [{:table {:table "NONEXISTENT_TABLE"},+ :type :all-columns,+ :metabase.driver.sql/bad-reference true}]}}+ errors))+ (is (= #{:card :transform}+ (set (keys errors))))+ (is (= #{direct-sql-card-id transformed-sql-card-id transformed-mbql-card-id}+ (set (keys (:card errors)))))+ (is (= #{(:id sql-transform)}+ (set (keys (:transform errors))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_provider_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_provider_test.cljnew file mode 100644index 00000000000..e108a389f64--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_provider_test.clj@@ -0,0 +1,131 @@+(ns metabase-enterprise.dependencies.metadata-provider-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [medley.core :as m]+ [metabase-enterprise.dependencies.metadata-provider :as deps.mp]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]+ [metabase.lib.test-util.macros :as lib.tu.macros]))++(deftest basic-cards-test+ (testing "overriding a single card, it is returned correctly"+ (let [card (:orders (lib.tu/mock-cards))+ mp (-> card+ lib.tu/metadata-provider-with-mock-card+ deps.mp/override-metadata-provider)]+ (deps.mp/add-override mp :card (:id card)+ (-> card+ (assoc-in [:dataset-query :query :fields]+ (lib.tu.macros/$ids orders+ [$subtotal $created-at]))+ (dissoc :result-metadata)))+ (is (=? [(meta/field-metadata :orders :subtotal)+ (meta/field-metadata :orders :created-at)]+ (->> (lib.metadata/card mp (:id card))+ (lib/query mp)+ lib/returned-columns))))))++(defn- replace-query [new-query card]+ (-> card+ (assoc :dataset-query (lib/->legacy-MBQL new-query))+ (dissoc :result-metadata)))++(deftest upstream-card-test+ (testing "overriding an upstream card that's consumed by a downstream card"+ (let [upstream (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/breakout (meta/field-metadata :products :category))+ (lib/aggregate (lib/count)))+ mp1 (lib.tu/metadata-provider-with-card-from-query 1 upstream)+ card1 (lib.metadata/card mp1 1)+ by-name (m/index-by :name (lib/returned-columns (lib/query mp1 card1)))+ downstream (-> (lib/query mp1 card1)+ (lib/filter (lib/= (get by-name "count") 1)))+ mp (-> (lib.tu/metadata-provider-with-card-from-query mp1 2 downstream)+ deps.mp/override-metadata-provider)]+ (deps.mp/add-override mp :card 1+ (-> upstream+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :day))+ (replace-query card1)))+ ;; NOTE: This line is important: this is what would happen to all the dependents returned by the graph.+ ;; There's no update (`nil`) but it still includes the downstream stuff in the scope of the+ ;; OverrideMetadataProvider.+ (deps.mp/add-override mp :card 2 nil)+ (testing "upstream card's columns are correct"+ (is (=? [{:name "CATEGORY"}+ {:name "CREATED_AT"}+ {:name "count"}]+ (->> (lib.metadata/card mp 1)+ (lib/query mp)+ lib/returned-columns))))+ (testing "downstream card (which selects all) is also correct"+ (is (=? [{:name "CATEGORY"}+ {:name "CREATED_AT"}+ {:name "count"}]+ (->> (lib.metadata/card mp 2)+ (lib/query mp)+ lib/returned-columns)))))))++(deftest card->transform->card-test+ (testing "changing the columns of an upstream card that is transformed and then used by a card"+ (let [upstream (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/breakout (meta/field-metadata :products :category))+ (lib/aggregate (lib/count)))+ mp1 (lib.tu/metadata-provider-with-card-from-query 1 upstream)+ card1 (lib.metadata/card mp1 1)+ card1-cols (-> (lib/query mp1 card1)+ lib/returned-columns)++ transform {:lib/type :metadata/transform+ :id 123+ :name "The Transform"+ :source {:type :query+ :query (-> (lib/query mp1 (lib.metadata/card mp1 1))+ lib/->legacy-MBQL)}+ :target {:type "table"+ :schema "transform_schema_123"+ :name "autobots"}}+ mp2 (lib.tu/mock-metadata-provider+ mp1 {:tables [{:id 123000000+ :lib/type :metadata/table+ :name "autobots"+ :schema "transform_schema_123"+ :db-id (meta/id)+ :visibility-type nil+ :active true}]+ :fields (map-indexed (fn [i col]+ (assoc col+ :id (+ 123000001 i)+ :table-id 123000000))+ card1-cols)})++ xform-cols (m/index-by :name (lib.metadata/fields mp2 123000000))+ downstream (-> (lib/query mp2 (lib.metadata/table mp2 123000000))+ (lib/filter (lib/= (get xform-cols "count") 1)))+ mp (-> (lib.tu/metadata-provider-with-card-from-query mp2 2 downstream)+ deps.mp/override-metadata-provider)]+ (deps.mp/add-override mp :card 1+ (-> upstream+ (lib/remove-clause (first (lib/breakouts upstream)))+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :day))+ (replace-query card1)))+ ;; NOTE: Theses lines are important: this is what would happen to all the dependents returned by the graph.+ ;; There's no update (`nil`) but it still includes the downstream stuff in the scope of the+ ;; OverrideMetadataProvider.+ (deps.mp/add-override mp :card 2 nil)+ (deps.mp/add-override mp :transform 123 transform)++ (testing "upstream card's columns are correct"+ (is (=? [{:name "CREATED_AT"}+ {:name "count"}]+ (->> (lib.metadata/card mp 1)+ (lib/query mp)+ lib/returned-columns))))+ (testing "downstream card (which selects all) is also correct"+ (is (=? [{:name "CREATED_AT"}+ {:name "count"}]+ (->> (lib.metadata/card mp 2)+ (lib/query mp)+ lib/returned-columns+ (sort-by :name))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.cljnew file mode 100644index 00000000000..af57d29840f--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj@@ -0,0 +1,216 @@+(ns metabase-enterprise.dependencies.models.dependency-test+ (:require+ [clojure.set :as set]+ [clojure.test :refer [deftest is testing]]+ [metabase-enterprise.dependencies.models.dependency :as deps.graph]+ [metabase.lib.core :as lib]+ [metabase.queries.models.card :as card]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(defn- depends-on-> [from-type from-id to-type to-id]+ {:from_entity_type from-type+ :from_entity_id from-id+ :to_entity_type to-type+ :to_entity_id to-id})++(defn- upstream-of [from-type from-id]+ (t2/select-fn-set #(select-keys % [:from_entity_type :from_entity_id :to_entity_type :to_entity_id])+ :model/Dependency :from_entity_type from-type :from_entity_id from-id))++(defn- basic-orders []+ {:name "Test card"+ :database_id (mt/id)+ :table_id (mt/id :orders)+ :display :table+ :query_type :query+ :type :question+ :dataset_query (mt/mbql-query orders)+ :visualization_settings {}})++(defn- wrap-card [inner-card]+ {:name "Downstream card"+ :database_id (mt/id)+ :display :table+ :query_type :query+ :type :question+ :dataset_query (mt/mbql-query nil+ {:source-table (str "card__" (:id inner-card))})+ :visualization_settings {}})++(deftest ^:sequential card-deps-maintenance-test-1-new-card+ (testing "upstream deps of a card are updated correctly"+ (mt/dataset test-data+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (let [card1 (card/create-card! (basic-orders) user)]+ (is (integer? (:id card1)))+ (testing "when creating a new card"+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}+ (upstream-of :card (:id card1))))++ (testing "that depends on another card"+ (let [card2 (card/create-card! (wrap-card card1) user)]+ (is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}+ (upstream-of :card (:id card2))))+ (testing "but that doesn't affect the upstream deps of the inner card"+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}+ (upstream-of :card (:id card1))))))))++ (testing "when updating an existing card"+ (testing "to add a new table dep"+ (card/update-card! {:card-before-update card1+ :card-updates+ {:dataset_query (mt/mbql-query orders+ {:joins [{:alias "Products"+ :source-table (mt/id :products)+ :condition [:= $id &Products.$products.id]}]})}})+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))+ (depends-on-> :card (:id card1) :table (mt/id :products))}+ (upstream-of :card (:id card1)))))+ (testing "to remove a table dep"+ (card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))+ :card-updates+ {:dataset_query (mt/mbql-query products)}})+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}+ (upstream-of :card (:id card1))))))))))))++(deftest ^:sequential card-deps-graph-test-1-mbql-card-chain+ (testing "deps graph is connected properly for a chain of MBQL cards"+ (mt/dataset test-data+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)+ {id3 :id :as card3} (card/create-card! (wrap-card card2) user)]+ (testing "raw deps are recorded correctly"+ (is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))+ (is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))+ (is (=? #{(depends-on-> :card id3 :card id2)} (upstream-of :card id3))))++ (testing "transitive deps are computed correctly"+ (testing "for each card"+ (is (=? {:card #{id2 id3}}+ (deps.graph/transitive-dependents {:card [card1]})))+ (is (=? {:card #{id3}}+ (deps.graph/transitive-dependents {:card [card2]})))+ (is (=? {}+ (deps.graph/transitive-dependents {:card [card3]}))))+ (testing "for the table"+ (is (set/subset? #{id1 id2 id3}+ (-> {:table [{:id (mt/id :orders)}]}+ deps.graph/transitive-dependents+ :card)))))))))))++(defn- sql-card [metadata-provider sql]+ {:name "SQL card"+ :database_id (mt/id)+ :display :table+ :query_type :query+ :type :question+ :dataset_query (-> (lib/native-query metadata-provider sql)+ lib/->legacy-MBQL)+ :visualization_settings {}})++(deftest ^:sequential card-deps-graph-test-2-native-card-chain+ (testing "deps graph is connected properly for a chain of native cards"+ (mt/dataset test-data+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (let [mp (mt/metadata-provider)+ {id1 :id :as card1} (card/create-card! (sql-card mp "SELECT * FROM orders;") user)+ {id2 :id :as card2} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id1 "}}")) user)+ {id3 :id :as card3} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id2 "}}")) user)]+ (testing "raw deps are recorded correctly"+ (is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))+ (is (=? #{(depends-on-> :card id2 :table (mt/id :orders))+ (depends-on-> :card id2 :card id1)} (upstream-of :card id2)))+ (is (=? #{(depends-on-> :card id3 :table (mt/id :orders))+ (depends-on-> :card id3 :card id2)} (upstream-of :card id3))))+ (testing "transitive deps are computed correctly"+ (testing "for each card"+ (is (=? {:card #{id2 id3}}+ (deps.graph/transitive-dependents {:card [card1]})))+ (is (=? {:card #{id3}}+ (deps.graph/transitive-dependents {:card [card2]})))+ (is (=? {}+ (deps.graph/transitive-dependents {:card [card3]}))))+ (testing "for the table"+ (is (set/subset? #{id1 id2 id3}+ (-> {:table [{:id (mt/id :orders)}]}+ deps.graph/transitive-dependents+ :card)))))))))))++(deftest ^:sequential card-deps-graph-metric-test+ (testing "deps graph is connected properly for a question using a metric"+ (mt/dataset test-data+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (let [metric-card (card/create-card! {:name "Test Metric"+ :database_id (mt/id)+ :table_id (mt/id :orders)+ :display :scalar+ :query_type :query+ :type :metric+ :dataset_query (mt/mbql-query orders {:aggregation [[:count]]})+ :visualization_settings {}}+ user)+ question-card (card/create-card! {:name "Question using Metric"+ :database_id (mt/id)+ :display :table+ :query_type :query+ :type :question+ :dataset_query (mt/mbql-query orders+ {:aggregation [[:metric (:id metric-card)]]})+ :visualization_settings {}}+ user)]+ (testing "raw deps are recorded correctly for question using metric"+ (is (=? #{(depends-on-> :card (:id question-card) :table (mt/id :orders))+ (depends-on-> :card (:id question-card) :card (:id metric-card))}+ (upstream-of :card (:id question-card)))))))))))++(deftest dependency-analysis-version-test+ (testing "dependency_analysis_version is updated when entities are created or updated"+ (testing "cards"+ (mt/with-temp [:model/Card card {:dependency_analysis_version 0}]+ (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))+ (mt/with-premium-features #{:dependencies}+ (card/update-card! {:card-before-update card+ :card-updates {:dataset_query (mt/mbql-query orders)}}))+ (is (= deps.graph/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))))+ (testing "transforms create"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Transform transform]+ (is (= deps.graph/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))+ (testing "transforms update"+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Transform transform]+ (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform))))+ (mt/with-premium-features #{:dependencies}+ (t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}}))+ (is (= deps.graph/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))+ (testing "snippets create"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/NativeQuerySnippet snippet]+ (is (= deps.graph/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))+ (testing "snippets update"+ (mt/with-premium-features #{}+ (mt/with-temp [:model/NativeQuerySnippet snippet]+ (is (zero? (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet))))+ (mt/with-premium-features #{:dependencies}+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))+ (is (= deps.graph/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))))++(deftest dependency-analysis-version-create-card-test+ (testing "dependency_analysis_version is updated when a card is created"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/Card]+ (mt/with-temp [:model/User user]+ (let [card (card/create-card! (basic-orders) user)]+ (is (= deps.graph/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Card :id (:id card))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.cljnew file mode 100644index 00000000000..a6e802b975e--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj@@ -0,0 +1,241 @@+(ns metabase-enterprise.dependencies.native-validation-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.dependencies.native-validation :as deps.native-validation]+ [metabase-enterprise.dependencies.test-util :as deps.tu]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]))++(defn- fake-query+ ([mp query]+ (fake-query mp query {}))+ ([mp query template-tags]+ {:database (:id (lib.metadata/database mp))+ :type :native+ :native {:query query+ :template-tags template-tags}}))++(defn- validates?+ [mp driver card-id expected]+ (is (=? expected+ (->> (lib.metadata/card mp card-id)+ :dataset-query+ (deps.native-validation/validate-native-query driver mp)))))++(deftest basic-deps-test+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (is (= #{{:table (meta/id :products)}}+ (->> (lib.metadata/card mp 4)+ :dataset-query+ (deps.native-validation/native-query-deps driver mp))))+ (is (= #{{:table (meta/id :products)}+ {:card 1}}+ (->> (lib.metadata/card mp 5)+ :dataset-query+ (deps.native-validation/native-query-deps driver mp))))+ (is (= #{{:table (meta/id :products)}+ {:card 1}+ {:snippet 1}}+ (->> (lib.metadata/card mp 6)+ :dataset-query+ (deps.native-validation/native-query-deps driver mp))))+ (is (= #{{:table (meta/id :products)}+ {:card 1}+ {:snippet 1}+ {:snippet 2}}+ (->> (lib.metadata/card mp 7)+ :dataset-query+ (deps.native-validation/native-query-deps driver mp))))+ (is (= #{{:table (meta/id :products)}+ {:table (meta/id :orders)}+ {:card 1}+ {:card 2}}+ (->> (lib.metadata/card mp 9)+ :dataset-query+ (deps.native-validation/native-query-deps driver mp))))))++(deftest validate-bad-queries-test+ (testing "validate-native-query handles nonsense queries"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "complete nonsense query"+ (is (= [{:error :bad-sql}]+ (deps.native-validation/validate-native-query+ driver mp+ (fake-query mp "this is not a query")))))+ (testing "bad table wildcard"+ (is (= [{:type :invalid-table-wildcard,+ :table "products",+ :metabase.driver.sql/bad-reference true}]+ (deps.native-validation/validate-native-query+ driver mp+ (fake-query mp "select products.* from orders")))))+ (testing "bad col reference"+ (is (= [{:column "BAD",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "PRODUCTS"}}]],+ :metabase.driver.sql/bad-reference true}]+ (deps.native-validation/validate-native-query+ driver mp+ (fake-query mp "select bad from products"))))))))++(deftest validate-native-query-with-subquery-columns-test+ (testing "validate-native-query should detect invalid columns in subqueries"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]++ (testing "Valid query - selecting existing columns from subquery"+ (validates? mp driver 10 empty?))++ (testing "Invalid query - selecting non-existent column from subquery"+ (validates? mp driver 11 [{:column "CATEGORY",+ :alias nil,+ :type :single-column,+ :source-columns [[]],+ :metabase.driver.sql/bad-reference true}])+ (validates? mp driver 12 [{:column "CATEGORY",+ :alias nil,+ :type :single-column,+ :source-columns [[]],+ :metabase.driver.sql/bad-reference true}]))++ (testing "Nested subqueries"+ (validates? mp driver 13 empty?)+ (validates? mp driver 14 [{:column "CATEGORY",+ :alias nil,+ :type :single-column,+ :source-columns [[]],+ :metabase.driver.sql/bad-reference true}]))++ (testing "SELECT * from subquery expands to subquery columns"+ (validates? mp driver 15 empty?)+ (validates? mp driver 16 empty?)+ (validates? mp driver 17 [{:column "EMAIL",+ :alias nil,+ :type :single-column,+ :source-columns [[]],+ :metabase.driver.sql/bad-reference true}])))))++(deftest validate-card-reference-after-expansion-test+ (testing "Validation of queries after card references have been expanded"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]++ (testing "Card reference expanded to subquery - valid columns"+ (validates? mp driver 18 empty?))++ (testing "Card reference expanded to subquery - invalid column"+ (validates? mp driver 19+ [{:column "DESCRIPTION",+ :alias nil,+ :type :single-column,+ :source-columns [[]],+ :metabase.driver.sql/bad-reference true}]))++ (testing "Card reference with alias - valid column"+ (validates? mp driver 20 empty?))++ (testing "Card reference with alias - invalid column"+ (validates? mp driver 21+ [{:column "PASSWORD",+ :alias nil,+ :type :single-column,+ :source-columns [[]],+ :metabase.driver.sql/bad-reference true}]))++ (testing "Wildcard selection from card reference"+ (validates? mp driver 22 empty?))++ (testing "Invalid column from aliased card"+ (validates? mp driver 23+ [{:column "LATITUDE",+ :alias nil,+ :type :single-column,+ :source-columns [[]],+ :metabase.driver.sql/bad-reference true}])))))++(defn- check-result-metadata [driver mp query expected]+ (is (=? expected+ (->> query+ (fake-query mp)+ (deps.native-validation/native-result-metadata driver mp)))))++(defn- add-desired-column-alias [fields]+ (map #(assoc % :lib/desired-column-alias (:name %)) fields))++(deftest result-metadata-test+ (testing "Calculates result metadata"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (testing "Selecting a wildcard"+ (check-result-metadata+ driver mp+ "select * from orders"+ (add-desired-column-alias (lib.metadata/fields mp (meta/id :orders)))))+ (testing "Selecting a table wildcard"+ (check-result-metadata+ driver mp+ "select orders.* from orders"+ (add-desired-column-alias (lib.metadata/fields mp (meta/id :orders)))))+ (testing "Selecting a single col"+ (check-result-metadata+ driver mp+ "select total from orders"+ (add-desired-column-alias [(lib.metadata/field mp (meta/id :orders :total))])))+ (testing "Selecting a nonexistent col"+ (check-result-metadata+ driver mp+ "select bad from orders"+ []))+ (testing "Selecting a col with an alias"+ (check-result-metadata+ driver mp+ "select subtotal as new_subtotal from orders"+ [{:lib/type :metadata/column,+ :base-type :type/Float,+ :semantic-type nil,+ :name "SUBTOTAL",+ :lib/desired-column-alias "NEW_SUBTOTAL"+ :database-type "DOUBLE PRECISION",+ :display-name "Subtotal"}]))+ (testing "Selecting a custom col"+ (check-result-metadata+ driver mp+ "select subtotal + tax as sum from orders"+ [{:base-type :type/*,+ :name "SUM",+ :display-name "Sum",+ :effective-type :type/*,+ :semantic-type :Semantic/*}]))+ (testing "Selecting a union"+ (check-result-metadata+ driver mp+ "select total from orders union select subtotal from orders"+ [{:name "TOTAL",+ :lib/desired-column-alias "TOTAL",+ :display-name "Total",+ :base-type :type/Float,+ :effective-type :type/Float,+ :semantic-type :Semantic/*}]))+ (testing "Selecting a union with different types"+ (check-result-metadata+ driver mp+ "select category from products union select title from products"+ [{:name "CATEGORY",+ :lib/desired-column-alias "CATEGORY",+ :display-name "Category",+ :base-type :type/Text,+ :effective-type :type/Text,+ :semantic-type :type/Category}]))+ (testing "Selecting a bad table wildcard"+ (check-result-metadata+ driver mp+ "select orders.* from products"+ []))+ (testing "Using a nonsense query"+ (check-result-metadata+ driver mp+ "this is not a query"+ [])))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljnew file mode 100644index 00000000000..cd8c1aa223f--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj@@ -0,0 +1,231 @@+(ns metabase-enterprise.dependencies.task.backfill-test+ (:require+ [clojure.test :refer :all]+ [environ.core :as env]+ [java-time.api :as t]+ [metabase-enterprise.dependencies.models.dependency :as dependencies.model]+ [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]+ [metabase.task.core :as task]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db))++(defn- backfill-all-existing-entities+ []+ (mt/with-premium-features #{:dependencies}+ (while (#'dependencies.backfill/backfill-dependencies))))++(defn- backfill-dependencies-single-trigger+ []+ (mt/with-premium-features #{:dependencies}+ (#'dependencies.backfill/backfill-dependencies)))++(deftest ^:sequential backfill-dependency-analysis-test+ (testing "Test that the backfill job correctly updates the dependency_analysis_version"+ (backfill-all-existing-entities)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]+ (let [query (mt/mbql-query orders)]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}+ :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}+ :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]+ (let [card-count (fn [& args]+ (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))+ current-version dependencies.model/current-dependency-analysis-version]+ (is (= 3 (card-count)))+ ;; first run, should process 2 cards+ (is (true? (backfill-dependencies-single-trigger)))+ (is (= 2 (card-count :dependency_analysis_version current-version)))+ ;; second run, should process the last card+ (is (false? (backfill-dependencies-single-trigger)))+ (is (= 3 (card-count :dependency_analysis_version current-version)))+ ;; third run, should not process anything+ (is (false? (backfill-dependencies-single-trigger)))+ (is (= 3 (card-count :dependency_analysis_version current-version))))))))))++(deftest ^:sequential backfill-transform-test+ (testing "Test that transforms are correctly backfilled"+ (backfill-all-existing-entities)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)+ :dependency_analysis_version 0}+ :model/Transform {transform-id :id} {:name "Test Transform"+ :source {:type "query"+ :query (mt/mbql-query nil+ {:source-table (str "card__" card-id)})}+ :target {:type "table" :name "test_table"}+ :dependency_analysis_version 0}]+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ (is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))+ (is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))+ (backfill-dependencies-single-trigger)+ (is (t2/exists? :model/Card :id card-id+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (is (t2/exists? :model/Transform :id transform-id+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (is (t2/exists? :model/Dependency+ :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders)))+ (is (t2/exists? :model/Dependency+ :from_entity_type :transform :from_entity_id transform-id+ :to_entity_type :card :to_entity_id card-id)))))))++(deftest ^:sequential backfill-snippet-test+ (testing "Test that cards with snippets are correctly backfilled"+ (backfill-all-existing-entities)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"+ :content "SELECT 1"+ :dependency_analysis_version 0}+ :model/Card {card-id :id} {:dataset_query {:database (mt/id)+ :type :native+ :native {:query "SELECT * FROM {{my_snippet}}"+ :template-tags+ {:my_snippet+ {:id (str (random-uuid))+ :name "my_snippet"+ :display-name "my_snippet"+ :type :snippet+ :snippet-name "my_snippet"+ :snippet-id snippet-id}}}}+ :dependency_analysis_version 0}]+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ (is (t2/exists? :model/NativeQuerySnippet :id snippet-id :dependency_analysis_version 0))+ (is (false? (t2/exists? :model/Dependency+ :from_entity_type :card :from_entity_id card-id+ :to_entity_type :snippet :to_entity_id snippet-id)))+ (backfill-dependencies-single-trigger)+ (is (t2/exists? :model/Card :id card-id+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (is (t2/exists? :model/NativeQuerySnippet :id snippet-id+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (is (t2/exists? :model/Dependency+ :from_entity_type :card :from_entity_id card-id+ :to_entity_type :snippet :to_entity_id snippet-id)))))))++(deftest ^:sequential backfill-idempotency-test+ (testing "Running the backfill multiple times should be idempotent"+ (backfill-all-existing-entities)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)+ :dependency_analysis_version 0}]+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders))))+ ;; First run+ (backfill-dependencies-single-trigger)+ (is (t2/exists? :model/Card :id card-id+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ ;; Second run - should not change anything+ (backfill-dependencies-single-trigger)+ (is (t2/exists? :model/Card :id card-id+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders))))))))++(defn- wait-for-condition+ [predicate timeout-ms]+ (let [limit (+ (System/currentTimeMillis) timeout-ms)]+ (loop []+ (Thread/sleep 100)+ (or (predicate)+ (when (< (System/currentTimeMillis) limit)+ (recur))))))++(deftest ^:sequential backfill-scheduling-test+ (testing "Test that the backfill job schedules and reschedules itself correctly"+ (backfill-all-existing-entities)+ (mt/with-temp-scheduler!+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-batch-size "1"+ :mb-dependency-backfill-delay-minutes "0"+ :mb-dependency-backfill-variance-minutes "0")]+ (let [card-data {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card1-id :id} card-data+ :model/Card {card2-id :id} card-data+ :model/Card {card3-id :id} card-data]+ (let [card-count (fn [& args]+ (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))+ current-version dependencies.model/current-dependency-analysis-version]+ (is (= 3 (card-count :dependency_analysis_version 0)))+ (mt/with-premium-features #{:dependencies}+ ;; Initialize the task, which should schedule the first run+ (task/init! ::dependencies.backfill/DependencyBackfill)+ (wait-for-condition #(= 3 (card-count :dependency_analysis_version current-version)) 10000)+ (is (= 3 (card-count :dependency_analysis_version current-version)))))))))))))++(deftest ^:sequential backfill-terminal-failure-test+ (testing "Entities should be marked as terminally broken after MAX_RETRIES failures"+ (backfill-all-existing-entities)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))++ (let [update-attempts (volatile! 0)+ failures (inc @#'dependencies.backfill/MAX_RETRIES)]+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-delay-minutes "0"+ :mb-dependency-backfill-variance-minutes "0")+ t2/update! (fn [model-kw id & args]+ (if (and (= model-kw :model/Card)+ (= id card-id)+ (< @update-attempts failures))+ (do+ (vswap! update-attempts inc)+ (throw (ex-info "Simulated DB error" {:id id})))+ (apply t2/update! model-kw id args)))]+ ;; fail MAX_RETRIES + 1 times+ (while (< @update-attempts failures)+ (backfill-dependencies-single-trigger))))++ ;; verify card is not processed and is terminally broken+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))++ ;; verify subsequent runs don't process it+ (backfill-dependencies-single-trigger)+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))))))++(deftest ^:sequential backfill-delayed-retry-test+ (testing "Failed entities should be retried after their delay period expires"+ (backfill-all-existing-entities)+ (let [current-version dependencies.model/current-dependency-analysis-version+ update-attempts (volatile! 0)]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-delay-minutes "1") ; 1 minute delay for retries+ t2/update! (fn [model-kw id & args]+ (if (and (= model-kw :model/Card)+ (= id card-id)+ (zero? @update-attempts))+ (do+ (vswap! update-attempts inc)+ (throw (ex-info "Simulated DB error" {:id id})))+ (apply t2/update! model-kw id args)))]+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))++ ;; first failure - should be put into retry state+ ;; fail MAX_RETRIES + 1 times+ (while (zero? @update-attempts)+ (backfill-dependencies-single-trigger))+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0)))++ ;; advance time by less than retry delay - should NOT be processed+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))+ (backfill-dependencies-single-trigger))+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))++ ;; advance time by more than retry delay - should be processed+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))+ (backfill-dependencies-single-trigger))+ (is (t2/exists? :model/Card :id card-id :dependency_analysis_version current-version)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.cljnew file mode 100644index 00000000000..57bd8f407b7--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj@@ -0,0 +1,125 @@+(ns metabase-enterprise.dependencies.test-util+ (:require+ [medley.core :as m]+ [metabase.lib.convert :as lib.convert]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util.metadata-providers.mock :as providers.mock]))++(defn mock-card [metadata-provider {:keys [id query details]}]+ (merge {:lib/type :metadata/card+ :id id+ :database-id (:id (lib.metadata/database metadata-provider))+ :dataset-query (if (string? query)+ {:database (:id (lib.metadata/database metadata-provider))+ :type :native+ :native {:query query+ :template-tags (lib/extract-template-tags metadata-provider query)}}+ (lib.convert/->legacy-MBQL query))+ :name (str (gensym))+ :result-metadata (when (not (string? query))+ (->> (lib/returned-columns query)+ (sort-by :id)))}+ details))++(defn mock-snippet [{:keys [id name content template-tags]}]+ {:description nil,+ :archived false,+ :collection-id nil,+ :id id+ :name name+ :content content+ :template-tags template-tags+ :lib/type :metadata/native-query-snippet})++(defn mock-metadata-provider [{:keys [cards snippets]}]+ (lib/composed-metadata-provider+ meta/metadata-provider+ (providers.mock/mock-metadata-provider+ {:cards cards+ :native-query-snippets snippets})))++(defn default-metadata-provider []+ (let [snippets (map mock-snippet+ [{:id 1+ :name "inner"+ :content "{{#card-ref}}"+ :template-tags {"#card-ref" {:card-id 1+ :type :card+ :name "#card-ref"+ :display-name "#Card Ref"+ :id (str (random-uuid))}}}+ {:id 2+ :name "outer"+ :content "{{snippet: inner}}"+ :template-tags {"snippet: inner" {:snippet-id 1+ :snippet-name "inner"+ :type :snippet+ :name "snippet: inner"+ :display-name "Snippet: Inner"+ :id (str (random-uuid))}}}])+ initial-mock-mp (mock-metadata-provider {:snippets snippets})+ initial-card-a (mock-card initial-mock-mp+ {:id 1+ :query (lib/query initial-mock-mp (meta/table-metadata :products))})+ initial-card-b (mock-card initial-mock-mp+ {:id 2+ :query (lib/query initial-mock-mp (meta/table-metadata :orders))})]+ (mock-metadata-provider+ {:cards (into [initial-card-a+ initial-card-b]+ (map #(mock-card initial-mock-mp %))+ [{:id 3+ :query (lib/query initial-mock-mp initial-card-a)}+ {:id 4+ :query "select * from products"}+ {:id 5+ :query "select * from {{#1}}"}+ {:id 6+ :query "select * from {{snippet: inner}}"}+ {:id 7+ :query "select * from {{snippet: outer}}"}+ {:id 8+ :query (as-> (lib/query initial-mock-mp initial-card-b) q+ (lib/join q (-> (lib/join-clause initial-card-a)+ (lib/with-join-conditions+ [(lib/=+ (m/find-first #(= (:name %) "PRODUCT_ID")+ (lib/join-condition-lhs-columns+ q initial-card-b nil nil))+ (m/find-first #(= (:name %) "ID")+ (lib/join-condition-lhs-columns+ q initial-card-a nil nil)))])+ (lib/with-join-fields :all))))}+ {:id 9+ :query "select * from {{#1}} inner join {{#2}}"}+ {:id 10+ :query "SELECT id, name FROM (SELECT id, name, email FROM PEOPLE)"}+ {:id 11+ :query "SELECT category FROM (SELECT id, name FROM PEOPLE)"}+ {:id 12+ :query "SELECT id, category FROM (SELECT id, name FROM PEOPLE)"}+ {:id 13+ :query "SELECT id FROM (SELECT id, name FROM (SELECT id, name, email FROM PEOPLE))"}+ {:id 14+ :query "SELECT category FROM (SELECT id, name FROM (SELECT id, name, email FROM PEOPLE))"}+ {:id 15+ :query "SELECT * FROM (SELECT id, name FROM PEOPLE)"}+ {:id 16+ :query "SELECT name FROM (SELECT * FROM (SELECT id, name FROM PEOPLE))"}+ {:id 17+ :query "SELECT email FROM (SELECT * FROM (SELECT id, name FROM PEOPLE))"}+ {:id 18+ :query "SELECT ID, TITLE FROM {{#1}}"}+ {:id 19+ :query "SELECT DESCRIPTION FROM {{#1}}"}+ {:id 20+ :query "SELECT ID FROM {{#1}} AS p"}+ {:id 21+ :query "SELECT PASSWORD FROM {{#1}} AS p"}+ {:id 22+ :query "SELECT * FROM {{#1}}"}+ {:id 23+ :query "SELECT p.LATITUDE FROM {{#1}} AS p"}])+ :snippets snippets})))diff --git a/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/models/entity_id_test.cljindex b080f26e0ed..18f036505b9 100644--- a/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj@@ -47,6 +47,7 @@:model/DashboardBookmark:model/DataPermissions:model/DatabaseRouter+ :model/Dependency:model/DocumentBookmark:model/CollectionPermissionGraphRevision:model/DashboardCardSeriesdiff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex 79bd31a85bd..d91fc3a11b5 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -224,8 +224,14 @@:model/Transform child (-> (->transform "transform2" (mt/mbql-query nil {:source-table table}))(assoc-in [:target :name] "orders_3"))](mt/with-premium-features #{:transforms}- (is (= [parent]- (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s/dependencies" (:id child)))))))))+ (is (=? [{:name "transform1"+ :source {:type "query", :query {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}}+ :id (:id parent)+ :entity_id (:entity_id parent)+ :target {:schema "public", :name "orders_2", :type "table"}}]+ (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s/dependencies" (:id child)))))))))(deftest put-transforms-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)diff --git a/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsx b/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsxindex c1abe7d2067..b6bac3b4cdf 100644--- a/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsx+++ b/enterprise/frontend/src/embedding-sdk-bundle/components/private/SdkQuestion/components/SdkSaveQuestionForm.tsx@@ -44,6 +44,7 @@ export const SdkSaveQuestionForm = ({ onCancel }: SaveQuestionFormProps) => {originalQuestion={originalQuestion ?? null}onCreate={onCreate}onSave={onSave}+ onCancel={onCancel}multiStep={false}targetCollection={targetCollection ?? undefined}>diff --git a/enterprise/frontend/src/metabase-enterprise/api/dependencies.ts b/enterprise/frontend/src/metabase-enterprise/api/dependencies.tsnew file mode 100644index 00000000000..0df4d751f84--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/api/dependencies.ts@@ -0,0 +1,49 @@+import type {+ CheckCardDependenciesRequest,+ CheckDependenciesResponse,+ CheckSnippetDependenciesRequest,+ CheckTransformDependenciesRequest,+} from "metabase-types/api";++import { EnterpriseApi } from "./api";++export const dependencyApi = EnterpriseApi.injectEndpoints({+ endpoints: (builder) => ({+ checkCardDependencies: builder.query<+ CheckDependenciesResponse,+ CheckCardDependenciesRequest+ >({+ query: (body) => ({+ method: "POST",+ url: "/api/ee/dependencies/check_card",+ body,+ }),+ }),+ checkSnippetDependencies: builder.query<+ CheckDependenciesResponse,+ CheckSnippetDependenciesRequest+ >({+ query: (body) => ({+ method: "POST",+ url: "/api/ee/dependencies/check_snippet",+ body,+ }),+ }),+ checkTransformDependencies: builder.query<+ CheckDependenciesResponse,+ CheckTransformDependenciesRequest+ >({+ query: (body) => ({+ method: "POST",+ url: "/api/ee/dependencies/check_transform",+ body,+ }),+ }),+ }),+});++export const {+ useLazyCheckCardDependenciesQuery,+ useLazyCheckSnippetDependenciesQuery,+ useLazyCheckTransformDependenciesQuery,+} = dependencyApi;diff --git a/enterprise/frontend/src/metabase-enterprise/api/index.ts b/enterprise/frontend/src/metabase-enterprise/api/index.tsindex ffeadd11cc7..01cedf6e0b6 100644--- a/enterprise/frontend/src/metabase-enterprise/api/index.ts+++ b/enterprise/frontend/src/metabase-enterprise/api/index.ts@@ -5,6 +5,7 @@ export * from "./billing-info";export * from "./comment";export * from "./content-translation";export * from "./db-routing";+export * from "./dependencies";export * from "./document";export * from "./gdrive";export * from "./metabot";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.module.css b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.module.cssnew file mode 100644index 00000000000..de3cba6c740--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.module.css@@ -0,0 +1,12 @@+.card {+ flex-shrink: 0;++ &:hover {+ background-color: var(--mb-color-background-light);+ }+}++.list {+ overflow: auto;+ max-height: 60vh;+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.tsxnew file mode 100644index 00000000000..60319e50cbb--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.tsx@@ -0,0 +1,239 @@+import { Fragment } from "react";+import { Link } from "react-router";+import { t } from "ttag";++import {+ Form,+ FormErrorMessage,+ FormProvider,+ FormSubmitButton,+} from "metabase/forms";+import * as Urls from "metabase/lib/urls";+import type { CheckDependenciesFormProps } from "metabase/plugins";+import {+ Anchor,+ Box,+ Button,+ Card,+ FixedSizeIcon,+ Group,+ type IconName,+ Stack,+ Text,+} from "metabase/ui";+import visualizations from "metabase/visualizations";+import { getTransformUrl } from "metabase-enterprise/transforms/urls";+import type {+ Card as ApiCard,+ CheckDependenciesResponse,+ Transform,+} from "metabase-types/api";++import S from "./CheckDependenciesForm.module.css";++type DependencyItem = CardDependencyItem | TransformDependencyItem;++type CardDependencyItem = {+ type: "card";+ card: ApiCard;+};++type TransformDependencyItem = {+ type: "transform";+ transform: Transform;+};++export function CheckDependenciesForm({+ checkData,+ onSave,+ onCancel,+}: CheckDependenciesFormProps) {+ const items = getDependencyItems(checkData);++ return (+ <FormProvider initialValues={{}} onSubmit={onSave}>+ <Form>+ <Text mb="md" px="xl">+ {t`The items below will break because of these changes:`}+ </Text>+ <Stack className={S.list} mb="xl" px="xl">+ {items.map((item, index) => (+ <DependencyItemCard key={index} item={item} />+ ))}+ </Stack>+ <Group px="xl">+ <Box flex={1}>+ <FormErrorMessage />+ </Box>+ <Button onClick={onCancel}>{t`Cancel`}</Button>+ <FormSubmitButton label={t`Save anyway`} variant="filled" />+ </Group>+ </Form>+ </FormProvider>+ );+}++function getDependencyItems({+ bad_cards = [],+ bad_transforms = [],+}: CheckDependenciesResponse): DependencyItem[] {+ return [+ ...bad_transforms.map(+ (transform): TransformDependencyItem => ({+ type: "transform",+ transform,+ }),+ ),+ ...bad_cards.map(+ (card): CardDependencyItem => ({+ type: "card",+ card,+ }),+ ),+ ];+}++type DependencyItemCardProps = {+ item: DependencyItem;+};++function DependencyItemCard({ item }: DependencyItemCardProps) {+ return (+ <Card+ className={S.card}+ component={Link}+ to={getItemLink(item)}+ p="md"+ shadow="none"+ withBorder+ >+ <Stack gap="xs">+ <Group c="brand" gap="sm" wrap="nowrap">+ <FixedSizeIcon name={getItemIcon(item)} />+ <Box fw="bold" lh="h4">+ {getItemName(item)}+ </Box>+ </Group>+ {getItemDescription(item)}+ </Stack>+ </Card>+ );+}++function getItemIcon(item: DependencyItem): IconName {+ if (item.type === "card") {+ const { card } = item;++ switch (card.type) {+ case "question":+ return visualizations.get(card.display)?.iconName ?? "table2";+ case "model":+ return "model";+ case "metric":+ return "metric";+ }+ }++ if (item.type === "transform") {+ return "refresh_downstream";+ }++ return "unknown";+}++function getItemName(item: DependencyItem) {+ switch (item.type) {+ case "card":+ return item.card.name;+ case "transform":+ return item.transform.name;+ default:+ return null;+ }+}++function getItemLink(item: DependencyItem) {+ switch (item.type) {+ case "card":+ return Urls.question(item.card);+ default:+ return getTransformUrl(item.transform.id);+ }+}++function getItemDescription(item: DependencyItem) {+ if (item.type === "card") {+ const { collection, dashboard } = item.card;++ if (collection != null) {+ const ancestors = collection.effective_ancestors ?? [];+ const breadcrumbs = [+ ...ancestors.map((ancestor) => ({+ title: ancestor.name,+ to: Urls.collection(ancestor),+ })),+ { title: collection.name, to: Urls.collection(collection) },+ ...(dashboard != null+ ? [{ title: dashboard.name, to: Urls.dashboard(dashboard) }]+ : []),+ ];++ return (+ <Group gap="sm" wrap="nowrap">+ <FixedSizeIcon+ c="text-secondary"+ name={dashboard != null ? "dashboard" : "collection"}+ flex="0 0 auto"+ />+ <BreadcrumbList items={breadcrumbs} />+ </Group>+ );+ }+ }++ if (item.type === "transform") {+ return (+ <Box fz="sm" lh="h5">+ {t`Transform`}+ </Box>+ );+ }++ return null;+}++type BreadcrumbItem = {+ title: string;+ to: string;+};++type BreadcrumbProps = {+ item: BreadcrumbItem;+};++function Breadcrumb({ item }: BreadcrumbProps) {+ return (+ <Anchor component={Link} to={item.to} c="text-secondary" fz="sm" lh="h5">+ {item.title}+ </Anchor>+ );+}++type BreadcrumbListProps = {+ items: BreadcrumbItem[];+};++function BreadcrumbList({ items }: BreadcrumbListProps) {+ return (+ <Group align="center" gap="xs">+ {items.map((item, itemIndex) => (+ <Fragment key={itemIndex}>+ {itemIndex > 0 && (+ <FixedSizeIcon name="chevronright" c="text-secondary" size={8} />+ )}+ <Breadcrumb item={item} />+ </Fragment>+ ))}+ </Group>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.unit.spec.tsxnew file mode 100644index 00000000000..91a1db194b1--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/CheckDependenciesForm.unit.spec.tsx@@ -0,0 +1,194 @@+import { Route } from "react-router";++import { getIcon, renderWithProviders, screen } from "__support__/ui";+import type { IconName } from "metabase/ui";+import registerVisualizations from "metabase/visualizations/register";+import type {+ CardDisplayType,+ CardType,+ CheckDependenciesResponse,+} from "metabase-types/api";+import {+ createMockCard,+ createMockCheckDependenciesResponse,+ createMockCollection,+ createMockTransform,+} from "metabase-types/api/mocks";++import { CheckDependenciesForm } from "./CheckDependenciesForm";++type SetupOpts = {+ checkData?: CheckDependenciesResponse;+};++function setup({+ checkData = createMockCheckDependenciesResponse(),+}: SetupOpts) {+ const onSave = jest.fn();+ const onCancel = jest.fn();+ renderWithProviders(+ <Route+ path="/"+ component={() => (+ <CheckDependenciesForm+ checkData={checkData}+ onSave={onSave}+ onCancel={onCancel}+ />+ )}+ />,+ { withRouter: true, initialRoute: "/" },+ );+ return { onSave, onCancel };+}++registerVisualizations();++describe("CheckDependenciesForm", () => {+ describe.each<CardType>(["question", "model", "metric"])("%s", (type) => {+ it("should display broken cards", () => {+ setup({+ checkData: createMockCheckDependenciesResponse({+ success: false,+ bad_cards: [+ createMockCard({+ name: "Card",+ type,+ }),+ ],+ }),+ });+ expect(screen.getByText("Card")).toBeInTheDocument();+ expect(screen.getByRole("link", { name: /Card/ })).toHaveAttribute(+ "href",+ `/${type}/1-card`,+ );+ });++ it("should display the root collection", () => {+ setup({+ checkData: createMockCheckDependenciesResponse({+ success: false,+ bad_cards: [+ createMockCard({+ name: "Card",+ type,+ collection: createMockCollection({+ id: "root",+ name: "Our analytics",+ }),+ }),+ ],+ }),+ });+ expect(+ screen.getByRole("link", { name: "Our analytics" }),+ ).toHaveAttribute("href", "/collection/root");+ });++ it("should display the nested collection with parent collections", () => {+ setup({+ checkData: createMockCheckDependenciesResponse({+ success: false,+ bad_cards: [+ createMockCard({+ name: "Card",+ type,+ collection: createMockCollection({+ id: 2,+ name: "Second collection",+ effective_ancestors: [+ createMockCollection({ id: 1, name: "First collection" }),+ createMockCollection({ id: "root", name: "Our analytics" }),+ ],+ }),+ }),+ ],+ }),+ });+ expect(+ screen.getByRole("link", { name: "Our analytics" }),+ ).toHaveAttribute("href", "/collection/root");+ expect(+ screen.getByRole("link", { name: "First collection" }),+ ).toHaveAttribute("href", "/collection/1-first-collection");+ expect(+ screen.getByRole("link", { name: "Second collection" }),+ ).toHaveAttribute("href", "/collection/2-second-collection");+ });+ });++ it.each<{ display: CardDisplayType; icon: IconName }>([+ { display: "line", icon: "line" },+ { display: "pie", icon: "pie" },+ ])(+ "should display the icon that corresponds to the display type",+ ({ display, icon }) => {+ setup({+ checkData: createMockCheckDependenciesResponse({+ success: false,+ bad_cards: [+ createMockCard({+ type: "question",+ display,+ }),+ ],+ }),+ });+ expect(getIcon(icon)).toBeInTheDocument();+ },+ );++ it("should display the dashboard that the question belongs to", () => {+ setup({+ checkData: createMockCheckDependenciesResponse({+ success: false,+ bad_cards: [+ createMockCard({+ name: "Card",+ type: "question",+ collection: createMockCollection({+ id: 1,+ name: "First collection",+ effective_ancestors: [+ createMockCollection({ id: "root", name: "Our analytics" }),+ ],+ }),+ dashboard: {+ id: 2,+ name: "Card dashboard",+ },+ }),+ ],+ }),+ });+ expect(screen.getByRole("link", { name: "Our analytics" })).toHaveAttribute(+ "href",+ "/collection/root",+ );+ expect(+ screen.getByRole("link", { name: "First collection" }),+ ).toHaveAttribute("href", "/collection/1-first-collection");+ expect(+ screen.getByRole("link", { name: "Card dashboard" }),+ ).toHaveAttribute("href", "/dashboard/2-card-dashboard");+ });++ it("should display transforms", () => {+ setup({+ checkData: createMockCheckDependenciesResponse({+ success: false,+ bad_transforms: [+ createMockTransform({+ id: 1,+ name: "SQL transform",+ }),+ ],+ }),+ });+ expect(screen.getByRole("link", { name: /SQL transform/ })).toHaveAttribute(+ "href",+ "/admin/transforms/1",+ );+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/index.tsnew file mode 100644index 00000000000..420134c7ca9--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesForm/index.ts@@ -0,0 +1 @@+export * from "./CheckDependenciesForm";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/CheckDependenciesModal.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/CheckDependenciesModal.tsxnew file mode 100644index 00000000000..bff78b63a82--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/CheckDependenciesModal.tsx@@ -0,0 +1,35 @@+import type { CheckDependenciesModalProps } from "metabase/plugins";+import { Flex, Modal } from "metabase/ui";++import { CheckDependenciesForm } from "../CheckDependenciesForm";+import { CheckDependenciesTitle } from "../CheckDependenciesTitle";++export function CheckDependenciesModal({+ checkData,+ opened,+ onSave,+ onClose,+}: CheckDependenciesModalProps) {+ return (+ <Modal.Root size="xl" padding="xl" opened={opened} onClose={onClose}>+ <Modal.Overlay />+ <Modal.Content>+ <Modal.Header px="xl">+ <Modal.Title>+ <CheckDependenciesTitle />+ </Modal.Title>+ <Flex justify="flex-end">+ <Modal.CloseButton />+ </Flex>+ </Modal.Header>+ <Modal.Body px={0}>+ <CheckDependenciesForm+ checkData={checkData}+ onSave={onSave}+ onCancel={onClose}+ />+ </Modal.Body>+ </Modal.Content>+ </Modal.Root>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/index.tsnew file mode 100644index 00000000000..6f83f37598d--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesModal/index.ts@@ -0,0 +1 @@+export * from "./CheckDependenciesModal";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/CheckDependenciesTitle.tsx b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/CheckDependenciesTitle.tsxnew file mode 100644index 00000000000..82f20ab86d3--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/CheckDependenciesTitle.tsx@@ -0,0 +1,11 @@+import { t } from "ttag";++import { Box } from "metabase/ui";++export function CheckDependenciesTitle() {+ return (+ <Box fz="h3" lh="h3">+ {t`These changes will break some other things. Save anyway?`}+ </Box>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/index.tsnew file mode 100644index 00000000000..815b05cb08e--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/components/CheckDependenciesTitle/index.ts@@ -0,0 +1 @@+export * from "./CheckDependenciesTitle";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/index.tsnew file mode 100644index 00000000000..bb1a1c1d203--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/index.ts@@ -0,0 +1 @@+export * from "./use-check-card-dependencies";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.tsnew file mode 100644index 00000000000..27dadd7ba9c--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.ts@@ -0,0 +1,40 @@+import { useCallback } from "react";++import { useStore } from "metabase/lib/redux";+import type {+ UseCheckDependenciesProps,+ UseCheckDependenciesResult,+} from "metabase/plugins";+import { getSubmittableQuestion } from "metabase/query_builder/selectors";+import { useLazyCheckCardDependenciesQuery } from "metabase-enterprise/api";+import type Question from "metabase-lib/v1/Question";+import type { CheckCardDependenciesRequest } from "metabase-types/api";++import { useCheckDependencies } from "../use-check-dependencies";++export function useCheckCardDependencies({+ onSave,+ onError,+}: UseCheckDependenciesProps<Question>): UseCheckDependenciesResult<Question> {+ const store = useStore();++ const getCheckDependenciesRequest = useCallback(+ (question: Question): CheckCardDependenciesRequest => {+ const submittableQuestion = getSubmittableQuestion(+ store.getState(),+ question,+ );+ const { id, type, dataset_query, result_metadata } =+ submittableQuestion.card();+ return { id, type, dataset_query, result_metadata };+ },+ [store],+ );++ return useCheckDependencies({+ getCheckDependenciesRequest,+ useLazyCheckDependenciesQuery: useLazyCheckCardDependenciesQuery,+ onSave,+ onError,+ });+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.unit.spec.tsnew file mode 100644index 00000000000..80f6242dc1c--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.unit.spec.ts@@ -0,0 +1,77 @@+import { setupCheckCardDependenciesEndpoint } from "__support__/server-mocks/dependencies";+import { act, renderHookWithProviders } from "__support__/ui";+import { SAMPLE_METADATA } from "metabase-lib/test-helpers";+import Question from "metabase-lib/v1/Question";+import type { CheckDependenciesResponse } from "metabase-types/api";+import { createMockCheckDependenciesResponse } from "metabase-types/api/mocks";++import { useCheckCardDependencies } from "./use-check-card-dependencies";++type SetupOpts = {+ response?: CheckDependenciesResponse;+};++function setup({+ response = createMockCheckDependenciesResponse(),+}: SetupOpts = {}) {+ setupCheckCardDependenciesEndpoint(response);++ const onSave = jest.fn();+ const onError = jest.fn();+ const { result } = renderHookWithProviders(+ () => useCheckCardDependencies({ onSave, onError }),+ {},+ );++ return { result, onSave, onError };+}++describe("useCheckCardDependencies", () => {+ it("should save if there are no broken dependencies", async () => {+ const question = Question.create({ metadata: SAMPLE_METADATA });+ const response = createMockCheckDependenciesResponse({ success: true });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(question));++ expect(onSave).toHaveBeenCalledWith(question);+ });++ it("should not save if there are broken dependencies", async () => {+ const question = Question.create({ metadata: SAMPLE_METADATA });+ const response = createMockCheckDependenciesResponse({ success: false });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(question));++ expect(result.current.checkData).toEqual(response);+ expect(result.current.isConfirmationShown).toBe(true);+ expect(onSave).not.toHaveBeenCalled();+ });++ it("should allow to confirm saving", async () => {+ const question = Question.create({ metadata: SAMPLE_METADATA });+ const response = createMockCheckDependenciesResponse({ success: false });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(question));+ await act(() => result.current.handleSaveAfterConfirmation());++ expect(result.current.checkData).toEqual(response);+ expect(result.current.isConfirmationShown).toBe(false);+ expect(onSave).toHaveBeenCalledWith(question);+ });++ it("should allow to cancel the confirmation screen", async () => {+ const question = Question.create({ metadata: SAMPLE_METADATA });+ const response = createMockCheckDependenciesResponse({ success: false });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(question));+ await act(() => result.current.handleCloseConfirmation());++ expect(result.current.checkData).toEqual(response);+ expect(result.current.isConfirmationShown).toBe(false);+ expect(onSave).not.toHaveBeenCalled();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/index.tsnew file mode 100644index 00000000000..db8905bf067--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/index.ts@@ -0,0 +1 @@+export * from "./use-check-dependencies";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/use-check-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/use-check-dependencies.tsnew file mode 100644index 00000000000..bc80ac71d41--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/use-check-dependencies.ts@@ -0,0 +1,80 @@+import { useCallback, useState } from "react";++import type { UseCheckDependenciesResult } from "metabase/plugins";+import type { CheckDependenciesResponse } from "metabase-types/api";++type UseCheckDependenciesQueryData = {+ data?: CheckDependenciesResponse;+ error?: unknown;+ isLoading: boolean;+};++type UseCheckDependenciesArgsData<TRequest> = {+ lastArg?: TRequest;+};++type UseCheckDependenciesQueryResult<TRequest> = [+ (request: TRequest) => Promise<UseCheckDependenciesQueryData>,+ UseCheckDependenciesQueryData,+ UseCheckDependenciesArgsData<TRequest>,+];++type UseCheckDependenciesProps<TChange, TRequest> = {+ getCheckDependenciesRequest: (change: TChange) => TRequest;+ useLazyCheckDependenciesQuery: () => UseCheckDependenciesQueryResult<TRequest>;+ onSave: (change: TChange) => Promise<void>;+ onError: (error: unknown) => void;+};++export function useCheckDependencies<TChange, TRequest>({+ getCheckDependenciesRequest,+ useLazyCheckDependenciesQuery,+ onSave,+ onError,+}: UseCheckDependenciesProps<+ TChange,+ TRequest+>): UseCheckDependenciesResult<TChange> {+ const [change, setChange] = useState<TChange | null>(null);+ const [isConfirmationShown, setIsConfirmationShown] = useState(false);+ const [checkDependencies, { data, isLoading }] =+ useLazyCheckDependenciesQuery();++ const handleInitialSave = useCallback(+ async (change: TChange) => {+ const { data, error } = await checkDependencies(+ getCheckDependenciesRequest(change),+ );+ if (error != null) {+ onError(error);+ } else if (data != null && !data.success) {+ setChange(change);+ setIsConfirmationShown(true);+ } else {+ await onSave(change);+ }+ },+ [getCheckDependenciesRequest, checkDependencies, onSave, onError],+ );++ const handleCloseConfirmation = useCallback(() => {+ setChange(null);+ setIsConfirmationShown(false);+ }, []);++ const handleSaveAfterConfirmation = useCallback(async () => {+ if (change != null) {+ await onSave(change);+ handleCloseConfirmation();+ }+ }, [change, onSave, handleCloseConfirmation]);++ return {+ checkData: data,+ isCheckingDependencies: isLoading,+ isConfirmationShown,+ handleInitialSave,+ handleSaveAfterConfirmation,+ handleCloseConfirmation,+ };+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/index.tsnew file mode 100644index 00000000000..d100a598a59--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/index.ts@@ -0,0 +1 @@+export * from "./use-check-snippet-dependencies";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.tsnew file mode 100644index 00000000000..8221931c6cf--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.ts@@ -0,0 +1,31 @@+import type {+ UseCheckDependenciesProps,+ UseCheckDependenciesResult,+} from "metabase/plugins";+import { useLazyCheckSnippetDependenciesQuery } from "metabase-enterprise/api";+import type {+ CheckSnippetDependenciesRequest,+ UpdateSnippetRequest,+} from "metabase-types/api";++import { useCheckDependencies } from "../use-check-dependencies";++export function useCheckSnippetDependencies({+ onSave,+ onError,+}: UseCheckDependenciesProps<UpdateSnippetRequest>): UseCheckDependenciesResult<UpdateSnippetRequest> {+ return useCheckDependencies({+ getCheckDependenciesRequest,+ useLazyCheckDependenciesQuery: useLazyCheckSnippetDependenciesQuery,+ onSave,+ onError,+ });+}++function getCheckDependenciesRequest({+ id,+ name,+ content,+}: UpdateSnippetRequest): CheckSnippetDependenciesRequest {+ return { id, name, content };+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.unit.spec.tsnew file mode 100644index 00000000000..6b0c28be68c--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.unit.spec.ts@@ -0,0 +1,52 @@+import { setupCheckSnippetDependenciesEndpoint } from "__support__/server-mocks/dependencies";+import { act, renderHookWithProviders } from "__support__/ui";+import type { CheckDependenciesResponse } from "metabase-types/api";+import {+ createMockCheckDependenciesResponse,+ createMockUpdateSnippetRequest,+} from "metabase-types/api/mocks";++import { useCheckSnippetDependencies } from "./use-check-snippet-dependencies";++type SetupOpts = {+ response?: CheckDependenciesResponse;+};++function setup({+ response = createMockCheckDependenciesResponse(),+}: SetupOpts = {}) {+ setupCheckSnippetDependenciesEndpoint(response);++ const onSave = jest.fn();+ const onError = jest.fn();+ const { result } = renderHookWithProviders(+ () => useCheckSnippetDependencies({ onSave, onError }),+ {},+ );++ return { result, onSave, onError };+}++describe("useCheckSnippetDependencies", () => {+ it("should save if there are no broken dependencies", async () => {+ const request = createMockUpdateSnippetRequest();+ const response = createMockCheckDependenciesResponse({ success: true });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(request));++ expect(onSave).toHaveBeenCalledWith(request);+ });++ it("should not save if there are broken dependencies", async () => {+ const request = createMockUpdateSnippetRequest();+ const response = createMockCheckDependenciesResponse({ success: false });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(request));++ expect(result.current.checkData).toEqual(response);+ expect(result.current.isConfirmationShown).toBe(true);+ expect(onSave).not.toHaveBeenCalled();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/index.tsnew file mode 100644index 00000000000..8b2fd65fcaa--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/index.ts@@ -0,0 +1 @@+export * from "./use-check-transform-dependencies";diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.tsnew file mode 100644index 00000000000..fb55a754d46--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.ts@@ -0,0 +1,30 @@+import type {+ UseCheckDependenciesProps,+ UseCheckDependenciesResult,+} from "metabase/plugins";+import { useLazyCheckTransformDependenciesQuery } from "metabase-enterprise/api";+import type {+ CheckTransformDependenciesRequest,+ UpdateTransformRequest,+} from "metabase-types/api";++import { useCheckDependencies } from "../use-check-dependencies";++export function useCheckTransformDependencies({+ onSave,+ onError,+}: UseCheckDependenciesProps<UpdateTransformRequest>): UseCheckDependenciesResult<UpdateTransformRequest> {+ return useCheckDependencies({+ getCheckDependenciesRequest,+ useLazyCheckDependenciesQuery: useLazyCheckTransformDependenciesQuery,+ onSave,+ onError,+ });+}++function getCheckDependenciesRequest({+ id,+ source,+}: UpdateTransformRequest): CheckTransformDependenciesRequest {+ return { id, source };+}diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.unit.spec.tsnew file mode 100644index 00000000000..19276764a62--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.unit.spec.ts@@ -0,0 +1,52 @@+import { setupCheckTransformDependenciesEndpoint } from "__support__/server-mocks/dependencies";+import { act, renderHookWithProviders } from "__support__/ui";+import type { CheckDependenciesResponse } from "metabase-types/api";+import {+ createMockCheckDependenciesResponse,+ createMockUpdateTransformRequest,+} from "metabase-types/api/mocks";++import { useCheckTransformDependencies } from "./use-check-transform-dependencies";++type SetupOpts = {+ response?: CheckDependenciesResponse;+};++function setup({+ response = createMockCheckDependenciesResponse(),+}: SetupOpts = {}) {+ setupCheckTransformDependenciesEndpoint(response);++ const onSave = jest.fn();+ const onError = jest.fn();+ const { result } = renderHookWithProviders(+ () => useCheckTransformDependencies({ onSave, onError }),+ {},+ );++ return { result, onSave, onError };+}++describe("useCheckTransformDependencies", () => {+ it("should save if there are no broken dependencies", async () => {+ const request = createMockUpdateTransformRequest();+ const response = createMockCheckDependenciesResponse({ success: true });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(request));++ expect(onSave).toHaveBeenCalledWith(request);+ });++ it("should not save if there are broken dependencies", async () => {+ const request = createMockUpdateTransformRequest();+ const response = createMockCheckDependenciesResponse({ success: false });+ const { result, onSave } = setup({ response });++ await act(() => result.current.handleInitialSave(request));++ expect(result.current.checkData).toEqual(response);+ expect(result.current.isConfirmationShown).toBe(true);+ expect(onSave).not.toHaveBeenCalled();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/dependencies/index.ts b/enterprise/frontend/src/metabase-enterprise/dependencies/index.tsnew file mode 100644index 00000000000..3e103cc4c9d--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/dependencies/index.ts@@ -0,0 +1,19 @@+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";+import { hasPremiumFeature } from "metabase-enterprise/settings";++import { CheckDependenciesForm } from "./components/CheckDependenciesForm";+import { CheckDependenciesModal } from "./components/CheckDependenciesModal";+import { CheckDependenciesTitle } from "./components/CheckDependenciesTitle";+import { useCheckCardDependencies } from "./hooks/use-check-card-dependencies";+import { useCheckSnippetDependencies } from "./hooks/use-check-snippet-dependencies";+import { useCheckTransformDependencies } from "./hooks/use-check-transform-dependencies";++if (hasPremiumFeature("dependencies")) {+ PLUGIN_DEPENDENCIES.CheckDependenciesForm = CheckDependenciesForm;+ PLUGIN_DEPENDENCIES.CheckDependenciesModal = CheckDependenciesModal;+ PLUGIN_DEPENDENCIES.CheckDependenciesTitle = CheckDependenciesTitle;+ PLUGIN_DEPENDENCIES.useCheckCardDependencies = useCheckCardDependencies;+ PLUGIN_DEPENDENCIES.useCheckSnippetDependencies = useCheckSnippetDependencies;+ PLUGIN_DEPENDENCIES.useCheckTransformDependencies =+ useCheckTransformDependencies;+}diff --git a/enterprise/frontend/src/metabase-enterprise/plugins.js b/enterprise/frontend/src/metabase-enterprise/plugins.jsindex 0a676e1f976..329d6bca338 100644--- a/enterprise/frontend/src/metabase-enterprise/plugins.js+++ b/enterprise/frontend/src/metabase-enterprise/plugins.js@@ -39,6 +39,7 @@ import "./clean_up";import "./metabot";import "./database_replication";import "./table-editing";+import "./dependencies";import "./documents";import "./semantic_search";import "./transforms";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsxindex 87fac2e71e8..7a50a218344 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx@@ -6,7 +6,10 @@ import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrimport { useDispatch } from "metabase/lib/redux";import * as Urls from "metabase/lib/urls";import { useMetadataToasts } from "metabase/metadata/hooks";-import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";+import {+ PLUGIN_DEPENDENCIES,+ PLUGIN_TRANSFORMS_PYTHON,+} from "metabase/plugins";import {useGetTransformQuery,useUpdateTransformMutation,@@ -50,22 +53,38 @@ type TransformQueryPageBodyProps = {export function TransformQueryPageBody({transform,}: TransformQueryPageBodyProps) {- const [updateTransform, { isLoading }] = useUpdateTransformMutation();+ const [updateTransform, { isLoading: isSaving }] =+ useUpdateTransformMutation();const dispatch = useDispatch();- const { sendErrorToast, sendSuccessToast } = useMetadataToasts();+ const { sendSuccessToast, sendErrorToast } = useMetadataToasts();- const handleSourceSave = async (source: TransformSource) => {- const { error } = await updateTransform({+ const {+ checkData,+ isCheckingDependencies,+ isConfirmationShown,+ handleInitialSave,+ handleSaveAfterConfirmation,+ handleCloseConfirmation,+ } = PLUGIN_DEPENDENCIES.useCheckTransformDependencies({+ onSave: async (request) => {+ const { error } = await updateTransform(request);+ if (error) {+ sendErrorToast(t`Failed to update transform query`);+ } else {+ sendSuccessToast(t`Transform query updated`);+ dispatch(push(getTransformUrl(transform.id)));+ }+ },+ onError: () => {+ sendErrorToast(t`Failed to update transform query`);+ },+ });++ const handleSaveSource = async (source: TransformSource) => {+ await handleInitialSave({id: transform.id,source,});-- if (error) {- sendErrorToast(t`Failed to update transform query`);- } else {- sendSuccessToast(t`Transform query updated`);- dispatch(push(getTransformUrl(transform.id)));- }};const handleCancel = () => {@@ -77,19 +96,29 @@ export function TransformQueryPageBody({<PLUGIN_TRANSFORMS_PYTHON.TransformEditorinitialSource={transform.source}isNew={false}- isSaving={isLoading}- onSave={handleSourceSave}+ isSaving={isSaving}+ onSave={handleSaveSource}onCancel={handleCancel}/>);}return (- <QueryEditor- initialSource={transform.source}- isNew={false}- isSaving={isLoading}- onSave={handleSourceSave}- onCancel={handleCancel}- />+ <>+ <QueryEditor+ initialSource={transform.source}+ isNew={false}+ isSaving={isSaving || isCheckingDependencies}+ onSave={handleSaveSource}+ onCancel={handleCancel}+ />+ {isConfirmationShown && checkData != null && (+ <PLUGIN_DEPENDENCIES.CheckDependenciesModal+ checkData={checkData}+ opened+ onSave={handleSaveAfterConfirmation}+ onClose={handleCloseConfirmation}+ />+ )}+ </>);}diff --git a/frontend/src/metabase-types/api/dependencies.ts b/frontend/src/metabase-types/api/dependencies.tsnew file mode 100644index 00000000000..968cedea519--- /dev/null+++ b/frontend/src/metabase-types/api/dependencies.ts@@ -0,0 +1,19 @@+import type { NativeQuerySnippet } from "metabase-types/api/snippets";++import type { Card } from "./card";+import type { Transform } from "./transform";++export type CheckDependenciesResponse = {+ success: boolean;+ bad_cards?: Card[];+ bad_transforms?: Transform[];+};++export type CheckCardDependenciesRequest = Pick<Card, "id"> &+ Partial<Pick<Card, "type" | "dataset_query" | "result_metadata">>;++export type CheckSnippetDependenciesRequest = Pick<NativeQuerySnippet, "id"> &+ Partial<Pick<NativeQuerySnippet, "name" | "content">>;++export type CheckTransformDependenciesRequest = Pick<Transform, "id"> &+ Partial<Pick<Transform, "source">>;diff --git a/frontend/src/metabase-types/api/index.ts b/frontend/src/metabase-types/api/index.tsindex 8b7fb609c6b..882c2b7fc3b 100644--- a/frontend/src/metabase-types/api/index.ts+++ b/frontend/src/metabase-types/api/index.ts@@ -16,6 +16,7 @@ export * from "./content-translation";export * from "./dashboard";export * from "./database";export * from "./dataset";+export * from "./dependencies";export * from "./email";export * from "./entity-id";export * from "./field";diff --git a/frontend/src/metabase-types/api/mocks/dependencies.ts b/frontend/src/metabase-types/api/mocks/dependencies.tsnew file mode 100644index 00000000000..9552e424e4d--- /dev/null+++ b/frontend/src/metabase-types/api/mocks/dependencies.ts@@ -0,0 +1,42 @@+import type {+ CheckCardDependenciesRequest,+ CheckDependenciesResponse,+ CheckSnippetDependenciesRequest,+ CheckTransformDependenciesRequest,+} from "metabase-types/api";++export function createMockCheckCardDependenciesRequest(+ opts?: Partial<CheckCardDependenciesRequest>,+): CheckCardDependenciesRequest {+ return {+ id: 1,+ ...opts,+ };+}++export function createMockCheckSnippetDependenciesRequest(+ opts?: Partial<CheckSnippetDependenciesRequest>,+): CheckSnippetDependenciesRequest {+ return {+ id: 1,+ ...opts,+ };+}++export function createMockCheckTransformDependenciesRequest(+ opts?: Partial<CheckTransformDependenciesRequest>,+): CheckTransformDependenciesRequest {+ return {+ id: 1,+ ...opts,+ };+}++export function createMockCheckDependenciesResponse(+ opts?: Partial<CheckDependenciesResponse>,+): CheckDependenciesResponse {+ return {+ success: true,+ ...opts,+ };+}diff --git a/frontend/src/metabase-types/api/mocks/index.ts b/frontend/src/metabase-types/api/mocks/index.tsindex 310b153c05a..344e437eee7 100644--- a/frontend/src/metabase-types/api/mocks/index.ts+++ b/frontend/src/metabase-types/api/mocks/index.ts@@ -9,6 +9,7 @@ export * from "./collection";export * from "./dashboard";export * from "./database";export * from "./dataset";+export * from "./dependencies";export * from "./document";export * from "./field";export * from "./geojson";diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.tsindex fc1a8e7b88f..98c57267de4 100644--- a/frontend/src/metabase-types/api/mocks/settings.ts+++ b/frontend/src/metabase-types/api/mocks/settings.ts@@ -101,6 +101,7 @@ export const createMockTokenFeatures = (advanced_permissions: false,audit_app: false,cache_granular_controls: false,+ dependencies: false,disable_password_login: false,cloud_custom_smtp: false,content_translation: false,diff --git a/frontend/src/metabase-types/api/mocks/snippets.ts b/frontend/src/metabase-types/api/mocks/snippets.tsindex ba9246beb24..68a4b4c4132 100644--- a/frontend/src/metabase-types/api/mocks/snippets.ts+++ b/frontend/src/metabase-types/api/mocks/snippets.ts@@ -1,4 +1,7 @@-import type { NativeQuerySnippet } from "metabase-types/api";+import type {+ NativeQuerySnippet,+ UpdateSnippetRequest,+} from "metabase-types/api";import { createMockEntityId } from "./entity-id";import { createMockUser } from "./user";@@ -22,3 +25,12 @@ export const createMockNativeQuerySnippet = ({updated_at: new Date().toISOString(),...opts,});++export function createMockUpdateSnippetRequest(+ opts?: Partial<UpdateSnippetRequest>,+): UpdateSnippetRequest {+ return {+ id: 1,+ ...opts,+ };+}diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.tsindex ef8c1333be4..c5651f120ed 100644--- a/frontend/src/metabase-types/api/mocks/transform.ts+++ b/frontend/src/metabase-types/api/mocks/transform.ts@@ -7,6 +7,7 @@ import type {TransformSource,TransformTag,TransformTarget,+ UpdateTransformRequest,} from "metabase-types/api";import { createMockStructuredDatasetQuery } from "./query";@@ -100,3 +101,12 @@ export function createMockTransformJob(...opts,};}++export function createMockUpdateTransformRequest(+ opts?: Partial<UpdateTransformRequest>,+): UpdateTransformRequest {+ return {+ id: 1,+ ...opts,+ };+}diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex 035b0290445..cb9d4d39387 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -321,6 +321,7 @@ export const tokenFeatures = ["etl_connections","etl_connections_pg","table_data_editing",+ "dependencies","documents","semantic_search","transforms",diff --git a/frontend/src/metabase-types/api/snippets.ts b/frontend/src/metabase-types/api/snippets.tsindex 1733a0a174d..e81bb9e7d7e 100644--- a/frontend/src/metabase-types/api/snippets.ts+++ b/frontend/src/metabase-types/api/snippets.ts@@ -25,17 +25,17 @@ export interface ListSnippetsParams {}export interface CreateSnippetRequest {- content: string;name: string;- description?: string;+ content: string;+ description?: string | null;collection_id?: RegularCollectionId | null;}export interface UpdateSnippetRequest {id: NativeQuerySnippetId;- archived?: boolean;- content?: string | null;- name?: string | null;+ name?: string;description?: string | null;+ content?: string;collection_id?: RegularCollectionId | null;+ archived?: boolean;}diff --git a/frontend/src/metabase/common/components/SaveQuestionForm/types.ts b/frontend/src/metabase/common/components/SaveQuestionForm/types.tsindex 2ac9148d41b..4bd6f4c5021 100644--- a/frontend/src/metabase/common/components/SaveQuestionForm/types.ts+++ b/frontend/src/metabase/common/components/SaveQuestionForm/types.ts@@ -15,6 +15,7 @@ export type SaveQuestionProps<C = CollectionId> = {},) => Promise<Question>;onSave: (question: Question) => Promise<void>;+ onCancel?: () => void;closeOnSuccess?: boolean;multiStep?: boolean;diff --git a/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx b/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsxindex 2e3ce0d4286..48332900d26 100644--- a/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx+++ b/frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx@@ -5,6 +5,7 @@ import {import { SaveQuestionProvider } from "metabase/common/components/SaveQuestionForm/context";import type { SaveQuestionProps } from "metabase/common/components/SaveQuestionForm/types";import { useEscapeToCloseModal } from "metabase/common/hooks/use-escape-to-close-modal";+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";import { Flex, Modal, type ModalProps } from "metabase/ui";type SaveQuestionModalProps = Omit<SaveQuestionProps, "initialDashboardTabId"> &@@ -14,6 +15,7 @@ export const SaveQuestionModal = ({multiStep,onCreate,onSave,+ onClose,originalQuestion,question,closeOnSuccess,@@ -21,7 +23,19 @@ export const SaveQuestionModal = ({targetCollection,...modalProps}: SaveQuestionModalProps) => {- useEscapeToCloseModal(modalProps.onClose);+ const {+ checkData,+ isConfirmationShown,+ handleInitialSave,+ handleSaveAfterConfirmation,+ } = PLUGIN_DEPENDENCIES.useCheckCardDependencies({+ onSave,+ onError: (error) => {+ throw error;+ },+ });+ useEscapeToCloseModal(onClose);+return (<SaveQuestionProviderquestion={question}@@ -30,32 +44,51 @@ export const SaveQuestionModal = ({const newQuestion = await onCreate(question, options);if (closeOnSuccess) {- modalProps.onClose();+ onClose();}return newQuestion;}}- onSave={onSave}+ onSave={handleInitialSave}+ onCancel={onClose}multiStep={multiStep}initialCollectionId={initialCollectionId}targetCollection={targetCollection}>- <Modal.Root padding="2.5rem" {...modalProps} closeOnEscape={false}>+ <Modal.Root+ padding="xl"+ {...modalProps}+ size={isConfirmationShown ? "xl" : undefined}+ closeOnEscape={false}+ onClose={onClose}+ ><Modal.Overlay /><Modal.Content data-testid="save-question-modal">- <Modal.Header>+ <Modal.Header px={isConfirmationShown ? "xl" : undefined}><Modal.Title>- <SaveQuestionTitle />+ {isConfirmationShown ? (+ <PLUGIN_DEPENDENCIES.CheckDependenciesTitle />+ ) : (+ <SaveQuestionTitle />+ )}</Modal.Title>- <Flex align="center" justify="flex-end" gap="sm">+ <Flex justify="flex-end"><Modal.CloseButton /></Flex></Modal.Header>- <Modal.Body>- <SaveQuestionForm- onSaveSuccess={() => closeOnSuccess && modalProps.onClose()}- onCancel={modalProps.onClose}- />+ <Modal.Body px={isConfirmationShown ? 0 : undefined}>+ {checkData != null && isConfirmationShown ? (+ <PLUGIN_DEPENDENCIES.CheckDependenciesForm+ checkData={checkData}+ onSave={handleSaveAfterConfirmation}+ onCancel={onClose}+ />+ ) : (+ <SaveQuestionForm+ onSaveSuccess={() => closeOnSuccess && onClose()}+ onCancel={onClose}+ />+ )}</Modal.Body></Modal.Content></Modal.Root>diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.tsindex a940c7f5731..7677c7e36b4 100644--- a/frontend/src/metabase/plugins/index.ts+++ b/frontend/src/metabase/plugins/index.ts@@ -58,6 +58,7 @@ import type {Bookmark,CacheableDashboard,CacheableModel,+ CheckDependenciesResponse,Collection,CollectionAuthorityLevelConfig,CollectionEssentials,@@ -86,6 +87,8 @@ import type {Timeline,TimelineEvent,Transform,+ UpdateSnippetRequest,+ UpdateTransformRequest,User,VisualizationDisplay,} from "metabase-types/api";@@ -871,3 +874,66 @@ export const PLUGIN_TRANSFORMS_PYTHON: PythonTransformsPlugin = {getTransformsNavLinks: () => null,getCreateTransformsMenuItems: () => null,};++type DependenciesPlugin = {+ CheckDependenciesForm: ComponentType<CheckDependenciesFormProps>;+ CheckDependenciesModal: ComponentType<CheckDependenciesModalProps>;+ CheckDependenciesTitle: ComponentType;+ useCheckCardDependencies: (+ props: UseCheckDependenciesProps<Question>,+ ) => UseCheckDependenciesResult<Question>;+ useCheckSnippetDependencies: (+ props: UseCheckDependenciesProps<UpdateSnippetRequest>,+ ) => UseCheckDependenciesResult<UpdateSnippetRequest>;+ useCheckTransformDependencies: (+ props: UseCheckDependenciesProps<UpdateTransformRequest>,+ ) => UseCheckDependenciesResult<UpdateTransformRequest>;+};++export type CheckDependenciesFormProps = {+ checkData: CheckDependenciesResponse;+ onSave: () => void | Promise<void>;+ onCancel: () => void;+};++export type CheckDependenciesModalProps = {+ checkData: CheckDependenciesResponse;+ opened: boolean;+ onSave: () => void | Promise<void>;+ onClose: () => void;+};++export type UseCheckDependenciesProps<TChange> = {+ onSave: (change: TChange) => Promise<void>;+ onError: (error: unknown) => void;+};++export type UseCheckDependenciesResult<TChange> = {+ checkData?: CheckDependenciesResponse;+ isCheckingDependencies: boolean;+ isConfirmationShown: boolean;+ handleInitialSave: (change: TChange) => Promise<void>;+ handleSaveAfterConfirmation: () => Promise<void>;+ handleCloseConfirmation: () => void;+};++function useCheckDependencies<TChange>({+ onSave,+}: UseCheckDependenciesProps<TChange>): UseCheckDependenciesResult<TChange> {+ return {+ isConfirmationShown: false,+ isCheckingDependencies: false,+ handleInitialSave: onSave,+ handleSaveAfterConfirmation: () => Promise.resolve(),+ handleCloseConfirmation: () => undefined,+ };+}++export const PLUGIN_DEPENDENCIES: DependenciesPlugin = {+ CheckDependenciesForm: PluginPlaceholder,+ CheckDependenciesModal: PluginPlaceholder,+ CheckDependenciesTitle: PluginPlaceholder,+ useCheckCardDependencies: useCheckDependencies,+ useCheckSnippetDependencies: useCheckDependencies,+ useCheckTransformDependencies: useCheckDependencies,+};diff --git a/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsx b/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsxindex e8e0aa53dd3..5eb24df3f43 100644--- a/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsx+++ b/frontend/src/metabase/query_builder/components/DatasetEditor/DatasetEditorInner.tsx@@ -5,14 +5,18 @@ import {type ReactNode,useCallback,useEffect,+ useLayoutEffect,useMemo,+ useRef,useState,} from "react";import { useMount, usePrevious } from "react-use";import { t } from "ttag";import { useListModelIndexesQuery } from "metabase/api";-import ActionButton from "metabase/common/components/ActionButton";+import ActionButton, {+ type ActionButtonHandle,+} from "metabase/common/components/ActionButton";import Button from "metabase/common/components/Button";import DebouncedFrame from "metabase/common/components/DebouncedFrame";import EditBar from "metabase/common/components/EditBar";@@ -21,6 +25,7 @@ import ButtonsS from "metabase/css/components/buttons.module.css";import CS from "metabase/css/core/index.css";import { connect, useDispatch } from "metabase/lib/redux";import { getSemanticTypeIcon } from "metabase/lib/schema_metadata";+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";import {setDatasetEditorTab,setUIControls,@@ -474,6 +479,28 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {}};+ const saveButtonRef = useRef<ActionButtonHandle>(null);+ const {+ checkData,+ isConfirmationShown,+ handleInitialSave,+ handleSaveAfterConfirmation,+ handleCloseConfirmation,+ } = PLUGIN_DEPENDENCIES.useCheckCardDependencies({+ onSave: async (question) => {+ await onSave(question, { rerunQuery: true });+ await setQueryBuilderMode("view");+ runQuestionQuery();+ },+ onError: (error) => {+ throw error;+ },+ });++ useLayoutEffect(() => {+ saveButtonRef.current?.resetState();+ }, [isConfirmationShown]);+const handleSave = useCallback(async () => {const canBeDataset = checkCanBeModel(question);const isBrandNewDataset = !question.id();@@ -495,9 +522,7 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {});onOpenModal(MODAL_TYPES.SAVE);} else if (canBeDataset) {- await onSave(questionWithMetadata, { rerunQuery: true });- await setQueryBuilderMode("view");- runQuestionQuery();+ await handleInitialSave(questionWithMetadata);} else {onOpenModal(MODAL_TYPES.CAN_NOT_CREATE_MODEL);throw new Error(t`Variables in models aren't supported yet`);@@ -510,9 +535,7 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {dispatch,updateQuestion,onOpenModal,- onSave,- setQueryBuilderMode,- runQuestionQuery,+ handleInitialSave,]);const handleColumnSelect = useCallback(@@ -670,6 +693,7 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {disabled={!saveButtonTooltipLabel}><ActionButton+ ref={saveButtonRef}key="save"disabled={!canSaveChanges}actionFn={handleSave}@@ -744,6 +768,15 @@ const _DatasetEditorInner = (props: DatasetEditorInnerProps) => {onConfirm={handleCancelEdit}onClose={closeModal}/>++ {isConfirmationShown && checkData != null && (+ <PLUGIN_DEPENDENCIES.CheckDependenciesModal+ checkData={checkData}+ opened+ onSave={handleSaveAfterConfirmation}+ onClose={handleCloseConfirmation}+ />+ )}</>);};diff --git a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsxindex 497db4ab688..970e8213331 100644--- a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx+++ b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx@@ -10,7 +10,6 @@ import { ResizableBox, type ResizableBoxProps } from "react-resizable";import _ from "underscore";import ExplicitSize from "metabase/common/components/ExplicitSize";-import Modal from "metabase/common/components/Modal";import Databases from "metabase/entities/databases";import SnippetCollections from "metabase/entities/snippet-collections";import Snippets from "metabase/entities/snippets";@@ -20,7 +19,7 @@ import {setIsNativeEditorOpen,setUIControls,} from "metabase/query_builder/actions";-import SnippetFormModal from "metabase/query_builder/components/template_tags/SnippetFormModal";+import { SnippetFormModal } from "metabase/query_builder/components/template_tags/SnippetFormModal";import type { QueryModalType } from "metabase/query_builder/constants";import { useNotebookScreenSize } from "metabase/query_builder/hooks/use-notebook-screen-size";import { Flex } from "metabase/ui";@@ -372,8 +371,9 @@ class NativeQueryEditor extends Component<Props, NativeQueryEditorState> {canSaveSnippets={canSaveSnippets}/>- {this.props.modalSnippet && (- <Modal onClose={this.props.closeSnippetModal}>+ {this.props.modalSnippet &&+ this.props.insertSnippet &&+ this.props.closeSnippetModal && (<SnippetFormModalsnippet={this.props.modalSnippet}onCreate={this.props.insertSnippet}@@ -387,8 +387,7 @@ class NativeQueryEditor extends Component<Props, NativeQueryEditorState> {}}onClose={this.props.closeSnippetModal}/>- </Modal>- )}+ )}</div>);}diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsx b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsxindex 6a6b99db2eb..653d668dd90 100644--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsx+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/SnippetForm.tsx@@ -1,26 +1,19 @@-import { useCallback, useMemo } from "react";+import { useMemo } from "react";import { t } from "ttag";import _ from "underscore";import * as Yup from "yup";import FormCollectionPicker from "metabase/collections/containers/FormCollectionPicker";-import { canonicalCollectionId } from "metabase/collections/utils";import Button from "metabase/common/components/Button";import FormErrorMessage from "metabase/common/components/FormErrorMessage";import FormInput from "metabase/common/components/FormInput";import FormSubmitButton from "metabase/common/components/FormSubmitButton";import FormTextArea from "metabase/common/components/FormTextArea";import SnippetCollections from "metabase/entities/snippet-collections";-import Snippets from "metabase/entities/snippets";import { Form, FormProvider } from "metabase/forms";import * as Errors from "metabase/lib/errors";-import { connect } from "metabase/lib/redux";import { Flex } from "metabase/ui";-import type {- Collection,- NativeQuerySnippet,- NativeQuerySnippetId,-} from "metabase-types/api";+import type { Collection, NativeQuerySnippet } from "metabase-types/api";import S from "./SnippetForm.module.css";@@ -37,60 +30,34 @@ const SNIPPET_SCHEMA = Yup.object({collection_id: Yup.number().nullable().default(null),});-type SnippetFormValues = Pick<+export type SnippetFormValues = Pick<NativeQuerySnippet,"name" | "description" | "content" | "collection_id">;-type UpdateSnippetFormValues = Partial<SnippetFormValues> &- Pick<NativeQuerySnippet, "id"> & {- archived?: boolean;- };-export interface SnippetFormOwnProps {- snippet: Partial<NativeQuerySnippet>;- onCreate?: (snippet: NativeQuerySnippet) => void;- onUpdate?: (- nextSnippet: NativeQuerySnippet,- originalSnippet: NativeQuerySnippet,- ) => void;- onArchive?: () => void;+ snippet?: Partial<SnippetFormValues>;+ isEditing: boolean;+ isDirty?: boolean;+ onSubmit: (snippet: SnippetFormValues) => void | Promise<void>;+ onArchive?: () => void | Promise<void>;onCancel?: () => void;}interface SnippetLoaderProps {snippetCollections: Collection[];}--interface SnippetFormDispatchProps {- handleCreateSnippet: (- snippet: SnippetFormValues,- ) => Promise<NativeQuerySnippet>;- handleUpdateSnippet: (- snippet: UpdateSnippetFormValues,- ) => Promise<NativeQuerySnippet>;-}--type SnippetFormProps = SnippetFormOwnProps &- SnippetLoaderProps &- SnippetFormDispatchProps;--const mapDispatchToProps = {- handleCreateSnippet: Snippets.actions.create,- handleUpdateSnippet: Snippets.actions.update,-};+type SnippetFormProps = SnippetFormOwnProps & SnippetLoaderProps;function SnippetForm({snippet,snippetCollections,- handleCreateSnippet,- handleUpdateSnippet,- onCreate,- onUpdate,+ isEditing,+ isDirty: isInitiallyDirty = false,+ onSubmit,onArchive,onCancel,}: SnippetFormProps) {- const isEditing = snippet.id != null;const hasManyCollections = snippetCollections.length > 1;const initialValues = useMemo(@@ -98,65 +65,26 @@ function SnippetForm({SNIPPET_SCHEMA.cast({...snippet,- content: snippet.content || "",- parent_id: canonicalCollectionId(snippet.id),+ content: snippet?.content || "",},{ stripUnknown: true },),[snippet],);- const handleCreate = useCallback(- async (values: SnippetFormValues) => {- const action = await handleCreateSnippet(values);- const snippet = Snippets.HACK_getObjectFromAction(action);- onCreate?.(snippet);- },- [handleCreateSnippet, onCreate],- );-- const handleUpdate = useCallback(- async (values: UpdateSnippetFormValues) => {- const action = await handleUpdateSnippet(values);- const nextSnippet = Snippets.HACK_getObjectFromAction(action);- onUpdate?.(nextSnippet, snippet as NativeQuerySnippet);- },- [snippet, handleUpdateSnippet, onUpdate],- );-- const handleSubmit = useCallback(- async (values: SnippetFormValues) => {- if (isEditing && snippet.id) {- await handleUpdate({ ...values, id: snippet.id });- } else {- await handleCreate(values);- }- },- [snippet.id, isEditing, handleCreate, handleUpdate],- );-- const handleArchive = useCallback(async () => {- await handleUpdateSnippet({- id: snippet.id as NativeQuerySnippetId,- archived: true,- });- onArchive?.();- }, [snippet.id, handleUpdateSnippet, onArchive]);-return (<FormProviderinitialValues={initialValues}validationSchema={SNIPPET_SCHEMA}- onSubmit={handleSubmit}+ onSubmit={onSubmit}>{({ dirty }) => (- <Form disabled={!dirty} className={S.SnippetForm}>+ <Form disabled={!dirty && !isInitiallyDirty} className={S.SnippetForm}><FormTextAreainputClassName={S.FormSnippetTextArea}name="content"title={t`Enter some SQL here so you can reuse it later`}placeholder="AND canceled_at IS null\nAND account_type = 'PAID'"- autoFocusrows={4}/><FormInput@@ -184,8 +112,10 @@ function SnippetForm({type="button"icon="archive"borderless- onClick={handleArchive}- >{t`Archive`}</Button>+ onClick={onArchive}+ >+ {t`Archive`}+ </Button>)}<FormErrorMessage inline /></Flex>@@ -193,7 +123,11 @@ function SnippetForm({{!!onCancel && (<Button type="button" onClick={onCancel}>{t`Cancel`}</Button>)}- <FormSubmitButton title={t`Save`} disabled={!dirty} primary />+ <FormSubmitButton+ title={t`Save`}+ disabled={!dirty && !isInitiallyDirty}+ primary+ /></Flex></Flex></Form>@@ -203,7 +137,4 @@ function SnippetForm({}// eslint-disable-next-line import/no-default-export -- deprecated usage-export default _.compose(- SnippetCollections.loadList(),- connect(null, mapDispatchToProps),-)(SnippetForm);+export default _.compose(SnippetCollections.loadList())(SnippetForm);diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.ts b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.tsindex d01b3b192e1..064e4847c96 100644--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.ts+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetForm/index.ts@@ -1,3 +1,3 @@// eslint-disable-next-line import/no-default-export -- deprecated usageexport { default } from "./SnippetForm";-export type { SnippetFormOwnProps } from "./SnippetForm";+export type { SnippetFormValues } from "./SnippetForm";diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsx b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsxindex c0664fcc887..d25dea8265f 100644--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsx+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.tsx@@ -1,60 +1,138 @@-import { useCallback } from "react";+import { useCallback, useState } from "react";import { t } from "ttag";-import ModalContent from "metabase/common/components/ModalContent";-import type { NativeQuerySnippet } from "metabase-types/api";+import Snippets from "metabase/entities/snippets";+import { useDispatch } from "metabase/lib/redux";+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";+import { Flex, Modal } from "metabase/ui";+import type {+ CreateSnippetRequest,+ NativeQuerySnippet,+ UpdateSnippetRequest,+} from "metabase-types/api";-import type { SnippetFormOwnProps } from "../SnippetForm";-import SnippetForm from "../SnippetForm";+import SnippetForm, { type SnippetFormValues } from "../SnippetForm";-interface SnippetFormModalOwnProps- extends Omit<SnippetFormOwnProps, "onCancel"> {- onClose?: () => void;-}--type SnippetModalProps = SnippetFormModalOwnProps;+type SnippetModalProps = {+ snippet: NativeQuerySnippet | Partial<Omit<NativeQuerySnippet, "id">>;+ onCreate: (snippet: NativeQuerySnippet) => void;+ onUpdate: (+ nextSnippet: NativeQuerySnippet,+ originalSnippet: NativeQuerySnippet,+ ) => void;+ onClose: () => void;+};-function SnippetFormModal({- snippet,+export function SnippetFormModal({+ snippet: initialSnippet,onCreate,onUpdate,onClose,- ...props}: SnippetModalProps) {- const isEditing = snippet.id != null;- const title = isEditing+ const [snippet, setSnippet] = useState(initialSnippet);+ const dispatch = useDispatch();+ const isEditing = isSavedSnippet(snippet);+ const modalTitle = isEditing? t`Editing ${snippet.name}`: t`Create your new snippet`;const handleCreate = useCallback(- (snippet: NativeQuerySnippet) => {+ async (values: CreateSnippetRequest) => {+ const action = await dispatch(Snippets.actions.create(values));+ const snippet = Snippets.HACK_getObjectFromAction(action);onCreate?.(snippet);onClose?.();},- [onCreate, onClose],+ [dispatch, onCreate, onClose],);const handleUpdate = useCallback(- (nextSnippet: NativeQuerySnippet, originalSnippet: NativeQuerySnippet) => {- onUpdate?.(nextSnippet, originalSnippet);+ async (values: UpdateSnippetRequest) => {+ if (!isSavedSnippet(snippet)) {+ return;+ }+ const action = await dispatch(Snippets.actions.update(snippet, values));+ const nextSnippet = Snippets.HACK_getObjectFromAction(action);+ onUpdate?.(nextSnippet, snippet);onClose?.();},- [onUpdate, onClose],+ [snippet, dispatch, onUpdate, onClose],+ );++ const handleArchive = useCallback(async () => {+ if (!isSavedSnippet(snippet)) {+ return;+ }+ await dispatch(Snippets.actions.update({ id: snippet.id, archived: true }));+ onClose?.();+ }, [snippet, dispatch, onClose]);++ const {+ checkData,+ isConfirmationShown,+ handleInitialSave,+ handleSaveAfterConfirmation,+ handleCloseConfirmation,+ } = PLUGIN_DEPENDENCIES.useCheckSnippetDependencies({+ onSave: handleUpdate,+ onError: (error) => {+ throw error;+ },+ });++ const handleSubmit = useCallback(+ async (values: SnippetFormValues) => {+ if (isSavedSnippet(snippet)) {+ setSnippet({ ...snippet, ...values });+ await handleInitialSave({ ...values, id: snippet.id });+ } else {+ await handleCreate(values);+ }+ },+ [snippet, handleCreate, handleInitialSave],);return (- <ModalContent title={title} onClose={onClose}>- <SnippetForm- {...props}- snippet={snippet}- onCreate={handleCreate}- onUpdate={handleUpdate}- onArchive={onClose}- onCancel={onClose}- />- </ModalContent>+ <Modal.Root padding="xl" opened onClose={onClose}>+ <Modal.Overlay />+ <Modal.Content>+ <Modal.Header px="xl">+ <Modal.Title>+ {isConfirmationShown ? (+ <PLUGIN_DEPENDENCIES.CheckDependenciesTitle />+ ) : (+ modalTitle+ )}+ </Modal.Title>+ <Flex justify="flex-end">+ <Modal.CloseButton />+ </Flex>+ </Modal.Header>+ <Modal.Body px={isConfirmationShown ? 0 : "xl"}>+ {isConfirmationShown && checkData != null ? (+ <PLUGIN_DEPENDENCIES.CheckDependenciesForm+ checkData={checkData}+ onSave={handleSaveAfterConfirmation}+ onCancel={handleCloseConfirmation}+ />+ ) : (+ <SnippetForm+ snippet={snippet}+ isEditing={isEditing}+ isDirty={checkData != null}+ onSubmit={handleSubmit}+ onArchive={handleArchive}+ onCancel={onClose}+ />+ )}+ </Modal.Body>+ </Modal.Content>+ </Modal.Root>);}-// eslint-disable-next-line import/no-default-export -- deprecated usage-export default SnippetFormModal;+function isSavedSnippet(+ snippet: NativeQuerySnippet | Partial<Omit<NativeQuerySnippet, "id">>,+): snippet is NativeQuerySnippet {+ return "id" in snippet;+}diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsx b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsxindex 2df4da027ed..a8c9c1fafba 100644--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsx+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/SnippetFormModal.unit.spec.tsx@@ -13,7 +13,7 @@ import {createMockNativeQuerySnippet,} from "metabase-types/api/mocks";-import SnippetFormModal from "./SnippetFormModal";+import { SnippetFormModal } from "./SnippetFormModal";const TOP_SNIPPETS_FOLDER = {id: "root",@@ -22,15 +22,15 @@ const TOP_SNIPPETS_FOLDER = {};type SetupOpts = {- snippet?: Partial<NativeQuerySnippet>;- onClose?: null | (() => void);+ snippet?:+ | NativeQuerySnippet+ | (Omit<Partial<NativeQuerySnippet>, "id"> & { id?: undefined });withDefaultFoldersList?: boolean;};async function setup({snippet = {},withDefaultFoldersList = true,- onClose = jest.fn(),}: SetupOpts = {}) {fetchMock.get({url: "path:/api/collection/root",@@ -63,8 +63,16 @@ async function setup({);}+ const onCreate = jest.fn();+ const onUpdate = jest.fn();+ const onClose = jest.fn();renderWithProviders(- <SnippetFormModal snippet={snippet} onClose={onClose || undefined} />,+ <SnippetFormModal+ snippet={snippet}+ onCreate={onCreate}+ onUpdate={onUpdate}+ onClose={onClose}+ />,);await waitForLoaderToBeRemoved();@@ -159,13 +167,6 @@ describe("SnippetFormModal", () => {});});- it("doesn't show cancel button if onClose props is not set", async () => {- await setup({ onClose: null });- expect(- screen.queryByRole("button", { name: "Cancel" }),- ).not.toBeInTheDocument();- });-it("calls onClose when cancel button is clicked", async () => {const { onClose } = await setup();await userEvent.click(screen.getByRole("button", { name: "Cancel" }));@@ -262,13 +263,6 @@ describe("SnippetFormModal", () => {});});- it("doesn't show cancel button if onClose props is not set", async () => {- await setupEditing({ onClose: null });- expect(- screen.queryByRole("button", { name: "Cancel" }),- ).not.toBeInTheDocument();- });-it("calls onClose when cancel button is clicked", async () => {const { onClose } = await setupEditing();await userEvent.click(screen.getByRole("button", { name: "Cancel" }));diff --git a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.ts b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.tsindex d719f2e77bf..4ab22ced876 100644--- a/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.ts+++ b/frontend/src/metabase/query_builder/components/template_tags/SnippetFormModal/index.ts@@ -1,2 +1 @@-// eslint-disable-next-line import/no-default-export -- deprecated usage-export { default } from "./SnippetFormModal";+export * from "./SnippetFormModal";diff --git a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsx b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsxindex 807963b3246..667db30f687 100644--- a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsx+++ b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditor.tsx@@ -2,6 +2,7 @@ import { forwardRef, useState } from "react";import { LeaveConfirmModal } from "metabase/common/components/LeaveConfirmModal";import { SaveQuestionModal } from "metabase/common/components/SaveQuestionModal";+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";import { Flex } from "metabase/ui";import * as Lib from "metabase-lib";import type Question from "metabase-lib/v1/Question";@@ -29,7 +30,7 @@ type MetricEditorProps = {};export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(- function MetricEditorInner(+ function MetricEditor({question,result,@@ -49,6 +50,18 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>() {const [modalType, setModalType] = useState<MetricModalType>();const isRunnable = Lib.canRun(question.query(), "metric");+ const {+ checkData,+ isConfirmationShown,+ handleInitialSave,+ handleSaveAfterConfirmation,+ handleCloseConfirmation,+ } = PLUGIN_DEPENDENCIES.useCheckCardDependencies({+ onSave,+ onError: (error) => {+ throw error;+ },+ });const handleCreate = (question: Question) => {return onCreate(question.setDefaultDisplay());@@ -59,7 +72,7 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(};const handleSave = async (question: Question) => {- await onSave(question.setDefaultDisplay());+ await handleInitialSave(question.setDefaultDisplay());};const handleConfirmCancel = () => {@@ -84,6 +97,7 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(question={question}isDirty={isDirty}isRunnable={isRunnable}+ isConfirmationShown={isConfirmationShown}onCreate={handleCreateStart}onSave={handleSave}onCancel={handleCancelStart}@@ -124,6 +138,14 @@ export const MetricEditor = forwardRef<HTMLDivElement, MetricEditorProps>(onClose={handleModalClose}/>)}+ {isConfirmationShown && checkData != null && (+ <PLUGIN_DEPENDENCIES.CheckDependenciesModal+ checkData={checkData}+ opened+ onSave={handleSaveAfterConfirmation}+ onClose={handleCloseConfirmation}+ />+ )}</Flex>);},diff --git a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsx b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsxindex 8c907a71927..8581c38239d 100644--- a/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsx+++ b/frontend/src/metabase/querying/metrics/components/MetricEditor/MetricEditorHeader/MetricEditorHeader.tsx@@ -1,7 +1,10 @@import cx from "classnames";+import { useLayoutEffect, useRef } from "react";import { t } from "ttag";-import ActionButton from "metabase/common/components/ActionButton";+import ActionButton, {+ type ActionButtonHandle,+} from "metabase/common/components/ActionButton";import Button from "metabase/common/components/Button";import EditBar from "metabase/common/components/EditBar";import ButtonsS from "metabase/css/components/buttons.module.css";@@ -13,6 +16,7 @@ type MetricEditorHeaderProps = {question: Question;isDirty: boolean;isRunnable: boolean;+ isConfirmationShown: boolean;onCreate: (question: Question) => void;onSave: (question: Question) => Promise<void>;onCancel: () => void;@@ -22,13 +26,19 @@ export function MetricEditorHeader({question,isDirty,isRunnable,+ isConfirmationShown,onCreate,onSave,onCancel,}: MetricEditorHeaderProps) {+ const saveButtonRef = useRef<ActionButtonHandle>(null);const handleCreate = () => onCreate(question);const handleSave = () => onSave(question);+ useLayoutEffect(() => {+ saveButtonRef.current?.resetState();+ }, [isConfirmationShown]);+return (<EditBarclassName={S.root}@@ -41,6 +51,7 @@ export function MetricEditorHeader({</Button>) : (<ActionButton+ ref={saveButtonRef}key="save"actionFn={handleSave}disabled={!isRunnable || !isDirty}diff --git a/frontend/test/__support__/server-mocks/dependencies.ts b/frontend/test/__support__/server-mocks/dependencies.tsnew file mode 100644index 00000000000..5c7696ec4e3--- /dev/null+++ b/frontend/test/__support__/server-mocks/dependencies.ts@@ -0,0 +1,21 @@+import fetchMock from "fetch-mock";++import type { CheckDependenciesResponse } from "metabase-types/api";++export function setupCheckCardDependenciesEndpoint(+ response: CheckDependenciesResponse,+) {+ fetchMock.post("path:/api/ee/dependencies/check_card", response);+}++export function setupCheckSnippetDependenciesEndpoint(+ response: CheckDependenciesResponse,+) {+ fetchMock.post("path:/api/ee/dependencies/check_snippet", response);+}++export function setupCheckTransformDependenciesEndpoint(+ response: CheckDependenciesResponse,+) {+ fetchMock.post("path:/api/ee/dependencies/check_transform", response);+}diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex d01238d7bb5..01585a6e792 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -15,6 +15,7 @@[metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as driver.sql-jdbc][metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]+ [metabase.driver.sql.normalize :as driver.sql.normalize][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.util :as sql.u][metabase.driver.sync :as driver.s]@@ -998,7 +999,7 @@transforms (t2/select [:model/Transform :id :target])](into #{} (comp(map :component)- (map #(assoc % :table (driver.sql/normalize-name driver (:table %))))+ (map #(assoc % :table (driver.sql.normalize/normalize-name driver (:table %))))(map #(let [parts (str/split (:table %) #"\.")]{:schema (first parts) :table (second parts)}))(keep #(driver.sql/find-table-or-transform driver db-tables transforms %)))diff --git a/resources/migrations/057_update_migrations.yaml b/resources/migrations/057_update_migrations.yamlindex 6ea57204974..eed31a70b25 100644--- a/resources/migrations/057_update_migrations.yaml+++ b/resources/migrations/057_update_migrations.yaml@@ -1352,6 +1352,187 @@ databaseChangeLog:- column:name: comment_idrollback: # will be removed along the table+ - changeSet:+ id: v57.2025-09-08T17:50:00+ author: metamben+ comment: Drop old dependency table+ preConditions:+ - onFail: MARK_RAN+ - tableExists:+ tableName: dependency+ changes:+ - dropTable:+ tableName: dependency+ rollback:+ - createTable:+ tableName: dependency+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true++ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: model+ type: varchar(32)+ constraints:+ nullable: false+ - column:+ name: model_id+ type: int+ constraints:+ nullable: false+ - column:+ name: dependent_on_model+ type: varchar(32)+ constraints:+ nullable: false+ - column:+ name: dependent_on_id+ type: int+ constraints:+ nullable: false+ - column:+ name: created_at+ type: DATETIME+ constraints:+ nullable: false+ - createIndex:+ tableName: dependency+ indexName: idx_dependency_model+ columns:+ column:+ name: model+ - createIndex:+ tableName: dependency+ indexName: idx_dependency_model_id+ columns:+ column:+ name: model_id+ - createIndex:+ tableName: dependency+ indexName: idx_dependency_dependent_on_model+ columns:+ column:+ name: dependent_on_model+ - createIndex:+ tableName: dependency+ indexName: idx_dependency_dependent_on_id+ columns:+ column:+ name: dependent_on_id+ # - modifySql: # doesn't work with liquibase 4.33.0+ # dbms: postgresql+ # replace:+ # replace: WITHOUT+ # with: WITH++ - changeSet:+ id: v57.2025-09-08T18:00:00+ author: metamben+ comment: Create dependency table to track dependencies in a general way++ preConditions:+ - onFail: MARK_RAN+ - not:+ - tableExists:+ tableName: dependency+ changes:+ - createTable:+ tableName: dependency+ remarks: A table to track dependencies between Metabase entities+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true++ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: from_entity_type+ type: varchar(20)+ remarks: The type of the dependent entity+ constraints:+ nullable: false+ - column:+ name: from_entity_id+ type: int+ remarks: The ID of the dependent entity+ constraints:+ nullable: false+ - column:+ name: to_entity_type+ type: varchar(20)+ remarks: The type of the entity depended on+ constraints:+ nullable: false+ - column:+ name: to_entity_id+ type: int+ remarks: The ID of the entity depended on+ constraints:+ nullable: false++ - changeSet:+ id: v57.2025-09-08T18:10:00+ author: metamben+ comment: Unique constraint to prevent duplicate dependencies+ preConditions:+ - onFail: MARK_RAN+ - not:+ - uniqueConstraintExists:+ tableName: dependency+ constraintName: idx_unique_dependency+ changes:+ - addUniqueConstraint:+ tableName: dependency+ columnNames: from_entity_type, from_entity_id, to_entity_type, to_entity_id+ constraintName: idx_unique_dependency++ - changeSet:+ id: v57.2025-09-08T18:20:00+ author: metamben+ comment: Index for finding dependencies FROM a specific entity+ preConditions:+ - onFail: MARK_RAN+ - not:+ - indexExists:+ tableName: dependency+ indexName: idx_dependency_from+ changes:+ - createIndex:+ tableName: dependency+ indexName: idx_dependency_from+ columns:+ - column:+ name: from_entity_type+ - column:+ name: from_entity_id++ - changeSet:+ id: v57.2025-09-08T18:30:00+ author: metamben+ comment: Index for finding dependencies TO a specific entity+ preConditions:+ - onFail: MARK_RAN+ - not:+ - indexExists:+ tableName: dependency+ indexName: idx_dependency_to+ changes:+ - createIndex:+ tableName: dependency+ indexName: idx_dependency_to+ columns:+ - column:+ name: to_entity_type+ - column:+ name: to_entity_id- changeSet:id: v57.2025-09-11T10:00:00@@ -1465,6 +1646,85 @@ databaseChangeLog:constraints:nullable: true+ - changeSet:+ id: v57.2025-09-18T10:00:00+ author: metamben+ comment: Add dependency_analysis_version to report_card+ changes:+ - addColumn:+ tableName: report_card+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ remarks: Version of the dependency analysis for this entity.+ constraints:+ nullable: false+ - changeSet:+ id: v57.2025-09-18T10:01:00+ author: metamben+ comment: Add index for dependency_analysis_version to report_card+ changes:+ - createIndex:+ tableName: report_card+ indexName: idx_report_card_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version+ - changeSet:+ id: v57.2025-09-18T10:02:00+ author: metamben+ comment: Add dependency_analysis_version to transform+ changes:+ - addColumn:+ tableName: transform+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ remarks: Version of the dependency analysis for this entity.+ constraints:+ nullable: false+ - changeSet:+ id: v57.2025-09-18T10:03:00+ author: metamben+ comment: Add index for dependency_analysis_version to transform+ changes:+ - createIndex:+ tableName: transform+ indexName: idx_transform_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version+ - changeSet:+ id: v57.2025-09-18T10:04:00+ author: metamben+ comment: Add dependency_analysis_version to native_query_snippet+ changes:+ - addColumn:+ tableName: native_query_snippet+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ remarks: Version of the dependency analysis for this entity.+ constraints:+ nullable: false+ - changeSet:+ id: v57.2025-09-18T10:05:00+ author: metamben+ comment: Add index for dependency_analysis_version to native_query_snippet+ changes:+ - createIndex:+ tableName: native_query_snippet+ indexName: idx_native_query_snippet_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version+- changeSet:id: v57.2025-09-19T11:30:20author: ranquilddiff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.cljindex 24ffd86dd74..8cc8067cffb 100644--- a/src/metabase/analytics/stats.clj+++ b/src/metabase/analytics/stats.clj@@ -887,7 +887,10 @@:enabled (premium-features/enable-transforms?)}{:name :transforms-python:available (premium-features/enable-python-transforms?)- :enabled (premium-features/enable-python-transforms?)}])+ :enabled (premium-features/enable-python-transforms?)}+ {:name :dependencies+ :available (premium-features/enable-dependencies?)+ :enabled (premium-features/enable-dependencies?)}])(defn- snowplow-features[]diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 72a26c449e3..6a955cf82b5 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -745,6 +745,9 @@;; Does this driver support executing python transforms?:transforms/python+ ;; Does this driver support calculating dependencies of native queries?+ :dependencies/native+;; Does this driver properly support the table-exists? method for checking table existence?:metadata/table-existence-check@@ -1195,7 +1198,20 @@"Gets the table dependencies of a given sql string (or equivalent).Drivers that support any of the `:transforms/...` features must implement this method."- {:added "0.57.0" :arglists '([driver query])}+ {:added "0.57.0" :arglists '([driver query] [driver query metadata-provider transforms])}+ dispatch-on-initialized-driver+ :hierarchy #'hierarchy)++(defmulti native-result-metadata+ "Gets the result-metadata for a native query using static analysis (i.e., without actually+ going to the database)."+ {:added "0.57.0" :arglists '([driver metadata-provider native-query])}+ dispatch-on-initialized-driver+ :hierarchy #'hierarchy)++(defmulti validate-native-query-fields+ "Validates a native query and returns a list of all 'bad' field references."+ {:added "0.57.0" :arglists '([driver metadata-provider native-query])}dispatch-on-initialized-driver:hierarchy #'hierarchy)diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.cljindex d438b5bbf40..b9e418a4129 100644--- a/src/metabase/driver/h2.clj+++ b/src/metabase/driver/h2.clj@@ -9,11 +9,13 @@[metabase.driver.common :as driver.common][metabase.driver.h2.actions :as h2.actions][metabase.driver.settings :as driver.settings]+ [metabase.driver.sql :as sql][metabase.driver.sql-jdbc :as sql-jdbc][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]+ [metabase.driver.sql.normalize :as sql.normalize][metabase.driver.sql.query-processor :as sql.qp][metabase.util :as u][metabase.util.honey-sql-2 :as h2x]@@ -668,3 +670,11 @@(defmethod sql-jdbc/impl-table-known-to-not-exist? :h2[_ e](#{"42S02" "42S03" "42S04"} (sql-jdbc/get-sql-state e)))++(defmethod sql.normalize/normalize-unquoted-name :h2+ [_ name-str]+ (u/upper-case-en name-str))++(defmethod sql/default-schema :h2+ [_]+ "PUBLIC")diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex 96011f887e3..a735196d350 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -3,22 +3,21 @@(:refer-clojure :exclude [some])(:require[clojure.set :as set]- [clojure.string :as str][macaw.core :as macaw][metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common.parameters.parse :as params.parse][metabase.driver.common.parameters.values :as params.values]+ [metabase.driver.sql.normalize :as sql.normalize][metabase.driver.sql.parameters.substitute :as sql.params.substitute][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp]+ [metabase.driver.sql.references :as sql.references][metabase.driver.sql.util :as sql.u]- [metabase.util :as u]+ [metabase.util.humanization :as u.humanization][metabase.util.malli :as mu][metabase.util.performance :refer [some]]- [potemkin :as p]- ^{:clj-kondo/ignore [:discouraged-namespace]}- [toucan2.core :as t2]))+ [potemkin :as p]))(comment sql.params.substitution/keep-me) ; this is so `cljr-clean-ns` and the linter don't remove the `:require`@@ -48,7 +47,8 @@:expressions/text:expressions/today:distinct-where- :database-routing]]+ :database-routing+ :dependencies/native]](defmethod driver/database-supports? [:sql feature] [_driver _feature _db] true))(defmethod driver/database-supports? [:sql :persist-models-enabled]@@ -138,21 +138,6 @@;; honeysql, and accepts a keyword too. This way we delegate proper escaping and qualification to honeysql.(driver/drop-table! driver (:id database) (qualified-name target)))-(defn normalize-name- "Normalizes the (primarily table/column) name passed in.- Should return a value that matches the name listed in the appdb."- [driver name-str]- (let [quote-style (sql.qp/quote-style driver)- quote-char (if (= quote-style :mysql) \` \")]- (if (and (= (first name-str) quote-char)- (= (last name-str) quote-char))- (let [quote-quote (str quote-char quote-char)- quote (str quote-char)]- (-> name-str- (subs 1 (dec (count name-str)))- (str/replace quote-quote quote)))- (u/lower-case-en name-str))))-(defmulti default-schema"Returns the default schema for a given database driver.@@ -168,13 +153,12 @@(defn find-table-or-transform"Given a table and schema that has been parsed out of a native query, finds either a matching table or a matching transform.It will return either {:table table-id} or {:transform transform-id}, or nil if neither is found."- [driver tables transforms {:keys [table schema]}]- (let [normalized-table (normalize-name driver table)- normalized-schema (or (some->> schema (normalize-name driver))- (default-schema driver))+ [driver tables transforms {search-table :table raw-schema :schema}]+ (let [search-schema (or raw-schema+ (default-schema driver))matches? (fn [db-table db-schema]- (and (= normalized-table db-table)- (= normalized-schema db-schema)))]+ (and (= search-table db-table)+ (= search-schema db-schema)))](or (some (fn [{:keys [name schema id]}](when (matches? name schema){:table id}))@@ -184,16 +168,131 @@{:transform id}))transforms))))+(defn- normalize-table-spec+ [driver {:keys [table schema]}]+ {:table (sql.normalize/normalize-name driver table)+ :schema (some->> schema (sql.normalize/normalize-name driver))})+(defmethod driver/native-query-deps :sql- [driver query]- (let [db-tables (driver-api/tables (driver-api/metadata-provider))- transforms (t2/select [:model/Transform :id :target])]- (->> query- macaw/parsed-query- macaw/query->components- :tables- (map :component)- (into #{} (keep #(find-table-or-transform driver db-tables transforms %))))))+ ([driver query]+ (driver/native-query-deps driver query+ (driver-api/metadata-provider)))+ ([driver query metadata-provider]+ (let [db-tables (driver-api/tables metadata-provider)+ db-transforms (driver-api/transforms metadata-provider)]+ (->> query+ macaw/parsed-query+ macaw/query->components+ :tables+ (map :component)+ (into #{} (keep #(->> (normalize-table-spec driver %)+ (find-table-or-transform driver db-tables db-transforms))))))))++;;; +----------------------------------------------------------------------------------------------------------------++;;; | Dependencies |+;;; +----------------------------------------------------------------------------------------------------------------+++(defmulti resolve-field+ "Resolves a field reference to one or more actual database fields.++ This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field+ references returned by sql.references/field-references.++ Note: this currently sets :lib/desired-column-alias but no other :lib/* fields, because the callers of this function+ don't need the other fields. If we care about other :lib/* fields in the future, we can add them then."+ {:added "0.57.0" :arglists '([driver metadata-provider col-spec])}+ (fn [_driver _metadata-provider col-spec]+ (:type col-spec)))++(defmethod resolve-field :all-columns+ [driver metadata-provider col-spec]+ (or (some->> (:table col-spec)+ (find-table-or-transform driver (driver-api/tables metadata-provider) (driver-api/transforms metadata-provider))+ :table+ (driver-api/active-fields metadata-provider)+ (map #(assoc % :lib/desired-column-alias (:name %))))+ [(assoc col-spec ::bad-reference true)]))++(defmethod resolve-field :single-column+ [driver metadata-provider {:keys [alias] :as col-spec}]+ [(if-let [{:keys [name] :as found}+ (->> (:source-columns col-spec)+ (some (fn [source-col-set]+ ;; in cases like `select (select blah from ...) from ...`, if blah refers to a+ ;; column in both the inner query and the outer query, the column from the inner+ ;; query will be preferred. However, if blah doesn't refer to something in the+ ;; inner query, it can also refer to something in the outer query.+ ;; sql.references/field-references organizes source-cols into a list of lists+ ;; to account for this.+ (->> (mapcat (partial resolve-field driver metadata-provider) source-col-set)+ (some #(when (= (:name %) (:column col-spec))+ %))))))]+ (assoc found :lib/desired-column-alias (or alias name))+ (assoc col-spec ::bad-reference true))])++(defn- get-name [m]+ (or (:alias m) (str (gensym "new-col"))))++(defn- get-display-name [m]+ (->> (get-name m)+ (u.humanization/name->human-readable-name :simple)))++(defmethod resolve-field :custom-field+ [_driver _metadata-provider col-spec]+ [{:base-type :type/*+ :name (get-name col-spec)+ :lib/desired-column-alias (get-name col-spec)+ :display-name (get-display-name col-spec)+ :effective-type :type/*+ :semantic-type :Semantic/*}])++(defmethod resolve-field :invalid-table-wildcard+ [_driver _metadata-provider col-spec]+ [(assoc col-spec ::bad-reference true)])++(defn- lca [default-type & types]+ (let [ancestor-sets (for [t types+ :when t]+ (conj (ancestors t) t))+ common-ancestors (when (seq ancestor-sets)+ (apply set/intersection ancestor-sets))]+ (if (seq common-ancestors)+ (apply (partial max-key (comp count ancestors)) common-ancestors)+ default-type)))++(defmethod resolve-field :composite-field+ [driver metadata-provider col-spec]+ (let [member-fields (mapcat (partial resolve-field driver metadata-provider)+ (:member-fields col-spec))]+ [{:name (get-name col-spec)+ :lib/desired-column-alias (get-name col-spec)+ :display-name (get-display-name col-spec)+ :base-type (apply lca :type/* (map :base-type member-fields))+ :effective-type (apply lca :type/* (map :effective-type member-fields))+ :semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}]))++(defmethod driver/native-result-metadata :sql+ [driver metadata-provider native-query]+ (let [{:keys [returned-fields]} (->> (macaw/parsed-query native-query)+ macaw/->ast+ (sql.references/field-references driver))]+ (->> (mapcat (partial resolve-field driver metadata-provider) returned-fields)+ (remove ::bad-reference))))++(defmethod driver/validate-native-query-fields :sql+ [driver metadata-provider native-query]+ (let [{:keys [used-fields returned-fields bad-sql]} (->> (macaw/parsed-query native-query)+ macaw/->ast+ (sql.references/field-references driver))+ check-fields #(mapcat (fn [col-spec]+ (->> (resolve-field driver metadata-provider col-spec)+ (filter ::bad-reference)))+ %)]+ (-> (concat (when bad-sql+ [{:error :bad-sql}])+ (check-fields used-fields)+ (check-fields returned-fields))+ distinct)));;; +----------------------------------------------------------------------------------------------------------------+;;; | Convenience Imports |diff --git a/src/metabase/driver/sql/normalize.clj b/src/metabase/driver/sql/normalize.cljnew file mode 100644index 00000000000..591b989c2aa--- /dev/null+++ b/src/metabase/driver/sql/normalize.clj@@ -0,0 +1,52 @@+(ns metabase.driver.sql.normalize+ (:require+ [clojure.string :as str]+ [metabase.driver :as driver]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.util :as u]))++(defmulti normalize-unquoted-name+ "Normalize an unquoted table/column name according to the database's rules."+ {:added "0.57.0", :arglists '([driver name-str])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod normalize-unquoted-name :sql+ [_ name-str]+ (u/lower-case-en name-str))++(defn normalize-name+ "Normalizes the (primarily table/column) name passed in.+ Should return a value that matches the name listed in the appdb."+ [driver name-str]+ (let [quote-style (sql.qp/quote-style driver)+ quote-char (if (= quote-style :mysql) \` \")]+ (if (and (= (first name-str) quote-char)+ (= (last name-str) quote-char))+ (let [quote-quote (str quote-char quote-char)+ quote (str quote-char)]+ (-> name-str+ (subs 1 (dec (count name-str)))+ (str/replace quote-quote quote)))+ (normalize-unquoted-name driver name-str))))++(defmulti reserved-literal+ "Checks whether a particular name is actually a literal value in a given sql dialect.++ For example, true and false are usually booleans, not normal names."+ {:added "0.57.0", :arglists '([driver name])}+ (fn [driver name]+ [(driver/dispatch-on-initialized-driver driver) name])+ :hierarchy #'driver/hierarchy)++(defmethod reserved-literal :default+ [_driver _name]+ nil)++(defmethod reserved-literal [:sql "true"]+ [_driver _name]+ {:value true})++(defmethod reserved-literal [:sql "false"]+ [_driver _name]+ {:value false})diff --git a/src/metabase/driver/sql/references.clj b/src/metabase/driver/sql/references.cljnew file mode 100644index 00000000000..4c79ddc1820--- /dev/null+++ b/src/metabase/driver/sql/references.clj@@ -0,0 +1,316 @@+(ns metabase.driver.sql.references+ (:require+ [macaw.ast-types :as macaw.ast-types]+ [metabase.driver :as driver]+ [metabase.driver.sql.normalize :as sql.normalize]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.performance :refer [every? mapv select-keys some]]))++(mr/def ::single-column+ [:map+ [:type [:= :single-column]]+ [:column :string]+ [:source-columns [:sequential [:sequential [:ref ::col-spec]]]]+ [:alias [:maybe :string]]])++(mr/def ::all-columns+ [:map+ [:type [:= :all-columns]]+ [:table [:map+ [:table :string]+ [:schema {:optional true} :string]+ [:database {:optional true} :string]+ [:table-alias {:optional true} :string]]]])++(mr/def ::custom-field+ [:map+ [:type [:= :custom-field]]+ [:alias [:maybe :string]]+ [:used-fields [:set [:ref ::col-spec]]]])++(mr/def ::composite-field+ [:map+ [:type [:= :composite-field]]+ [:alias [:maybe :string]]+ [:member-fields [:sequential ::col-spec]]])++(mr/def ::invalid-table-wildcard+ [:map+ [:type [:= :invalid-table-wildcard]]+ [:table :string]+ [:schema {:optional true} :string]+ [:database {:optional true} :string]+ [:table-alias {:optional true} :string]])++(mr/def ::col-spec+ [:multi {:dispatch :type}+ [:single-column [:ref ::single-column]]+ [:all-columns [:ref ::all-columns]]+ [:custom-field [:ref ::custom-field]]+ [:composite-field [:ref ::composite-field]]+ [:invalid-table-wildcard [:ref ::invalid-table-wildcard]]])++(mr/def ::field-references+ [:map+ [:used-fields [:set [:ref ::col-spec]]]+ [:returned-fields [:sequential [:ref ::col-spec]]]+ [:bad-sql {:optional true} :boolean]])++(defn- normalize-fields [driver m]+ (update-vals m+ #(if (string? %)+ (sql.normalize/normalize-name driver %)+ %)))++(defn- col-fields [driver m]+ (->> (select-keys m [:type :column :table :schema :database :alias])+ (normalize-fields driver)))++(defn- table-matches? [search table]+ (or (when (and (not (:schema search))+ (not (:database search)))+ (= (:table search) (:table-alias table)))+ (->> (select-keys search [:table :schema :database])+ keys+ (every? #(= (% search) (% table))))))++(defn- find-source [search sources]+ (some (fn [sublist]+ (some #(when (table-matches? search (:names %))+ %)+ sublist))+ sources))++(defn- get-column [driver sources raw-col]+ (if (and (nil? (:table raw-col))+ (nil? (:schema raw-col))+ (nil? (:database raw-col))+ (sql.normalize/reserved-literal driver (:column raw-col)))+ []+ (let [{:keys [alias column table] :as expr} (col-fields driver raw-col)+ valid-sources (if table+ [[(find-source expr sources)]]+ sources)+ source-columns (into []+ (map #(into []+ (comp (mapcat :returned-fields)+ (filter (fn [{field-alias :alias+ field-type :type+ field-column :column}]+ (or (= field-type :all-columns)+ (= (or field-alias field-column) column)))))+ %))+ valid-sources)+ source-column (some #(when (= column (or (:alias %) (:column %)))+ %)+ (first source-columns))]+ [(if source-column+ (cond-> source-column+ alias (assoc :alias alias))+ (-> {:column column+ :alias alias+ :type :single-column}+ (assoc :source-columns source-columns)))])))++(defmulti find-used-fields+ "Finds the fields used in a given sql expression."+ {:added "0.57.0", :arglists '([driver sources withs expr])}+ (fn [driver _sources _withs expr]+ [(driver/dispatch-on-initialized-driver driver) (:type expr)])+ :hierarchy #'driver/hierarchy)++(defmethod find-used-fields :default+ [_driver _sources _withs _expr]+ nil)++(defmethod find-used-fields [:sql :macaw.ast/column]+ [driver sources _withs expr]+ (get-column driver sources expr))++(defmethod find-used-fields [:sql :macaw.ast/unary-expression]+ [driver sources withs expr]+ (find-used-fields driver sources withs (:expression expr)))++(defmethod find-used-fields [:sql :macaw.ast/binary-expression]+ [driver sources withs expr]+ (concat (find-used-fields driver sources withs (:left expr))+ (find-used-fields driver sources withs (:right expr))))++(defmethod find-used-fields [:sql :macaw.ast/expression-list]+ [driver sources withs expr]+ (mapcat (partial find-used-fields driver sources withs) (:expressions expr)))++(defmethod find-used-fields [:sql :macaw.ast/join]+ [driver sources withs expr]+ (mapcat (partial find-used-fields driver sources withs) (:condition expr)))++(defmethod find-used-fields [:sql :macaw.ast/function]+ [driver sources withs expr]+ (mapcat (partial find-used-fields driver sources withs) (:params expr)))++(defmethod find-used-fields [:sql :macaw.ast/case]+ [driver sources withs expr]+ (concat (find-used-fields driver sources withs (:switch expr))+ (find-used-fields driver sources withs (:else expr))+ (mapcat (fn [{when-clause :when then-clause :then}]+ (concat (find-used-fields driver sources withs when-clause)+ (find-used-fields driver sources withs then-clause)))+ (:when-clauses expr))))++(defmethod find-used-fields [:sql :macaw.ast/between]+ [driver sources withs expr]+ (concat (find-used-fields driver sources withs (:expression expr))+ (find-used-fields driver sources withs (:start expr))+ (find-used-fields driver sources withs (:end expr))))++(defmethod find-used-fields [:sql :macaw.ast/set-operation]+ [driver sources withs expr]+ (mapcat (partial find-used-fields driver sources withs)+ (:selects expr)))++(defmethod find-used-fields [:sql :macaw.ast/analytic-expression]+ [driver sources withs expr]+ (let [rec (partial find-used-fields driver sources withs)]+ (concat (rec (:expression expr))+ (rec (:offset expr))+ (rec (:window expr))+ (mapcat rec (:partition-by expr))+ (mapcat rec (:order-by expr)))))++(defmulti find-returned-fields+ "Finds the fields returned by a given sql query/expression."+ {:added "0.57.0", :arglists '([driver sources withs expr])}+ (fn [driver _sources _withs expr]+ [(driver/dispatch-on-initialized-driver driver) (:type expr)])+ :hierarchy #'driver/hierarchy)++(defmethod find-returned-fields :default+ [driver sources withs {:keys [alias] :as expr}]+ [{:alias (when alias+ (sql.normalize/normalize-name driver alias))+ :type :custom-field+ :used-fields (into #{} (find-used-fields driver sources withs expr))}])++(defmethod find-returned-fields [:sql :macaw.ast/column]+ [driver sources _withs expr]+ (get-column driver sources expr))++(defmethod find-returned-fields [:sql :macaw.ast/wildcard]+ [_driver sources _withs _expr]+ (into []+ (mapcat :returned-fields)+ (first sources)))++(defmethod find-returned-fields [:sql :macaw.ast/table-wildcard]+ [driver sources _withs expr]+ (or (some-> (find-source (col-fields driver expr) sources)+ :returned-fields)+ [(assoc expr+ :type :invalid-table-wildcard)]))++(defmethod find-returned-fields [:sql :macaw.ast/set-operation]+ [driver sources withs expr]+ (->> (map (partial find-returned-fields driver sources withs)+ (:selects expr))+ (apply map (fn [& fields]+ {:alias (or (:alias (first fields))+ (:column (first fields)))+ :type :composite-field+ :member-fields (into [] fields)}))))++(defmulti field-references-impl+ "Implementation of field-references."+ {:added "0.57.0", :arglists '([driver sources withs expr])}+ (fn [driver _outside-sources _withs expr]+ [(driver/dispatch-on-initialized-driver driver) (:type expr)])+ :hierarchy #'driver/hierarchy)++(defmethod field-references-impl [:sql :macaw.ast/table]+ [driver _outside-sources withs expr]+ (or (when-let [source (find-source expr [withs])]+ (update source :names assoc :table-alias (:table-alias expr)))+ {:used-fields #{}+ :returned-fields [{:type :all-columns+ :table (->> (select-keys expr [:table :schema :database :table-alias])+ (normalize-fields driver))}]+ :names (normalize-fields driver expr)}))++(defn- get-select-sources+ [driver outside-sources withs expr]+ (cond-> (mapv (partial field-references-impl driver outside-sources withs) (:join expr))+ (:from expr) (conj (field-references-impl driver outside-sources withs (:from expr)))))++(defmethod find-used-fields [:sql :macaw.ast/select]+ [driver outside-sources withs expr]+ (let [local-sources (get-select-sources driver outside-sources withs expr)+ sources (cons local-sources outside-sources)+ rec (partial find-used-fields driver sources withs)]+ (-> (into #{}+ (mapcat rec)+ (:select expr))+ (into (rec (:where expr)))+ (into (mapcat rec)+ (:join expr))+ (into (mapcat rec)+ (:group-by expr))+ (into (mapcat rec)+ (:order-by expr))+ (into (mapcat :used-fields)+ local-sources))))++(defmethod find-returned-fields [:sql :macaw.ast/select]+ [driver outside-sources withs expr]+ (let [sources (cons (get-select-sources driver outside-sources withs expr)+ outside-sources)+ returned-fields (mapcat (partial find-returned-fields driver sources withs) (:select expr))]+ (if (:alias expr)+ ;; :alias is a column alias, so this is presumably something like `select (select * from ...)`+ ;; there should only be one field returned, and that field should have the appropriate alias+ (do (assert (= 1 (count returned-fields)))+ (map #(assoc % :alias (sql.normalize/normalize-name driver (:alias expr)))+ returned-fields))+ returned-fields)))++(defmethod field-references-impl [:sql :macaw.ast/select]+ [driver outside-sources outside-withs expr]+ (let [local-withs (map (partial field-references-impl driver outside-sources outside-withs)+ (:with expr))+ withs (into outside-withs local-withs)]+ {:used-fields (into (find-used-fields driver outside-sources withs expr)+ (mapcat :used-fields)+ local-withs)+ :returned-fields (into []+ (find-returned-fields driver outside-sources withs expr))+ :names (when-let [alias (:table-alias expr)]+ {:table-alias (sql.normalize/normalize-name driver alias)})}))++(defmethod field-references-impl [:sql :macaw.ast/join]+ [driver outside-sources withs expr]+ (field-references-impl driver outside-sources withs (:source expr)))++(defmethod field-references-impl [:sql :macaw.ast/set-operation]+ [driver outside-sources withs expr]+ {:used-fields (->> (find-used-fields driver outside-sources withs expr)+ (into #{}))+ :returned-fields (->> (find-returned-fields driver outside-sources withs expr)+ (into []))+ :names (when-let [alias (:table-alias expr)]+ {:table-alias (sql.normalize/normalize-name driver alias)})})++(defmethod field-references-impl :default+ [_driver _outside-sources _outside-withs _expr]+ {:used-fields #{}+ :returned-fields []+ :names nil+ :bad-sql true})++(mu/defn field-references :- [:ref ::field-references]+ "Takes a sql query in the macaw ast format and returns the fields referenced by a query.++ Specifically, this returns a set of the fields used and a list of the fields returned, along with a boolean that+ denotes when this is given an invalid query of some sort."+ [driver :- :keyword+ expr :- macaw.ast-types/ast]+ (-> (field-references-impl driver nil #{} expr)+ (dissoc :names)))diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.cljindex e44bc5593a6..2ec9e1d06b4 100644--- a/src/metabase/driver_api/core.clj+++ b/src/metabase/driver_api/core.clj@@ -104,8 +104,10 @@lib.metadata/databaselib.metadata/fieldlib.metadata/fields+ lib.metadata/active-fieldslib.metadata/tablelib.metadata/tables+ lib.metadata/transformslib.schema.common/instance-of-classlib.schema.temporal-bucketing/date-bucketing-unitslib.types.isa/temporal?diff --git a/src/metabase/events/schema.clj b/src/metabase/events/schema.cljindex 5ac3685496b..3fb897006ed 100644--- a/src/metabase/events/schema.clj+++ b/src/metabase/events/schema.clj@@ -161,3 +161,13 @@[:map {:closed true}[:user-id [:maybe pos-int?]][:model [:or :keyword :string]]])++;; snippet events++(mr/def ::snippet+ [:map {:closed true}+ [:object [:fn #(t2/instance-of? :model/NativeQuerySnippet %)]]])++(mr/def :event/snippet-create ::snippet)+(mr/def :event/snippet-update ::snippet)+(mr/def :event/snippet-delete ::snippet)diff --git a/src/metabase/graph/core.cljc b/src/metabase/graph/core.cljcnew file mode 100644index 00000000000..e389054aee1--- /dev/null+++ b/src/metabase/graph/core.cljc@@ -0,0 +1,91 @@+(ns metabase.graph.core+ "Interface to a library of generic graph walking functionality.++ Used for eg. walking the `enterprise/dependencies` graph, or similar directed graphs.++ Implementation-agnostic! The graph representation is wrapped by a protocol exported from this module, which can be+ used to write graph searches which can run in memory, against AppDB, on the filesystem, etc.++ Each graph defines an arbitrary **key**. This can be any hash map key, such as a number, a `[type id]` pair, etc."+ (:require+ #?@(:cljs ([flatland.ordered.set :as oset]))+ [metabase.util.malli.registry :as mr]+ [potemkin :as p])+ #?@(:clj ((:import java.util.LinkedHashSet))))++#?(:clj (set! *warn-on-reflection* true))++(p/defprotocol+ Graph+ (children-of [graph key-seq]+ "Given a graph and a seq of keys, returns a map from each input key to the set of keys **directly reachable** from+ that key.++ If you want a *transitive* search, call [[transitive]].++ Contract details:+ - If a key is not known in the graph, it should be missing from the output map.+ - If a key is found in the graph, and it has no children, its value is `#{}`.+ - If the input `key-seq` is empty, return `{}`."))++(defn- stable-iteration-set []+ ;; Both of these sets explicitly have stable iteration order: values are returned in the order of (first) insertion.+ #?(:clj (let [s (LinkedHashSet.)]+ {:insert-many! #(.addAll s %)+ :seen? #(.contains s %)+ :->iterable (constantly s)})+ :cljs (let [s (volatile! (oset/ordered-set))]+ {:insert-many! #(vswap! s into %)+ :seen? #(contains? @s %)+ :->iterable #(deref s)})))++(defn transitive+ "Given a graph and `key-seq`, returns a seq of all transitive children of those starting keys.++ The returned seq:+ - Excludes the input keys, even if they're reachable from the input keys.+ - Contains each key only once, even if it can be reached by multiple paths.+ - Is **topologically sorted**: all keys reachable in `n` steps from the starters appear before any keys reachable+ in `n+1` steps, etc.+ - Makes no guarantee about the order of keys at the same \"level\".+ - That is, if two keys `k1` and `k2` are reachable in no fewer than `n` steps from the starters, then `k1` might+ precede or follow `k2` in the output seq."+ [graph key-seq]+ (let [{:keys [insert-many! seen? ->iterable]} (stable-iteration-set)]+ (insert-many! key-seq)+ (loop [new-keys key-seq]+ (if (seq new-keys)+ (let [k->children (children-of graph new-keys)+ new-children (into #{} (comp cat (remove seen?)) (vals k->children))]+ (insert-many! new-children)+ (recur new-children))+ (drop (count key-seq) (->iterable))))))++(defn graph?+ "Whether `x` is a valid `Graph`."+ [x]+ #?(:clj (extends? Graph (class x))+ :cljs (satisfies? Graph x)))++(mr/def ::graph+ "Schema for anything that satisfies the [[Graph]] protocol."+ [:fn+ {:error/message "Valid Graph instance"}+ #'graph?])++;; ## In-memory Graphs+(p/deftype+ InMemoryGraph [adjacency-map]+ Graph+ (children-of [_this key-seq]+ (select-keys adjacency-map key-seq)))++(defn in-memory+ "Given a map of which keys are reachable from each key, `{key #{key...}, ...}`, returns an instance of [[Graph]]+ backed by this in-memory graph.++ Note that this map is in the same form as [[children-of]] returns."+ [adjacency-map]+ (->InMemoryGraph adjacency-map))++;; TODO: (Braden, 09/19/2025) We may find use for some extra helpers. For example, a wrapper that implements the+;; multi-key [[children-of]] on top of a single-key `(children graph key) => #{keys...}` function.+;; But YAGNI, so since I don't have a use-case in mind right now, I'm just leaving a note.diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex 6597fcf4da9..0f290f2f049 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -55,6 +55,7 @@[metabase.lib.template-tags :as lib.template-tags][metabase.lib.temporal-bucket :as lib.temporal-bucket][metabase.lib.util :as lib.util]+ [metabase.lib.validate :as lib.validate][metabase.lib.walk.util][metabase.util.namespaces :as shared.ns]))@@ -360,6 +361,7 @@metadata-provider?metadata-providerable?][lib.native+ add-parameters-for-template-tagsengineextract-template-tagshas-template-tag-variables?@@ -458,6 +460,8 @@source-table-idsource-card-idupdate-query-stage]+ [lib.validate+ find-bad-refs][metabase.lib.walk.utilall-field-idsall-source-card-idsdiff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljcindex 374ed0d50b6..7fdb4631c7c 100644--- a/src/metabase/lib/metadata.cljc+++ b/src/metabase/lib/metadata.cljc@@ -120,6 +120,18 @@(when-let [remap-field-id (get-in column [:lib/external-remap :field-id])](field metadata-providerable remap-field-id))))+;; TODO: Better schemas for transforms coming out of the metadata provider.+(mu/defn transform :- [:maybe [:map]]+ "Gets a Transform by ID, or nil if it does not exist."+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ transform-id :- :int]+ (lib.metadata.protocols/transform (->metadata-provider metadata-providerable) transform-id))++(mu/defn transforms :- [:maybe [:sequential [:map]]]+ "Gets all Transforms"+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable]+ (lib.metadata.protocols/transforms (->metadata-provider metadata-providerable)))+(mu/defn setting :- any?"Get the value of a Metabase setting for the instance we're querying."([metadata-providerable :- ::lib.schema.metadata/metadata-providerablediff --git a/src/metabase/lib/metadata/cached_provider.cljc b/src/metabase/lib/metadata/cached_provider.cljcindex 55d8e2a9c73..e0fcb481b2c 100644--- a/src/metabase/lib/metadata/cached_provider.cljc+++ b/src/metabase/lib/metadata/cached_provider.cljc@@ -30,16 +30,17 @@id :- pos-int?metadata :- [:multi{:dispatch :lib/type}- [:metadata/database ::lib.schema.metadata/database]- [:metadata/table ::lib.schema.metadata/table]- [:metadata/column ::lib.schema.metadata/column]- [:metadata/card ::lib.schema.metadata/card]- [:metadata/metric ::lib.schema.metadata/metric]- [:metadata/segment ::lib.schema.metadata/segment]]]+ [:metadata/database ::lib.schema.metadata/database]+ [:metadata/table ::lib.schema.metadata/table]+ [:metadata/column ::lib.schema.metadata/column]+ [:metadata/card ::lib.schema.metadata/card]+ [:metadata/metric ::lib.schema.metadata/metric]+ [:metadata/segment ::lib.schema.metadata/segment]+ [:metadata/native-query-snippet ::lib.schema.metadata/native-query-snippet]]](let [metadata (-> metadata(update-keys u/->kebab-case-en)(assoc :lib/type metadata-type))]- (store-in-cache! cache [metadata-type id] metadata))+ (store-in-cache! cache [metadata-type :id id] metadata))true)(defn- get-in-cache-or-fetch [cache ks fetch-thunk]diff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljcindex fc15b25f32f..c9eb3c34248 100644--- a/src/metabase/lib/metadata/protocols.cljc+++ b/src/metabase/lib/metadata/protocols.cljc@@ -10,7 +10,7 @@(mr/def ::metadata-type-excluding-database"Database metadata is stored separately/in a special way. These are the types of metadata that are stored with theother non-Database methods."- [:enum :metadata/table :metadata/column :metadata/card :metadata/metric :metadata/segment :metadata/native-query-snippet])+ [:enum :metadata/table :metadata/column :metadata/card :metadata/metric :metadata/segment :metadata/native-query-snippet :metadata/transform])(mr/def ::metadata-spec"Spec for fetching objects from a metadata provider. `:lib/type` is the type of the object to fetch, and the other@@ -42,8 +42,8 @@(or (not (:card-id spec))(#{:metadata/metric} (:lib/type spec))))][:fn- {:error/message "All metadata types except for :metadata/table must include at least one filter"}- (some-fn :id :name :table-id :card-id #(= (:lib/type %) :metadata/table))]])+ {:error/message "All metadata types except for :metadata/table and :metadata/transform must include at least one filter"}+ (some-fn :id :name :table-id :card-id #(= (:lib/type %) :metadata/table) #(= (:lib/type %) :metadata/transform))]])(mu/defn default-spec-filter-xform"Create a `filter` transducer to a sequence of objects according to `metadata-spec`. Assumes objects are all the@@ -239,6 +239,18 @@card-id :- ::lib.schema.id/card](metadata-by-id metadata-provider :metadata/card card-id))+;; TODO: Better schemas for transforms.+(mu/defn transform :- [:maybe [:map]]+ "Return information about a specific Transform. Nil if it does not exist."+ [metadata-provider :- ::metadata-provider+ card-id :- ::lib.schema.id/card]+ (metadata-by-id metadata-provider :metadata/transform card-id))++(mu/defn transforms :- [:maybe [:sequential [:map]]]+ "Return information about all Transforms."+ [metadata-provider :- ::metadata-provider]+ (metadatas metadata-provider {:lib/type :metadata/transform}))+(mu/defn native-query-snippet :- [:maybe ::lib.schema.metadata/native-query-snippet]"Get metadata for a NativeQuerySnippet with `snippet-id` if it can be found."[metadata-provider :- ::metadata-providerdiff --git a/src/metabase/lib/native.cljc b/src/metabase/lib/native.cljcindex 55f88aa6ceb..017579dd241 100644--- a/src/metabase/lib/native.cljc+++ b/src/metabase/lib/native.cljc@@ -7,6 +7,7 @@[clojure.string :as str][medley.core :as m][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options][metabase.lib.parse :as lib.parse][metabase.lib.query :as lib.query][metabase.lib.schema :as lib.schema]@@ -364,3 +365,54 @@[query :- ::lib.schema/query](assert-native-query (lib.util/query-stage query 0))(:engine (lib.metadata/database query)))++(defn- get-parameter-value+ [tag-name {:keys [id dimension], param-type :type}]+ ;; note that the actual values chosen are completely arbitrary. We just need to provide some+ ;; value so that the query will compile.+ (case param-type+ :text {:id id,+ :type :string/=,+ :value ["foo"],+ :target ["variable" ["template-tag" tag-name]]}+ :number {:id id,+ :type :number/=,+ :value ["0"],+ :target ["variable" ["template-tag" tag-name]]}+ :date {:id id,+ :type :date/single,+ :value "1970-01-01",+ :target ["variable" ["template-tag" tag-name]]}+ :boolean {:id id,+ :type :boolean/=,+ :value [false],+ :target ["variable" ["template-tag" tag-name]]}+ :dimension (merge {:id id,+ :type :string/=,+ :value ["foo"],+ :target ["dimension" ["template-tag" tag-name]]}+ (when (isa? (-> dimension lib.options/options :effective-type) :type/Number)+ {:type :number/=,+ :value ["0"]}))+ :temporal-unit {:id id,+ :type :temporal-unit,+ :value "week",+ :target ["dimension" ["template-tag" tag-name]]}+ nil))++(defn add-parameters-for-template-tags+ "Adds dummy values for parameters that don't have one.+ This is so that the resulting native query can be parsed. It's not expected to be executable."+ [query]+ (let [ttags (-> (lib.util/query-stage query 0)+ :template-tags)+ parameters (:parameters query)+ params-by-id (m/index-by :id parameters)+ new-parameters (into []+ (keep (fn [[tag-name {:keys [id] :as tag}]]+ (or (params-by-id id)+ (get-parameter-value tag-name tag))))++ ttags)]+ (cond-> query+ (seq new-parameters) (assoc :parameters new-parameters))))diff --git a/src/metabase/lib/schema/id.cljc b/src/metabase/lib/schema/id.cljcindex 68d076e4a47..02899dfaa98 100644--- a/src/metabase/lib/schema/id.cljc+++ b/src/metabase/lib/schema/id.cljc@@ -79,6 +79,9 @@"Valid Native Query Snippet ID"pos-int?)+(mr/def ::transform+ [:schema {:doc/title "Valid Transform ID"} pos-int?])+(mr/def ::collection"Valid Collection ID"pos-int?)diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex a02683e3c70..46fe497a94e 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -566,7 +566,7 @@separate table in the app DB, but now that they're a subtype of Card it's probably not important anymore, we canprobably just use `:metadata/card` here."[:enum :metadata/database :metadata/table :metadata/column :metadata/card :metadata/metric- :metadata/segment])+ :metadata/segment :metadata/native-query-snippet])(mr/def ::lib-or-legacy-column"Schema for the maps in card `:result-metadata` and similar. These can be eitherdiff --git a/src/metabase/lib/util.cljc b/src/metabase/lib/util.cljcindex 447445e66fc..8b328e88957 100644--- a/src/metabase/lib/util.cljc+++ b/src/metabase/lib/util.cljc@@ -745,3 +745,31 @@(concat (collect-source-tables source-query) from-joins)(cond->> from-joins(:source-table legacy-query) (cons (:source-table legacy-query))))))++(defn- split-tables-and-legacy-card-refs [source-ids]+ (-> (reduce (fn [m src]+ (if-let [card-id (legacy-string-table-id->card-id src)]+ (update m :card conj! card-id)+ (update m :table conj! src)))+ {:card (transient #{})+ :table (transient #{})}+ source-ids)+ (update-vals persistent!)))++(mu/defn source-tables-and-cards :- [:map+ [:card {:optional true} [:set ::lib.schema.id/card]]+ [:table {:optional true} [:set ::lib.schema.id/table]]]+ "Returns all the table and card IDs that this list of **legacy, MBQL 4 queries** depend on."+ [legacy-queries]+ (let [metric-ids (-> (lib.util.match/match legacy-queries+ [:metric (id :guard pos-int?) & _]+ id)+ set)+ source-ids (-> #{}+ (into (map #(str "card__" %)) metric-ids)+ (into+ ;; existing usage -- do not use in new code+ #_{:clj-kondo/ignore [:deprecated-var]}+ (mapcat #(collect-source-tables (:query %)))+ legacy-queries))]+ (split-tables-and-legacy-card-refs source-ids)))diff --git a/src/metabase/lib/validate.cljc b/src/metabase/lib/validate.cljcnew file mode 100644index 00000000000..f057b4b0ef5--- /dev/null+++ b/src/metabase/lib/validate.cljc@@ -0,0 +1,41 @@+(ns metabase.lib.validate+ "Checks and validation for queries."+ (:require+ [metabase.lib.field.resolution :as lib.field.resolution]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]+ [metabase.lib.walk :as lib.walk]+ [metabase.util.malli :as mu]))++(mu/defn find-bad-refs :- [:maybe [:sequential ::lib.schema.mbql-clause/clause]]+ "Returns a list of bad `:field` refs on this query.++ Returns nil if all refs on the query are sound, that is if they can be resolved to a column from some source."+ [query :- ::lib.schema/query]+ (let [bad-fields (volatile! [])]+ (lib.walk/walk-clauses+ query (fn [query path-type path clause]+ (when (and (= path-type :lib.walk/stage)+ (vector? clause)+ (= (first clause) :field))+ (let [column (lib.walk/apply-f-for-stage-at-path+ lib.field.resolution/resolve-field-ref+ query path clause)]+ (when (or (not column)+ (::lib.field.resolution/fallback-metadata? column)+ (not (:active column true)))+ (vswap! bad-fields conj clause))))+ nil))+ (not-empty @bad-fields)))++(comment+ (require '[metabase.lib.core :as lib])+ (require '[metabase.lib.test-metadata :as meta])+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/breakout (meta/field-metadata :orders :quantity))+ (lib/aggregate (lib/count))+ lib/append-stage)+ [_category cnt] (lib/visible-columns base)+ query (lib/filter base (lib/> cnt 100))]+ {:valid-query (find-bad-refs query)+ :bad-query (find-bad-refs (assoc-in query [:stages 1 :filters 0 2 2] "n"))}))diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex ade2af7fce6..a5ba12c99d3 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -348,6 +348,28 @@[snippet](instance->metadata snippet :metadata/native-query-snippet))+;;;+;;; Transforms+;;;++(derive :metadata/transform :model/Transform)++(methodical/defmethod t2.model/resolve-model :metadata/transform+ [model]+ (t2/resolve-model :model/Transform) ; for side-effects+ model)++(methodical/defmethod t2.pipeline/build [#_query-type :toucan.query-type/select.*+ #_model :metadata/transform+ #_resolved-query clojure.lang.IPersistentMap]+ [query-type model parsed-args honeysql]+ (merge (next-method query-type model parsed-args honeysql)+ {:select [:id :name :source :target]}))++(t2/define-after-select :metadata/transform+ [snippet]+ (instance->metadata snippet :metadata/transform))+;;;;;; MetadataProvider;;;@@ -367,7 +389,8 @@:metadata/card :card/database_id:metadata/metric :database_id:metadata/segment :table/db_id- :metadata/native-query-snippet nil))+ :metadata/native-query-snippet nil+ :metadata/transform nil))(defn- id-key [metadata-type](case metadata-type@@ -376,7 +399,8 @@:metadata/card :card/id:metadata/metric :id:metadata/segment :segment/id- :metadata/native-query-snippet :id))+ :metadata/native-query-snippet :id+ :metadata/transform :id))(defn- name-key [metadata-type](case metadata-type@@ -385,7 +409,8 @@:metadata/card :card/name:metadata/metric :name:metadata/segment :segment/name- :metadata/native-query-snippet :name))+ :metadata/native-query-snippet :name+ :metadata/transform :name))(defn- table-id-key [metadata-type];; types not in the case statement do not support Table IDdiff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex fde952e936e..15d96a69fc9 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -230,6 +230,13 @@(and (map? query) (seq query))normalize)))+(defn maybe-normalize-transform-source+ "Normalizes the `source` of a transform."+ [in-or-out raw-source]+ (let [{source-type :type :as source} (m/map-keys keyword raw-source)]+ (cond-> source+ (= (keyword source-type) :query) (update :query (partial maybe-normalize-query in-or-out)))))+(defn catch-normalization-exceptions"Wraps normalization fn `f` and returns a version that gracefully handles Exceptions during normalization. Wheninvalid queries (etc.) come out of the Database, it's best we handle normalization failures gracefully rather than@@ -264,6 +271,11 @@{:in (comp json-in (partial maybe-normalize-query :in)):out (comp (catch-normalization-exceptions (partial maybe-normalize-query :out)) json-out-without-keywordization)})+(def transform-transform-source+ "Transform for transform source fields."+ {:in (comp json-in (partial maybe-normalize-transform-source :in))+ :out (comp (catch-normalization-exceptions (partial maybe-normalize-transform-source :out)) json-out-without-keywordization)})+(def transform-parameters-list"Transform for parameters list."{:in (comp json-in normalize-parameters-list)diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex 47afe26daee..ef9a0f2e1a5 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -38,6 +38,7 @@:model/DataPermissions metabase.permissions.models.data-permissions:model/Database metabase.warehouses.models.database:model/DatabaseRouter metabase-enterprise.database-routing.model+ :model/Dependency metabase-enterprise.dependencies.models.dependency:model/Dimension metabase.warehouse-schema.models.dimension:model/Document metabase-enterprise.documents.models.document:model/DocumentBookmark metabase-enterprise.bookmarks.models.bookmarkdiff --git a/src/metabase/native_query_snippets/core.clj b/src/metabase/native_query_snippets/core.cljindex 4ecdf3cd545..0bc73e177e6 100644--- a/src/metabase/native_query_snippets/core.clj+++ b/src/metabase/native_query_snippets/core.clj@@ -1,10 +1,16 @@(ns metabase.native-query-snippets.core(:require+ [metabase.native-query-snippets.models.native-query-snippet][metabase.native-query-snippets.models.native-query-snippet.permissions][potemkin :as p]))-(comment metabase.native-query-snippets.models.native-query-snippet.permissions/keep-me)+(comment+ metabase.native-query-snippets.models.native-query-snippet.permissions/keep-me+ metabase.native-query-snippets.models.native-query-snippet/keep-me)(p/import-vars+ [metabase.native-query-snippets.models.native-query-snippet+ add-template-tags+ NativeQuerySnippetName][metabase.native-query-snippets.models.native-query-snippet.permissionshas-any-native-permissions?])diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex abb821a52fe..e4b8d6f60b7 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -2,6 +2,7 @@(:require[honey.sql.helpers :as sql.helpers][metabase.collections.models.collection :as collection]+ [metabase.events.core :as events][metabase.lib.core :as lib][metabase.lib.normalize :as lib.normalize][metabase.models.interface :as mi]@@ -11,7 +12,8 @@[metabase.util.i18n :refer [deferred-tru tru]][metabase.util.malli :as mu][methodical.core :as methodical]- [toucan2.core :as t2]))+ [toucan2.core :as t2]+ [toucan2.realize :as t2.realize]));;; ----------------------------------------------- Entity & Lifecycle -----------------------------------------------@@ -32,7 +34,13 @@[_]#{:snippets})-(defn- add-template-tags [{old-tags :template_tags :as snippet}]+(derive ::event :metabase/event)+(doseq [e [:event/snippet-create :event/snippet-update :event/snippet-delete]]+ (derive e ::event))++(defn add-template-tags+ "Update the template tags based on the new contents."+ [{old-tags :template_tags :as snippet}];; Parse the snippet content to identify all template tags (like {{snippet: FilterA}} or {{var}}).;; For snippet references, we need to resolve them to snippet IDs while preserving reference stability.;;@@ -67,6 +75,11 @@(u/prog1 (add-template-tags snippet)(collection/check-collection-namespace :model/NativeQuerySnippet (:collection_id snippet))))+(t2/define-after-insert :model/NativeQuerySnippet+ [snippet]+ (u/prog1 (t2.realize/realize snippet)+ (events/publish-event! :event/snippet-create {:object <>})))+(t2/define-before-update :model/NativeQuerySnippet[snippet](u/prog1 (cond-> snippet@@ -76,6 +89,16 @@(throw (UnsupportedOperationException. (tru "You cannot update the creator_id of a NativeQuerySnippet."))))(collection/check-collection-namespace :model/NativeQuerySnippet (:collection_id snippet))))+(t2/define-after-update :model/NativeQuerySnippet+ [snippet]+ (u/prog1 (t2.realize/realize snippet)+ (events/publish-event! :event/snippet-update {:object <>})))++(t2/define-before-delete :model/NativeQuerySnippet+ [snippet]+ (u/prog1 snippet+ (events/publish-event! :event/snippet-delete {:object <>})))+(defmethod serdes/hash-fields :model/NativeQuerySnippet[_snippet][:name (serdes/hydrated-hash :collection) :created_at])@@ -123,6 +146,7 @@(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]{:copy [:archived :content :description :entity_id :name :template_tags]+ :skip [:dependency_analysis_version]:transform {:created_at (serdes/date):collection_id (serdes/fk :model/Collection):creator_id (serdes/fk :model/User)}})diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.cljindex 3b69a16f5ca..4c1ce9af441 100644--- a/src/metabase/premium_features/core.clj+++ b/src/metabase/premium_features/core.clj@@ -52,6 +52,7 @@enable-dashboard-subscription-filters?enable-database-auth-providers?enable-database-routing?+ enable-dependencies?enable-documents?enable-email-allow-list?enable-email-restrict-recipients?diff --git a/src/metabase/premium_features/settings.clj b/src/metabase/premium_features/settings.cljindex da840c557c2..a1388135c04 100644--- a/src/metabase/premium_features/settings.clj+++ b/src/metabase/premium_features/settings.clj@@ -299,6 +299,10 @@"Should we allow users to use Python transforms?":transforms-python)+(define-premium-feature ^{:added "0.57.0"} enable-dependencies?+ "Should we allow users to use dependency tracking?"+ :dependencies)+(defn- -token-features []{:advanced_permissions (enable-advanced-permissions?):ai_sql_fixer (enable-ai-sql-fixer?)@@ -316,6 +320,7 @@:dashboard_subscription_filters (enable-dashboard-subscription-filters?):database_auth_providers (enable-database-auth-providers?):database_routing (enable-database-routing?)+ :dependencies (enable-dependencies?):development_mode (development-mode?):disable_password_login (can-disable-password-login?):documents (enable-documents?)diff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.cljindex 998d3399e6f..c2caa525a87 100644--- a/src/metabase/queries/metadata.clj+++ b/src/metabase/queries/metadata.clj@@ -49,16 +49,6 @@(t2/select-fn-set :table_id :model/Field :id [:in field-ids])#{}))-(defn- split-tables-and-legacy-card-refs [source-ids]- (-> (reduce (fn [m src]- (if-let [card-id (lib.util/legacy-string-table-id->card-id src)]- (update m :cards conj! card-id)- (update m :tables conj! src)))- {:cards (transient #{})- :tables (transient #{})}- source-ids)- (update-vals persistent!)))-(defn- query->template-tags[query](-> query :native :template-tags vals))@@ -124,13 +114,8 @@(defn- batch-fetch-query-metadata*"Fetch dependent metadata for ad-hoc queries."[queries]- (let [source-ids (into #{}- ;; existing usage -- do not use in new code- #_{:clj-kondo/ignore [:deprecated-var]}- (mapcat #(lib.util/collect-source-tables (:query %)))- queries)- {source-table-ids :tables- source-card-ids :cards} (split-tables-and-legacy-card-refs source-ids)+ (let [{source-table-ids :table+ source-card-ids :card} (lib.util/source-tables-and-cards queries)source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids)(schema.table/batch-fetch-card-query-metadatas source-card-ids{:include-database? false}))diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 9ce2cd0e21d..1f8a46a08ed 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1189,7 +1189,9 @@;; this is data migration column:dataset_query_metrics_v2_migration_backup;; this column is not used anymore- :cache_ttl]+ :cache_ttl+ ;; dependencies aren't serialized, so the version of dependency analysis done shouldn't be serialized+ :dependency_analysis_version]:transform{:created_at (serdes/date):database_id (serdes/fk :model/Database :name)diff --git a/src/metabase/query_processor.clj b/src/metabase/query_processor.cljindex b505b5d7298..db160ec0279 100644--- a/src/metabase/query_processor.clj+++ b/src/metabase/query_processor.clj@@ -7,6 +7,7 @@Various REST API endpoints, such as `POST /api/dataset`, return the results of queries; they usuallyuse [[userland-query]] or [[userland-query-with-default-constraints]] (see below)."(:require+ [metabase.lib.core :as lib][metabase.lib.schema.info :as lib.schema.info][metabase.query-processor.compile :as qp.compile][metabase.query-processor.debug :as qp.debug]@@ -80,6 +81,13 @@(let [rff (or rff qp.reducible/default-rff)](process-query* query rff)))))+(mu/defn compile-query-with-metadata-provider+ "Compiles a given query with the provided metadata provider."+ [metadata-providerable :- :metabase.lib.schema.metadata/metadata-providerable+ query :- ::qp.schema/any-query]+ (-> (lib/query metadata-providerable query)+ qp.compile/compile))+(mu/defn userland-query :- ::qp.schema/any-query"Add middleware options and `:info` to a `query` so it is ran as a 'userland' query, which slightly changes the QPbehavior:diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex 67862862661..de89abb37d9 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -32,6 +32,7 @@:model/ContentTranslation:model/DashboardFavorite:model/DatabaseRouter+ :model/Dependency:model/PythonLibrary:model/Query:model/QueryCachediff --git a/test/metabase/driver/sql/references_test.clj b/test/metabase/driver/sql/references_test.cljnew file mode 100644index 00000000000..0423877ac83--- /dev/null+++ b/test/metabase/driver/sql/references_test.clj@@ -0,0 +1,2172 @@+(ns metabase.driver.sql.references-test+ (:require+ [clojure.test :refer :all]+ [macaw.core :as macaw]+ [metabase.driver.sql.references :as sql.references]))++(defn- ->references [query]+ (->> query macaw/parsed-query macaw/->ast (sql.references/field-references :sql)))++(deftest garbage-test+ (is (= {:used-fields #{}+ :returned-fields []+ :bad-sql true}+ (->references "nothing"))))++(deftest basic-select-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}]}+ (->references "select * from (select a, b from products)"))))++(deftest basic-join-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields+ [{:type :all-columns, :table {:table "products"}}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]}+ (->references "select products.*, orders.id from products inner join orders on products.id = orders.product_id"))))++(deftest indeterminite-join-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "orders"}}+ {:type :all-columns, :table {:table "products"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "orders"}}+ {:type :all-columns, :table {:table "products"}}]]}]}+ (->references "select id from products inner join orders on products.id = orders.product_id"))))++(deftest table-wildcard-join-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:type :all-columns, :table {:table "orders"}}]}+ (->references "select orders.* from products inner join orders on products.id = orders.product_id"))))++(deftest wildcard-join-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:type :all-columns, :table {:table "orders"}}+ {:type :all-columns, :table {:table "products"}}]}+ (->references "select * from products inner join orders on products.id = orders.product_id"))))++(deftest basic-alias-test+ (is (= {:used-fields+ #{{:column "c",+ :alias "d",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}+ {:column "a",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}+ {:column "c",+ :alias "d",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}]}+ (->references "select p.a as b, p.c as d from products p"))))++(deftest basic-nested-query-test+ (is (= {:used-fields+ #{{:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}]}+ (->references "select a, b from (select a, b from products)"))))++(deftest renamed-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "a",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields+ [{:column "a",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}]}+ (->references "select b as c from (select a as b from products)"))))++(deftest broken-nested-query-test+ (is (= {:used-fields+ #{{:column "b",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[]]}]}+ (->references "select a from (select b from products)"))))++(deftest different-case-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}]}+ (->references "select A from (select a from orders)"))))++(deftest wildcard-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}]}+ (->references "select * from (select a from orders)"))))++(deftest table-wildcard-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}]}+ (->references "select o.* from (select a from orders) o"))))++(deftest bad-table-wildcard-nested-query-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:type :invalid-table-wildcard, :table "foo"}]}+ (->references "select foo.* from (select a from orders)"))))++(deftest bad-table-name-test+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[]]}]}+ (->references "select bad.a from products"))))++(deftest basic-where-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]}+ (->references "select * from products where category = 'hello'"))))++(deftest basic-aggregation-test+ (is (=? {:used-fields #{},+ :returned-fields [{:alias nil, :type :custom-field, :used-fields #{}}]}+ (->references "select count(*) from products"))))++(deftest named-aggregation-test+ (is (=? {:used-fields #{},+ :returned-fields [{:alias "count", :type :custom-field, :used-fields #{}}]}+ (->references "select count(*) as count from products"))))++(deftest extra-names-test+ (is (= {:used-fields+ #{{:column "col",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:database "db",+ :schema "schema",+ :table "table"}}]]}},+ :returned-fields+ [{:column "col",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:database "db",+ :schema "schema",+ :table "table"}}]]}]}+ (->references "select db.schema.table.col from db.schema.table"))))++(deftest basic-grouping-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:alias "sum",+ :type :custom-field,+ :used-fields+ #{{:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]}+ (->references "select sum(total) as sum from orders group by category"))))++(deftest basic-arg-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]}+ (->references "select * from products where category = ?"))))++(deftest basic-case-test+ (is (=? {:used-fields+ #{{:column "tax",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "subtotal",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "tax",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "total",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "subtotal",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}}}]}+ (->references "select case when total < 0 then -subtotal else tax end from orders"))))++(deftest switch-case-test+ (is (=? {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields+ [{:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}}}]}+ (->references "select case category when 'Gizmo' then 'is gizmo' else 'is not gizmo' end from products"))))++(deftest basic-select-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]}+ (->references "select (select category from products where products.id = orders.product_id), * from orders"))))++(deftest named-select-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias "category2",+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]}+ (->references "select (select category from products where products.id = orders.product_id) as category2, * from orders"))))++(deftest nested-select-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ []+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ []+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]}+ (->references "select (select (select category from products where products.id = orders.product_id)), * from orders"))))++(deftest select-subquery-with-normal-subquery-test+ (is (= {:used-fields+ #{{:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]+ [{:type :all-columns,+ :table {:table "orders"}}]]}+ {:type :all-columns, :table {:table "orders"}}]}+ (->references "select (select * from (select category from products where products.id = orders.product_id) sub), * from orders"))))++(deftest nested-select-subquery-with-reference-to-middle-select-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "category",+ :alias nil,+ :type :single-column,+ :source-columns+ [[]+ [{:type :all-columns, :table {:table "products"}}]+ [{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields+ [{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns+ [[]+ [{:type :all-columns, :table {:table "products"}}]+ [{:type :all-columns, :table {:table "orders"}}]]}]}+ (->references "select (select (select category) from products where products.id = orders.product_id) from orders"))))++(deftest nested-select-subquery-with-direct-match-in-outer-query+ (is (= {:used-fields+ #{{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}+ {:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "t1"}}]+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}},+ :returned-fields+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table "t1"}}]+ [{:column "a",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}]}+ (->references "select (select a from t1) from (select a from t2)"))))++(deftest basic-exists-test+ (is (= {:used-fields+ #{{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}+ {:column "user_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "o", :table "orders"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}+ {:column "email",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}},+ :returned-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}+ {:column "email",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table-alias "u", :table "users"}}]]}]}+ (->references "SELECT u.name, u.email+FROM users u+WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))++(deftest basic-not-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]}+ (->references "select * from products where not (category = 'Gizmo')"))))++(deftest basic-is-null-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]}+ (->references "select * from products where category is null"))))++(deftest negated-is-null-test+ (is (= {:used-fields+ #{{:column "category",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "products"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]}+ (->references "select * from products where category is not null"))))++(deftest basic-between-test+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "left",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}+ {:column "right",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "orders"}}]]}},+ :returned-fields [{:type :all-columns, :table {:table "orders"}}]}+ (->references "select * from orders where created_at between left and right"))))++(deftest basic-cte-test+ (is (= {:used-fields+ #{{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "active",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}},+ :returned-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}]}+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)+SELECT * FROM active_users"))))++(deftest unused-cte-test+ (is (= {:used-fields+ #{{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "active",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}},+ :returned-fields [{:type :all-columns,+ :table {:table "products"}}]}+ (->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)+SELECT * FROM products"))))++(deftest shadowed-cte-test+ (is (= {:used-fields+ #{{:column "y",+ :alias "x",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "y",+ :alias "a",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "z",+ :alias "y",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "x",+ :alias "z",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "z",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "x",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}},+ :returned-fields+ [{:column "y",+ :alias "a",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "z",+ :alias "b",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}+ {:column "x",+ :alias "c",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "a"}}]]}]}+ (->references "WITH c AS (+ WITH b AS (+ SELECT+ x as z,+ y as x,+ z as y+ FROM a+ )+ SELECT+ x as a,+ y as b,+ z as c+ FROM b+)+SELECT+ a,+ b,+ c+FROM c;"))))++(deftest recursive-cte-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "h", :table "emp_hierarchy"}}]]}+ {:alias "name",+ :type :composite-field,+ :member-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}]}+ {:column "manager_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}+ {:column "level",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "h",+ :table "emp_hierarchy"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}+ {:alias "level",+ :type :composite-field,+ :member-fields+ [{:alias "level", :type :custom-field, :used-fields #{}}+ {:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "level",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "h",+ :table "emp_hierarchy"}}]]}}}]}+ {:column "manager_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}},+ :returned-fields+ [{:alias "name",+ :type :composite-field,+ :member-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "e",+ :table "employees"}}]]}]}+ {:alias "level",+ :type :composite-field,+ :member-fields+ [{:alias "level", :type :custom-field, :used-fields #{}}+ {:alias nil,+ :type :custom-field,+ :used-fields+ #{{:column "level",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "h",+ :table "emp_hierarchy"}}]]}}}]}]}+ (->references "WITH RECURSIVE emp_hierarchy AS (+ SELECT id, name, manager_id, 0 AS level+ FROM employees+ WHERE manager_id IS NULL+ UNION ALL+ SELECT e.id, e.name, e.manager_id, h.level + 1+ FROM employees e+ JOIN emp_hierarchy h ON e.manager_id = h.id+)+SELECT name, level FROM emp_hierarchy"))))++(deftest basic-union-test+ (is (= {:used-fields+ #{{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}},+ :returned-fields+ [{:alias "id",+ :type :composite-field,+ :member-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}]}+ {:alias "name",+ :type :composite-field,+ :member-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "users"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "archived_users"}}]]}]}]}+ (->references "SELECT id, name FROM users+UNION+SELECT id, name FROM archived_users"))))++(deftest row-number-test+ (is (= {:used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}},+ :returned-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}+ {:alias "rank",+ :type :custom-field,+ :used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns,+ :table {:table "employees"}}]]}}}]}+ (->references "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) AS rank+FROM employees"))))++(deftest partition-by-test+ (is (= {:used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "department",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}},+ :returned-fields+ [{:column "name",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:alias "dept_rank",+ :type :custom-field,+ :used-fields+ #{{:column "salary",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}+ {:column "department",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table "employees"}}]]}}}]}+ (->references "SELECT name,+ RANK() OVER (PARTITION BY department ORDER BY salary DESC) AS dept_rank+FROM employees"))))++(deftest week-test+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}},+ :returned-fields+ [{:alias "created_at",+ :type :custom-field,+ :used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "count", :type :custom-field, :used-fields #{}}]}+ (->references "SELECT+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ ) AS \"created_at\",+ COUNT(*) AS \"count\"+FROM+ \"public\".\"orders\"+GROUP BY+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ORDER BY+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ ) ASC"))))++(deftest week-of-year-test+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}},+ :returned-fields+ [{:alias "created_at",+ :type :custom-field,+ :used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "count", :type :custom-field, :used-fields #{}}]}+ (->references "SELECT+ CEIL(+ (+ CAST(+ extract(+ doy+ from+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ ) AS integer+ ) / 7.0+ )+ ) AS \"created_at\",+ COUNT(*) AS \"count\"+FROM+ \"public\".\"orders\"+GROUP BY+ CEIL(+ (+ CAST(+ extract(+ doy+ from+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ ) AS integer+ ) / 7.0+ )+ )+ORDER BY+ CEIL(+ (+ CAST(+ extract(+ doy+ from+ (+ DATE_TRUNC(+ 'week',+ (\"public\".\"orders\".\"created_at\" + INTERVAL '1 day')+ ) + INTERVAL '-1 day'+ )+ ) AS integer+ ) / 7.0+ )+ ) ASC"))))++;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}+(deftest complicated-test-1+ (is (= {:used-fields+ #{{:column "title",+ :alias "title",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "discount",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "rating",+ :alias "rating",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:alias "upper_category",+ :type :custom-field,+ :used-fields+ #{{:column "category",+ :alias "category",+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}}}+ {:column "ean",+ :alias "ean",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "created_at",+ :alias "created_at",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "product_id",+ :alias "product_id",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "category",+ :alias "Products__category",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "product_id",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "category",+ :alias "category",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "created_at",+ :alias "created_at",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "discount",+ :alias "discount",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "vendor",+ :alias "vendor",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "total",+ :alias "total",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}+ {:column "id",+ :alias "Products__id",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "id",+ :alias "id",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}+ {:column "price",+ :alias "price",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}},+ :returned-fields+ [{:alias "created_at",+ :type :custom-field,+ :used-fields+ #{{:column "created_at",+ :alias "created_at",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "upper_category",+ :type :custom-field,+ :used-fields+ #{{:column "category",+ :alias "category",+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:schema "public",+ :table "products"}}]]}}}+ {:alias "sum",+ :type :custom-field,+ :used-fields+ #{{:column "total",+ :alias "total",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "sum_2",+ :type :custom-field,+ :used-fields+ #{{:column "discount",+ :alias "discount",+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:schema "public",+ :table "orders"}}]]}}}+ {:alias "count", :type :custom-field, :used-fields #{}}]}+ (->references "SELECT+ DATE_TRUNC('month', \"source\".\"created_at\") AS \"created_at\",+ \"source\".\"upper_category\" AS \"upper_category\",+ SUM(\"source\".\"total\") AS \"sum\",+ SUM(\"source\".\"discount\") AS \"sum_2\",+ COUNT(*) AS \"count\"+FROM+ (+ SELECT+ \"public\".\"orders\".\"product_id\" AS \"product_id\",+ \"public\".\"orders\".\"total\" AS \"total\",+ \"public\".\"orders\".\"discount\" AS \"discount\",+ \"public\".\"orders\".\"created_at\" AS \"created_at\",+ UPPER(\"Products\".\"category\") AS \"upper_category\",+ \"Products\".\"id\" AS \"Products__id\",+ \"Products\".\"category\" AS \"Products__category\"+ FROM+ \"public\".\"orders\"++LEFT JOIN (+ SELECT+ \"public\".\"products\".\"id\" AS \"id\",+ \"public\".\"products\".\"ean\" AS \"ean\",+ \"public\".\"products\".\"title\" AS \"title\",+ \"public\".\"products\".\"category\" AS \"category\",+ \"public\".\"products\".\"vendor\" AS \"vendor\",+ \"public\".\"products\".\"price\" AS \"price\",+ \"public\".\"products\".\"rating\" AS \"rating\",+ \"public\".\"products\".\"created_at\" AS \"created_at\"+ FROM+ \"public\".\"products\"+ ) AS \"Products\" ON \"public\".\"orders\".\"product_id\" = \"Products\".\"id\"++WHERE+ \"public\".\"orders\".\"discount\" > 0+ ) AS \"source\"+GROUP BY+ DATE_TRUNC('month', \"source\".\"created_at\"),+ \"source\".\"upper_category\"+ORDER BY+ DATE_TRUNC('month', \"source\".\"created_at\") ASC,+ \"source\".\"upper_category\" ASC"))))++;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the+;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.+^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}+(deftest complicated-test-2+ (is (= {:used-fields+ #{{:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "is_converted",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "annual_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "base_fee",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "account_id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_alias",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_paid_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_active_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_active_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_paid_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_trialing_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "is_active_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:alias "is_active_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_active_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_trialing_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:column "customer_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_trialing",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:column "is_active",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:alias "is_paid_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_paid_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_paid_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}+ {:column "valid_to",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "valid_from",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table+ {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}},+ :returned-fields+ [{:column "id",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "customer_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "created_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "is_converted",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "is_active",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_alias",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "base_fee",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "annual_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}+ {:alias "is_trialing",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_trialing_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_active_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_cloud",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_trialing_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_active_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_self_hosted",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "deployment",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_active_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_enterprise",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_active_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_starter",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}+ {:alias "is_active_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "subscription_status",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}}}+ {:alias "is_paid_accounts_pro",+ :type :custom-field,+ :used-fields+ #{{:column "trial_ended_at",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "lifetime_value",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "a", :schema "dbt_models", :table "account"}}]]}+ {:column "plan_name",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns,+ :table {:table-alias "ah", :schema "dbt_models", :table "account_history"}}]]}+ {:column "month",+ :alias nil,+ :type :single-column,+ :source-columns+ [[{:type :all-columns, :table {:table-alias "m", :table "months"}}]]}}}]}+ (->references "with months as (+ select+ generate_series(date_trunc('month', '2024-01-01'::date),+ date_trunc('month', current_date) - interval '1 month',+ '1 month')::date as month++)+, monthly_account_subscription_status as (+ select+ a.id,+ m.month,++ max(case when ah.subscription_status is not null and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing,++ -- cloud+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_cloud,+ max(case when ah.subscription_status is not null and a.deployment = 'cloud' then 1 else 0 end) as is_active_accounts_cloud,+ max(case when a.lifetime_value > 0 and a.deployment = 'cloud' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_cloud,++ -- self-hosted+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' and m.month <= date_trunc('month', a.trial_ended_at) then 1 else 0 end) as is_trialing_accounts_self_hosted,+ max(case when ah.subscription_status is not null and a.deployment = 'self-hosted' then 1 else 0 end) as is_active_accounts_self_hosted,+ max(case when a.lifetime_value > 0 and a.deployment = 'self-hosted' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_self_hosted,++ -- enterprise+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'enterprise' then 1 else 0 end) as is_active_accounts_enterprise,+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'enterprise' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_enterprise,++ -- starter+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'starter' then 1 else 0 end) as is_active_accounts_starter,+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'starter' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_starter,++ -- pro+ max(case when ah.subscription_status is not null and coalesce(ah.plan_name, a.plan_name) = 'pro' then 1 else 0 end) as is_active_accounts_pro,+ max(case when a.lifetime_value > 0 and coalesce(ah.plan_name, a.plan_name) = 'pro' and (m.month >= date_trunc('month', a.trial_ended_at) or a.trial_ended_at is null) then 1 else 0 end) as is_paid_accounts_pro+ from dbt_models.account a+ left join dbt_models.account_history ah+ on a.id = ah.account_id+ left join months m+ on m.month between date_trunc('month', ah.valid_from) and date_trunc('month', ah.valid_to)+ group by 1,2+)+select+ a.id,+ a.customer_name,+ a.created_at,+ a.is_converted,+ a.lifetime_value,+ a.is_active,+ a.subscription_status,+ a.deployment,+ a.plan_name,+ a.plan_alias,+ a.base_fee,+ a.annual_value,++ -- monthly account status+ mas.month,+ mas.is_trialing,+ mas.is_trialing_accounts_cloud,+ mas.is_active_accounts_cloud,+ mas.is_paid_accounts_cloud,+ mas.is_trialing_accounts_self_hosted,+ mas.is_active_accounts_self_hosted,+ mas.is_paid_accounts_self_hosted,+ mas.is_active_accounts_enterprise,+ mas.is_paid_accounts_enterprise,+ mas.is_active_accounts_starter,+ mas.is_paid_accounts_starter,+ mas.is_active_accounts_pro,+ mas.is_paid_accounts_pro++from dbt_models.account a+left join monthly_account_subscription_status mas+ on a.id = mas.id+;"))))diff --git a/test/metabase/lib/test_util/metadata_providers/mock.cljc b/test/metabase/lib/test_util/metadata_providers/mock.cljcindex d4a64fe88c0..60a32f69723 100644--- a/test/metabase/lib/test_util/metadata_providers/mock.cljc+++ b/test/metabase/lib/test_util/metadata_providers/mock.cljc@@ -39,6 +39,8 @@[:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/segment :metadata/segment)]]][:native-query-snippets {:optional true}[:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/native-query-snippet :metadata/native-query-snippet)]]]+ [:transforms {:optional true}+ [:maybe [:sequential (with-optional-lib-type :map :metadata/transform)]]][:settings {:optional true}[:maybe [:map-of :keyword any?]]]])@@ -54,7 +56,8 @@:metadata/card :cards:metadata/segment :segments:metadata/native-query-snippet :native-query-snippets- :metadata/metric :cards)]+ :metadata/metric :cards+ :metadata/transform :transforms)](into [](comp (metadata.protocols/default-spec-filter-xform metadata-spec)(map #(assoc % :lib/type metadata-type)))diff --git a/test/metabase/revisions/impl/card_test.clj b/test/metabase/revisions/impl/card_test.cljindex 0109d9c2c9e..29ea1bfb0f2 100644--- a/test/metabase/revisions/impl/card_test.clj+++ b/test/metabase/revisions/impl/card_test.clj@@ -137,7 +137,10 @@:card_schema;; we don't expect a description for this column because it should never change;; once created by the migration- :dataset_query_metrics_v2_migration_backup} col)+ :dataset_query_metrics_v2_migration_backup+ ;; `dependency_analysis_version` is an internal bookkeeping field. It doesn't affect the+ ;; actual card itself, so no description is necessary.+ :dependency_analysis_version} col)(testing (format "we should have a revision description for %s" col)(let [diff-strings (revision/diff-strings;; TODO -- huh? Shouldn't this be testing against `:model/Card` here???