The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Transforms inspector (#68675)
Nicola Mometto09ee4178ca31
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 54eb4f82bde..dc47a3256f6 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1575,12 +1575,10 @@transforms{:team "Querying"- :api #{metabase.transforms.api.transform- metabase.transforms.api.transform-job- metabase.transforms.api.transform-tag- metabase.transforms.core+ :api #{metabase.transforms.coremetabase.transforms.feature-gatingmetabase.transforms.init+ metabase.transforms.interfacemetabase.transforms.schemametabase.transforms.util}:uses #{analytics@@ -1595,10 +1593,8 @@premium-featuresqueriesquery-processor- requestrevisionssettings- sql-toolssynctasktask-history@@ -1607,6 +1603,35 @@enterprise/transforms-pythonenterprise/workspaces}}+ transforms-inspector+ {:team "Gadget"+ :api #{metabase.transforms-inspector.core+ metabase.transforms-inspector.schema}+ :uses #{driver+ lib+ lib-be+ query-processor+ sql-tools+ transforms+ util}}++ transforms-rest+ {:team "Gadget"+ :api #{metabase.transforms-rest.api.transform+ metabase.transforms-rest.api.transform-job+ metabase.transforms-rest.api.transform-tag}+ :uses #{api+ driver+ events+ models+ query-processor+ request+ server+ sql-tools+ transforms+ transforms-inspector+ util}}+types{:team "Querying":api #{metabase.types.corediff --git a/.gitignore b/.gitignoreindex a735d20cfd7..3a5d3a011a6 100644--- a/.gitignore+++ b/.gitignore@@ -16,6 +16,7 @@.nrepl-port.portal.vscode+/.zed/*.h2.db/*.lock.db/*.mv.dbdiff --git a/deps.edn b/deps.ednindex 64d2522efd3..d6f381e5dce 100644--- a/deps.edn+++ b/deps.edn@@ -84,7 +84,7 @@io.github.eerohele/pp {#_#_:git/tag "2024-11-13.77" ; super fast pretty-printing library:git/sha "ffc4edc482c057cd9412c62f018a41e9140c618b":git/url "https://github.com/eerohele/pp"}- io.github.metabase/macaw {:mvn/version "0.2.33"} ; Parse native SQL queries+ io.github.metabase/macaw {:mvn/version "0.2.36"} ; Parse native SQL queriesio.netty/netty-codec-http {:mvn/version "4.2.9.Final"} ; override of transitive dep from aws s3io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collectorjunegunn/grouper {:mvn/version "0.1.1"} ; Batch processing helperdiff --git a/docs/api.json b/docs/api.jsonindex 49a0d87e0a3..ff20f43410a 100644--- a/docs/api.json+++ b/docs/api.json@@ -5176,10 +5176,10 @@"required" : [ "conversation_id" ]},"metabase-enterprise.metabot-v3.tools.transforms.transform-source" : {- "$ref" : "#/components/schemas/metabase.transforms.api.transform.transform-source"+ "$ref" : "#/components/schemas/metabase.transforms-rest.api.transform.transform-source"},"metabase-enterprise.metabot-v3.tools.transforms.transform-target" : {- "$ref" : "#/components/schemas/metabase.transforms.api.transform.transform-target"+ "$ref" : "#/components/schemas/metabase.transforms-rest.api.transform.transform-target"},"metabase-enterprise.permission-debug.impl.group-id" : {"type" : "integer"@@ -13922,14 +13922,14 @@"type" : "string","enum" : [ "events" ]},- "metabase.transforms.api.transform.run-trigger" : {+ "metabase.transforms-rest.api.transform.run-trigger" : {"type" : "string","enum" : [ "none", "global-schedule" ]},- "metabase.transforms.api.transform.transform-source" : {+ "metabase.transforms-rest.api.transform.transform-source" : {"$ref" : "#/components/schemas/metabase.transforms.schema.transform-source"},- "metabase.transforms.api.transform.transform-target" : {+ "metabase.transforms-rest.api.transform.transform-target" : {"$ref" : "#/components/schemas/metabase.transforms.schema.transform-target"},"metabase.transforms.schema.append-config" : {@@ -43968,7 +43968,7 @@"$ref" : "#/components/schemas/metabase.transforms.schema.transform-target"},"run_trigger" : {- "$ref" : "#/components/schemas/metabase.transforms.api.transform.run-trigger"+ "$ref" : "#/components/schemas/metabase.transforms-rest.api.transform.run-trigger"},"description" : {"oneOf" : [ {@@ -45141,7 +45141,7 @@"$ref" : "#/components/schemas/metabase.transforms.schema.transform-target"},"run_trigger" : {- "$ref" : "#/components/schemas/metabase.transforms.api.transform.run-trigger"+ "$ref" : "#/components/schemas/metabase.transforms-rest.api.transform.run-trigger"},"description" : {"oneOf" : [ {diff --git a/e2e/support/helpers/e2e-data-studio-helpers.ts b/e2e/support/helpers/e2e-data-studio-helpers.tsindex ece94a175cc..f9e14dbf4ad 100644--- a/e2e/support/helpers/e2e-data-studio-helpers.ts+++ b/e2e/support/helpers/e2e-data-studio-helpers.ts@@ -52,6 +52,7 @@ export const DataStudio = {definitionTab: () =>DataStudio.Transforms.header().findByText("Definition"),runTab: () => DataStudio.Transforms.header().findByText("Run"),+ inspectTab: () => DataStudio.Transforms.header().findByText("Inspect"),targetTab: () => DataStudio.Transforms.header().findByText("Target"),settingsTab: () => DataStudio.Transforms.header().findByText("Settings"),dependenciesTab: () =>@@ -60,6 +61,9 @@ export const DataStudio = {cy.visit("/data-studio/transforms");DataStudio.Transforms.list().should("be.visible");},+ visitInspect: (transformId: TransformId) => {+ cy.visit(`/data-studio/transforms/${transformId}/inspect`);+ },visitSettingsTab: (transformId: TransformId) =>cy.visit(`/data-studio/transforms/${transformId}/settings`),pythonResults: () => cy.findByTestId("python-results"),diff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.tsindex 8d7ca36d99d..ec203205767 100644--- a/e2e/support/helpers/e2e-transform-helpers.ts+++ b/e2e/support/helpers/e2e-transform-helpers.ts@@ -3,6 +3,7 @@ import type {Collection,CollectionId,PythonTransformTableAliases,+ SchemaName,TransformId,TransformRun,TransformRunStatus,@@ -128,6 +129,7 @@ export function createSqlTransform({visitTransform,sourceCheckpointStrategy,name = "SQL transform",+ wrapId = true,}: {name?: string;sourceQuery: string;@@ -136,6 +138,7 @@ export function createSqlTransform({tagIds?: TransformTagId[];visitTransform?: boolean;sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;+ wrapId?: boolean;}) {return createTransform({@@ -159,7 +162,7 @@ export function createSqlTransform({},tag_ids: tagIds,},- { wrapId: true, visitTransform },+ { wrapId, visitTransform },);}@@ -237,3 +240,43 @@ export function createAndRunMbqlTransform({});});}++/**+ * Creates a SQL transform and runs it to create a table.+ * @return: information about the created transform+ */+export function createAndRunSqlTransform({+ sourceQuery,+ targetTable,+ targetSchema,+ tagIds,+ sourceCheckpointStrategy,+ name = "Test SQL transform",+}: {+ sourceQuery: string;+ targetTable: string;+ targetSchema: SchemaName;+ tagIds?: TransformTagId[];+ sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;+ name?: string;+}): Cypress.Chainable<{+ transformId: TransformId;+}> {+ return createSqlTransform({+ sourceQuery,+ targetTable,+ targetSchema,+ tagIds,+ name,+ sourceCheckpointStrategy,+ visitTransform: false,+ wrapId: false,+ }).then(({ body: transform }) => {+ cy.request("POST", `/api/transform/${transform.id}/run`);+ waitForSucceededTransformRuns();++ return cy.wrap({+ transformId: transform.id,+ });+ });+}diff --git a/e2e/test/scenarios/data-studio/transforms/transforms-inspect.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms-inspect.cy.spec.tsnew file mode 100644index 00000000000..1651b73f409--- /dev/null+++ b/e2e/test/scenarios/data-studio/transforms/transforms-inspect.cy.spec.ts@@ -0,0 +1,419 @@+import { WRITABLE_DB_ID } from "e2e/support/cypress_data";+import type { JoinStrategy, SchemaName, TransformId } from "metabase-types/api";++const { H } = cy;++const SOURCE_TABLE = "Animals";+const TARGET_SCHEMA = "Schema A";+const JOIN_SCHEMA = "Schema B";++describe("scenarios > data-studio > transforms > inspect", () => {+ beforeEach(() => {+ H.restore("postgres-writable");+ H.resetTestTable({ type: "postgres", table: "many_schemas" });+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: SOURCE_TABLE });++ cy.intercept("GET", "/api/transform/*/inspect").as("inspectorDiscovery");+ cy.intercept("GET", "/api/transform/*/inspect/*").as("inspectorLens");++ H.resetSnowplow();+ H.enableTracking();+ });++ afterEach(() => {+ H.expectNoBadSnowplowEvents();+ });++ describe("pre-run state", () => {+ it("should show alert when transform has not been run, then show inspect after running", () => {+ H.createMbqlTransform({+ sourceTable: SOURCE_TABLE,+ targetTable: "inspect_prerun_table",+ targetSchema: TARGET_SCHEMA,+ name: "Pre-run inspect transform",+ visitTransform: false,+ }).then(({ body: transform }) => {+ H.DataStudio.Transforms.visitInspect(transform.id);+ });++ cy.findByRole("alert").should(+ "have.text",+ "To inspect the transform you need to run it first.",+ );++ cy.findAllByTestId("run-button").first().click();++ cy.wait("@inspectorDiscovery");++ cy.findByRole("alert").should("not.exist");+ cy.findByRole("tab", { name: /Summary/ }).should("be.visible");+ cy.findByRole("heading", { name: /1 input table/i }).should("be.visible");+ cy.findByRole("heading", { name: /1 output table/i }).should(+ "be.visible",+ );++ cy.findAllByRole("treegrid").should("have.length", 4);+ cy.findAllByRole("treegrid")+ .first()+ .findByText("Animals")+ .should("be.visible");+ });+ });++ describe("generic-summary lens", () => {+ it("should show Summary tab after running an MBQL transform", () => {+ H.createAndRunMbqlTransform({+ sourceTable: SOURCE_TABLE,+ targetTable: "inspect_mbql_table",+ targetSchema: TARGET_SCHEMA,+ name: "MBQL inspect transform",+ }).then(({ transformId }) => {+ H.DataStudio.Transforms.visitInspect(transformId);+ });++ cy.wait("@inspectorDiscovery");++ cy.findByRole("tab", { name: /Summary/ }).should(+ "have.attr",+ "aria-selected",+ "true",+ );++ cy.findByRole("heading", { name: /1 input table/i }).should("be.visible");+ cy.findByRole("heading", { name: /1 output table/i }).should(+ "be.visible",+ );++ cy.findByTestId("generic-summary-tables").within(() => {+ cy.findAllByRole("treegrid")+ .eq(0)+ .within(() => {+ cy.findByRole("row").within(() => {+ cy.findAllByRole("gridcell").eq(0).should("have.text", "Animals");+ cy.findAllByRole("gridcell").eq(1).should("have.text", "3");+ cy.findAllByRole("gridcell").eq(2).should("have.text", "2");+ });+ });++ cy.findAllByRole("treegrid")+ .eq(1)+ .within(() => {+ cy.findByRole("row").within(() => {+ cy.findAllByRole("gridcell")+ .eq(0)+ .should("have.text", "inspect_mbql_table");+ cy.findAllByRole("gridcell").eq(1).should("have.text", "3");+ cy.findAllByRole("gridcell").eq(2).should("have.text", "2");+ });+ });+ });++ cy.findByTestId("generic-summary-fields").within(() => {+ cy.findAllByRole("treegrid")+ .eq(0)+ .within(() => {+ cy.findAllByRole("row").should("have.length", 3);+ cy.findAllByRole("row").eq(0).should("have.text", "Animals (2)");+ cy.findAllByRole("row")+ .eq(1)+ .within(() => {+ cy.findAllByRole("gridcell").eq(0).should("have.text", "Name");+ cy.findAllByRole("gridcell").eq(1).should("have.text", "Text");+ cy.findAllByRole("gridcell").eq(2).should("have.text", "3");+ cy.findAllByRole("gridcell")+ .eq(3)+ .should("have.text", "0.00 %");+ });+ cy.findAllByRole("row")+ .eq(2)+ .within(() => {+ cy.findAllByRole("gridcell").eq(0).should("have.text", "Score");+ cy.findAllByRole("gridcell")+ .eq(1)+ .should("have.text", "Integer");+ cy.findAllByRole("gridcell").eq(2).should("have.text", "3");+ cy.findAllByRole("gridcell")+ .eq(3)+ .should("have.text", "0.00 %");+ });+ });+ });++ H.expectUnstructuredSnowplowEvent({+ event: "transform_inspect_lens_loaded",+ event_detail: "generic-summary",+ });+ });+ });++ describe("join-analysis lens", () => {+ it("should show Join Analysis tab when transform has joins", () => {+ createAndRunMbqlJoinTransform({+ name: "Join MBQL inspect transform",+ sourceSchema: TARGET_SCHEMA,+ targetTable: "inspect_join_table",+ });++ cy.wait("@inspectorDiscovery");++ cy.findByRole("tab", { name: /Summary/ }).should("be.visible");+ cy.findByRole("tab", { name: /Join Analysis/ }).should("be.visible");+ });++ it("should display join step data in tree table", () => {+ createAndRunMbqlJoinTransform({+ name: "Join tree inspect transform",+ sourceSchema: TARGET_SCHEMA,+ targetTable: "inspect_join_tree_table",+ });++ const tabName = /Join Analysis/;++ cy.wait("@inspectorDiscovery");+ cy.wait("@inspectorLens");++ cy.findByRole("tab", { name: tabName }).within(() => {+ cy.findByLabelText(/clock icon/i).should("be.visible");+ });+ cy.findByRole("tab", { name: tabName }).click();++ cy.wait("@inspectorLens");++ cy.findByRole("tab", { name: tabName }).within(() => {+ cy.findByLabelText(/clock icon/i).should("not.exist");+ });++ cy.findByRole("treegrid").within(() => {+ cy.findByText("Join").should("be.visible");+ cy.findByText("Output").should("be.visible");+ cy.findByText("Matched").should("be.visible");+ cy.findByText("Table rows").should("be.visible");+ });++ cy.findByRole("heading", { name: /1 join/i }).should("be.visible");+ });++ it("should show unmatched rows alert for left join with non-matching rows", () => {+ H.resetTestTable({ type: "postgres", table: "no_pk_table" });+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: "no_pk_table" });++ createAndRunMbqlJoinTransform({+ name: "Left join unmatched transform",+ targetTable: "inspect_unmatched_table",+ sourceTable: "no_pk_table",+ sourceSchema: undefined,+ joinTable: SOURCE_TABLE,+ joinSchema: TARGET_SCHEMA,+ joinStrategy: "left-join",+ });++ cy.wait("@inspectorDiscovery");+ cy.wait("@inspectorLens");++ cy.findByRole("tab", { name: /Join Analysis/ }).click();++ cy.wait("@inspectorLens");++ // Wait for trigger evaluation — drill button appears once card stats are loaded+ cy.findByRole("button", {+ name: /Unmatched rows in Animals - Name/i,+ }).should("be.visible");++ // Expand the alert by clicking the warning icon in the first cell+ cy.findByRole("treegrid").within(() => {+ cy.findAllByRole("gridcell").first().findByRole("button").click();+ cy.findByText(/Join 'Animals - Name' has >20% unmatched rows/).should(+ "be.visible",+ );+ });++ H.expectUnstructuredSnowplowEvent({+ event: "transform_inspect_alert_clicked",+ });+ });+ });++ describe("drill-down lenses", () => {+ it("loads unmatched-rows drill-down lens when triggered", () => {+ H.resetTestTable({ type: "postgres", table: "no_pk_table" });+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: "no_pk_table" });+ createAndRunMbqlJoinTransform({+ name: "Left join unmatched transform",+ targetTable: "inspect_unmatched_table",+ sourceTable: "no_pk_table",+ sourceSchema: undefined,+ joinTable: SOURCE_TABLE,+ joinSchema: TARGET_SCHEMA,+ joinStrategy: "left-join",+ });++ cy.findByRole("tab", { name: /Join Analysis/ }).click();++ cy.findByRole("button", {+ name: /Unmatched rows in Animals - Name/,+ }).click();++ H.expectUnstructuredSnowplowEvent({+ event: "transform_inspect_drill_lens_clicked",+ triggered_from: "join_analysis",+ });++ const tabName = /Unmatched Rows/;++ cy.wait("@inspectorLens");+ cy.findByRole("tab", { name: tabName }).click();++ cy.findByRole("heading", { name: /Unmatched Row Samples/ }).should(+ "be.visible",+ );+ cy.findByRole("link", {+ name: /Animals - Name: Rows with key but no match/,+ }).should("be.visible");++ cy.findAllByTestId("visualization-root")+ .eq(0)+ .within(() => {+ cy.findByTestId("table-footer").should("have.text", "3 rows");+ });++ H.expectUnstructuredSnowplowEvent({+ event: "transform_inspect_lens_loaded",+ event_detail: "unmatched-rows",+ });++ cy.findByRole("tab", { name: tabName }).within(() => {+ cy.findByRole("button", { name: /Close tab/i }).click();+ });++ cy.findByRole("link", {+ name: tabName,+ }).should("not.exist");++ H.expectUnstructuredSnowplowEvent({+ event: "transform_inspect_drill_lens_closed",+ });+ });+ });++ describe("column-comparison lens", () => {+ it("should show Column Distributions lens", () => {+ H.createAndRunMbqlTransform({+ sourceTable: SOURCE_TABLE,+ targetTable: "inspect_coldist_table",+ targetSchema: TARGET_SCHEMA,+ name: "ColDist inspect transform",+ }).then(({ transformId }) => {+ H.DataStudio.Transforms.visitInspect(transformId);+ });++ cy.wait("@inspectorDiscovery");++ cy.findByRole("tab", { name: /Column Distributions/ }).click();++ cy.findByRole("heading", { name: /2 matched columns/i }).should(+ "be.visible",+ );++ cy.findAllByTestId("visualization-root")+ .should("have.length", 4)+ .each((visualization) => {+ cy.wrap(visualization).within(() => {+ cy.findByRole("link").should("exist");+ });+ });+ });+ });++ describe("sql transforms", () => {+ it("should show Summary tab for a SQL transform", () => {+ H.createAndRunSqlTransform({+ name: "SQL inspect transform",+ sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,+ targetTable: "inspect_sql_table",+ targetSchema: TARGET_SCHEMA,+ }).then(({ transformId }) => {+ H.DataStudio.Transforms.visitInspect(transformId);+ });++ cy.wait("@inspectorDiscovery");++ cy.findByRole("tab", { name: /Summary/ }).should(+ "have.attr",+ "aria-selected",+ "true",+ );++ cy.findByRole("heading", { name: /1 input table/i }).should("be.visible");+ cy.findByRole("heading", { name: /1 output table/i }).should(+ "be.visible",+ );+ });+ });+});++// This transform with 50% unmatched rows triggers >20% alert and drill lens+function createAndRunMbqlJoinTransform({+ name,+ targetTable,+ sourceTable = SOURCE_TABLE,+ sourceSchema,+ joinTable = SOURCE_TABLE,+ joinSchema = JOIN_SCHEMA,+ joinStrategy = "inner-join",+}: {+ name: string;+ targetTable: string;+ sourceTable?: string;+ sourceSchema: SchemaName | undefined;+ joinTable?: string;+ joinSchema?: SchemaName;+ joinStrategy?: JoinStrategy;+}): Cypress.Chainable<{ transformId: TransformId }> {+ return H.cypressWaitAll([+ H.getTableId({ name: sourceTable, schema: sourceSchema }),+ H.getTableId({ name: joinTable, schema: joinSchema }),+ ]).then(([sourceTableId, joinTableId]) => {+ return H.createTestQuery({+ database: WRITABLE_DB_ID,+ stages: [+ {+ source: { type: "table", id: sourceTableId },+ joins: [+ {+ source: { type: "table", id: joinTableId },+ strategy: joinStrategy,+ conditions: [+ {+ operator: "=",+ left: { type: "column", name: "name" },+ right: { type: "column", name: "name" },+ },+ ],+ },+ ],+ },+ ],+ }).then((query) => {+ return H.createTransform({+ name,+ source: { type: "query", query },+ target: {+ type: "table",+ database: WRITABLE_DB_ID,+ name: targetTable,+ schema: TARGET_SCHEMA,+ },+ })+ .then(({ body: transform }) => {+ cy.request("POST", `/api/transform/${transform.id}/run`);+ H.waitForSucceededTransformRuns();+ return cy.wrap({ transformId: transform.id });+ })+ .then(({ transformId }) => {+ H.DataStudio.Transforms.visitInspect(transformId);+ });+ });+ });+}diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/transforms.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/transforms.cljindex bf462ce5d4c..e012a523ad0 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/transforms.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/transforms.clj@@ -2,19 +2,19 @@(:require[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u][metabase-enterprise.transforms-python.api :as transforms-python.api]- [metabase.transforms.api.transform :as api.transforms][metabase.transforms.core :as transforms]+ [metabase.transforms.schema :as transforms.schema][metabase.util.malli.registry :as mr]))-(mr/def ::transform-source ::api.transforms/transform-source)-(mr/def ::transform-target ::api.transforms/transform-target)+(mr/def ::transform-source ::transforms.schema/transform-source)+(mr/def ::transform-target ::transforms.schema/transform-target)(defn get-transforms"Get a list of all known transforms."[_args](try{:structured_output- (->> (api.transforms/get-transforms)+ (->> (transforms/get-transforms)(into [] (comp (filter #(or (transforms/python-transform? %)(transforms/native-query-transform? %)))(filter :source_readable)@@ -27,7 +27,7 @@[{:keys [transform-id]}](try{:structured_output- (api.transforms/get-transform transform-id)}+ (transforms/get-transform transform-id)}(catch Exception e(metabot-v3.tools.u/handle-agent-error e))))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/events.clj b/enterprise/backend/src/metabase_enterprise/workspaces/events.cljindex 679ee989e68..6ae60e5cb7c 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/events.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/events.clj@@ -18,7 +18,7 @@(when (pos? (:next.jdbc/update-count result 0))(log/debug "Incremented graph_version for all workspaces due to global transform change"))))-;; These events are published by metabase.transforms.api when transforms are+;; These events are published by metabase.transforms-rest.api when transforms are;; created, updated, or deleted. We subscribe to increment graph versions.(derive ::workspace-staleness :metabase/event)diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/merge.clj b/enterprise/backend/src/metabase_enterprise/workspaces/merge.cljindex 288a806daa7..4630678bcf2 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/merge.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/merge.clj@@ -5,7 +5,7 @@[metabase-enterprise.workspaces.models.workspace-merge][metabase-enterprise.workspaces.models.workspace-merge-transform][metabase-enterprise.workspaces.types :as ws.types]- [metabase.transforms.api.transform :as transforms.api]+ [metabase.transforms.core :as transforms.api][metabase.util.malli :as mu][toucan2.core :as t2])(:import (java.sql Connection)))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex dc182324431..6b8e3cc8687 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -21,7 +21,7 @@[metabase.lib.metadata :as lib.metadata][metabase.query-processor.compile :as qp.compile][metabase.test :as mt]- [metabase.transforms.api.transform :as transforms.api]+ [metabase.transforms.core :as transforms.core][metabase.transforms.execute :as transforms.execute][metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]@@ -315,13 +315,13 @@(t2/select-one-fn :name [:model/WorkspaceTransform :name] :workspace_id ws-id :ref_id ws-x-2-id)))))(testing "No updates are propagated back to core app on merge failure"- (let [update-transform! transforms.api/update-transform!]- (with-redefs [transforms.api/update-transform! (let [call-count (atom 0)]- (fn [& args]- (when (> @call-count 0)- (throw (Exception. "boom")))- (swap! call-count inc)- (apply update-transform! args)))]+ (let [update-transform! transforms.core/update-transform!]+ (with-redefs [transforms.core/update-transform! (let [call-count (atom 0)]+ (fn [& args]+ (when (> @call-count 0)+ (throw (Exception. "boom")))+ (swap! call-count inc)+ (apply update-transform! args)))](testing "API response: empty merged, single error"(let [resp (mt/user-http-request :crowberto :post 200 (ws-url ws-id "/merge"))](is (empty? (get-in resp [:merged :transforms])))@@ -574,8 +574,8 @@_ (mt/user-http-request :crowberto :post 204(ws-url ws-id "/transform" ws-x-1-id "/archive"))](testing "Failure on merge"- (with-redefs [transforms.api/delete-transform! (fn [& _args]- (throw (Exception. "boom")))]+ (with-redefs [transforms.core/delete-transform! (fn [& _args]+ (throw (Exception. "boom")))](let [resp (mt/user-http-request :crowberto :post 500(ws-url ws-id "/transform" ws-x-1-id "/merge"))](testing "Response"diff --git a/frontend/src/metabase-types/analytics/event.ts b/frontend/src/metabase-types/analytics/event.tsindex 0c890f61c03..5031ff108f0 100644--- a/frontend/src/metabase-types/analytics/event.ts+++ b/frontend/src/metabase-types/analytics/event.ts@@ -308,6 +308,38 @@ export type TransformRunTagsUpdated = ValidateEvent<{target_id: number;}>;+export type TransformInspectLensLoadedEvent = ValidateEvent<{+ event: "transform_inspect_lens_loaded";+ target_id: TransformId;+ event_detail: string;+ duration_ms: number;+}>;++export type TransformInspectDrillLensClickedEvent = ValidateEvent<{+ event: "transform_inspect_drill_lens_clicked";+ target_id: TransformId;+ event_detail: string;+ triggered_from: "card_drills" | "join_analysis";+}>;++export type TransformInspectAlertClickedEvent = ValidateEvent<{+ event: "transform_inspect_alert_clicked";+ target_id: TransformId;+ event_detail: string;+}>;++export type TransformInspectDrillLensClosedEvent = ValidateEvent<{+ event: "transform_inspect_drill_lens_closed";+ target_id: TransformId;+ event_detail: string;+}>;++export type TransformInspectEvent =+ | TransformInspectLensLoadedEvent+ | TransformInspectDrillLensClickedEvent+ | TransformInspectAlertClickedEvent+ | TransformInspectDrillLensClosedEvent;+export type DocumentCreatedEvent = ValidateEvent<{event: "document_created";target_id: number;@@ -681,6 +713,7 @@ export type SimpleEvent =| TransformCreatedEvent| TransformCreateEvent| TransformRunTagsUpdated+ | TransformInspectEvent| DocumentAddCardEvent| DocumentAddSmartLinkEvent| DocumentAddSupportingTextEventdiff --git a/frontend/src/metabase-types/api/card.ts b/frontend/src/metabase-types/api/card.tsindex 1be0f5a6caa..bcbc7dee3a4 100644--- a/frontend/src/metabase-types/api/card.ts+++ b/frontend/src/metabase-types/api/card.ts@@ -113,6 +113,7 @@ export interface UnsavedCard<Q extends DatasetQuery = DatasetQuery> {// Not part of the card API contract, a field used by query builder for showing lineageoriginal_card_id?: number;+ displayIsLocked?: boolean;}export type LineSize = "S" | "M" | "L";diff --git a/frontend/src/metabase-types/api/collection.ts b/frontend/src/metabase-types/api/collection.tsindex 3e71e2df5f4..d67d89ebb11 100644--- a/frontend/src/metabase-types/api/collection.ts+++ b/frontend/src/metabase-types/api/collection.ts@@ -15,7 +15,6 @@ import type { DatabaseId } from "./database";import type { SortingOptions } from "./sorting";import type { TableId } from "./table";import type { UserId, UserInfo } from "./user";-export type CollectionNamespace =| null| "snippets"diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.tsindex e6a34858c57..69282f86dad 100644--- a/frontend/src/metabase-types/api/mocks/transform.ts+++ b/frontend/src/metabase-types/api/mocks/transform.ts@@ -1,5 +1,7 @@import type {DatabaseId,+ InspectorCard,+ InspectorSource,ListTransformRunsResponse,PythonTransformTableAliases,Transform,@@ -12,7 +14,10 @@ import type {UpdateTransformRequest,} from "metabase-types/api";-import { createMockStructuredDatasetQuery } from "./query";+import {+ createMockNativeDatasetQuery,+ createMockStructuredDatasetQuery,+} from "./query";export function createMockTransformOwner(opts?: Partial<TransformOwner>,@@ -168,3 +173,30 @@ export function createMockUpdateTransformRequest(...opts,};}++export function createMockTransformInspectSource(+ opts?: Partial<InspectorSource>,+): InspectorSource {+ return {+ table_name: "Table",+ column_count: 0,+ fields: [],+ ...opts,+ };+}++export function createMockInspectorCard(+ opts?: Partial<InspectorCard>,+): InspectorCard {+ return {+ id: "card-1",+ title: "Card",+ display: "scalar",+ dataset_query: createMockNativeDatasetQuery(),+ metadata: {+ card_type: "table_count",+ dedup_key: ["table-1"],+ },+ ...opts,+ };+}diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex 8899a52d2b9..11390fde65b 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -2,17 +2,22 @@ import type { Collection, CollectionId } from "./collection";import type { DatabaseId } from "./database";import type { RowValue } from "./dataset";import type { PaginationRequest, PaginationResponse } from "./pagination";-import type { DatasetQuery } from "./query";+import type { DatasetQuery, JoinStrategy } from "./query";import type { ScheduleDisplayType } from "./settings";import type { SortDirection } from "./sorting";import type { ConcreteTableId, SchemaName, Table } from "./table";import type { UserId, UserInfo } from "./user";+import type { CardDisplayType } from "./visualization";export type TransformId = number;export type TransformTagId = number;export type TransformJobId = number;export type TransformRunId = number;+export type InspectorLensId = string;+export type InspectorCardId = string;+export type InspectorSectionId = string;+export type TransformOwner = Pick<UserInfo,"id" | "email" | "first_name" | "last_name"@@ -303,7 +308,205 @@ export type QueryComplexity = {reason: string;};+export type InspectorFieldStats = {+ distinct_count?: number;+ nil_percent?: number;+ // Numeric stats+ min?: number;+ max?: number;+ avg?: number;+ q1?: number;+ q3?: number;+ // Temporal stats+ earliest?: string;+ latest?: string;+};++export type InspectorField = {+ id?: number;+ name: string;+ display_name?: string;+ base_type?: string;+ semantic_type?: string;+ stats?: InspectorFieldStats;+};++export type InspectorSummaryTable = {+ table_name: string;+ row_count?: number;+ column_count: number;+};++export type InspectorSummary = {+ inputs: InspectorSummaryTable[];+ output: InspectorSummaryTable;+};++export type InspectorSource = {+ table_id?: ConcreteTableId;+ table_name: string;+ schema?: SchemaName;+ db_id?: DatabaseId;+ row_count?: number;+ column_count: number;+ fields: InspectorField[];+};++export type InspectorTarget = {+ table_id: ConcreteTableId;+ table_name: string;+ schema?: SchemaName;+ row_count?: number;+ column_count: number;+ fields: InspectorField[];+};++export type InspectorComparisonCard = {+ id: InspectorCardId;+ source: "input" | "output";+ table_name: string;+ field_name: string;+ title: string;+ display: CardDisplayType;+ dataset_query: DatasetQuery;+};++export type InspectorColumnComparison = {+ id: string;+ output_column: string;+ cards: InspectorComparisonCard[];+};++export type InspectorStatus = "not-run" | "ready";++export type InspectorVisitedFields = {+ all?: number[];+};++export type InspectorLensComplexityLevel = "fast" | "slow" | "very-slow";++export type InspectorLensComplexity = {+ level: InspectorLensComplexityLevel;+ score?: number;+};++export type InspectorLensMetadata = {+ id: InspectorLensId;+ display_name: string;+ description?: string;+ complexity?: InspectorLensComplexity;+};++export type InspectorDiscoveryResponse = {+ name: string;+ description?: string;+ status: InspectorStatus;+ sources: InspectorSource[];+ target?: InspectorTarget;+ visited_fields?: InspectorVisitedFields;+ available_lenses: InspectorLensMetadata[];+};++export type InspectorLayoutType = "flat" | "comparison";++export type InspectorSection = {+ id: InspectorSectionId;+ title: string;+ description?: string;+ layout?: InspectorLayoutType;+};++export type InspectorCardDisplayType = CardDisplayType | "hidden";++type InspectorCardMetadata = {+ card_type: "join_step" | "table_count" | "base_count";+ dedup_key: Array<string | number>;+ table_id?: ConcreteTableId;+ join_step?: number;+ join_alias?: string;+ join_strategy?: JoinStrategy;+ group_id?: string;+ group_role?: "input" | "output";+ group_order?: number;+ field_id?: number;+};++export type InspectorCard = {+ id: InspectorCardId;+ section_id?: InspectorSectionId;+ title: string;+ display: InspectorCardDisplayType;+ dataset_query: DatasetQuery;+ interestingness?: number;+ summary?: boolean;+ visualization_settings?: Record<string, unknown>;+ metadata: InspectorCardMetadata;+};++export type InspectorSummaryHighlight = {+ label: string;+ value?: string | number;+ card_id?: InspectorCardId;+};++export type InspectorLensSummary = {+ text?: string;+ highlights?: InspectorSummaryHighlight[];+};++// open schema with name key always present+export type InspectorTriggerCondition = {+ name: string;+ card_id: InspectorCardId;+ [key: string]: unknown;+};++export type InspectorAlertTrigger = {+ id: string;+ condition: InspectorTriggerCondition;+ severity: "info" | "warning" | "error";+ message: string;+};++export type LensParams = Record<string, string | number>;++export type InspectorDrillLensTrigger = {+ lens_id: InspectorLensId;+ condition: InspectorTriggerCondition;+ params?: LensParams;+ reason?: string;+};++export type InspectorLens = {+ id: InspectorLensId;+ display_name: string;+ summary?: InspectorLensSummary;+ sections: InspectorSection[];+ cards: InspectorCard[];+ drill_lenses?: InspectorLensMetadata[];+ alert_triggers?: InspectorAlertTrigger[];+ drill_lens_triggers?: InspectorDrillLensTrigger[];+};++export type GetInspectorLensRequest = {+ transformId: TransformId;+ lensId: InspectorLensId;+ lensParams?: LensParams;+};+export type MetabotSuggestedTransform = SuggestedTransform & {active: boolean;suggestionId: string; // internal unique identifier for marking active/inactive};++export type RunInspectorQueryRequest = {+ transformId: TransformId;+ lensId: InspectorLensId;+ query: DatasetQuery;+ lensParams?: unknown;+};++export type LensHandle = {+ id: InspectorLensId;+ params?: LensParams;+};diff --git a/frontend/src/metabase/api/tags/constants.ts b/frontend/src/metabase/api/tags/constants.tsindex 47f94e01d7f..7664906e792 100644--- a/frontend/src/metabase/api/tags/constants.ts+++ b/frontend/src/metabase/api/tags/constants.ts@@ -51,6 +51,7 @@ export const TAG_TYPES = ["transform-job","transform-job-via-tag","transform-run",+ "transform-inspector-lens","user","public-dashboard","embed-dashboard",diff --git a/frontend/src/metabase/api/tags/utils.ts b/frontend/src/metabase/api/tags/utils.tsindex 39f85644c38..ec932038ff7 100644--- a/frontend/src/metabase/api/tags/utils.ts+++ b/frontend/src/metabase/api/tags/utils.ts@@ -23,9 +23,11 @@ import type {FieldDimension,FieldId,ForeignKey,+ GetInspectorLensRequest,GetUserKeyValueRequest,Group,GroupListQuery,+ InspectorLens,LoggerPreset,Measure,ModelCacheRefreshStatus,@@ -59,6 +61,8 @@ import {import type { CloudMigration } from "metabase-types/api/cloud-migration";import type { Notification } from "metabase-types/api/notification";+import { getLensKey } from "../utils/transform-inspector-lens";+import type { TagType } from "./constants";import { TAG_TYPE_MAPPING } from "./constants";@@ -785,3 +789,15 @@ export function provideTransformJobListTags(): TagDescription<TagType>[] {return [listTag("transform-job"), ...jobs.flatMap(provideTransformJobTags)];}++export function provideInspectorLensTags(+ lens: InspectorLens | undefined,+ _error: unknown,+ { transformId, lensParams }: GetInspectorLensRequest,+): TagDescription<TagType>[] {+ const lensKey = getLensKey({ id: lens?.id ?? "", params: lensParams });+ return [+ idTag("transform", transformId),+ idTag("transform-inspector-lens", lensKey),+ ];+}diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.tsindex 3cee0c36d61..d7e3f760390 100644--- a/frontend/src/metabase/api/transform.ts+++ b/frontend/src/metabase/api/transform.ts@@ -2,12 +2,17 @@ import { isResourceNotFoundError } from "metabase/lib/errors";import type {CheckQueryComplexityRequest,CreateTransformRequest,+ Dataset,ExtractColumnsFromQueryRequest,ExtractColumnsFromQueryResponse,+ GetInspectorLensRequest,+ InspectorDiscoveryResponse,+ InspectorLens,ListTransformRunsRequest,ListTransformRunsResponse,ListTransformsRequest,QueryComplexity,+ RunInspectorQueryRequest,RunTransformResponse,Transform,TransformId,@@ -19,6 +24,7 @@ import {idTag,invalidateTags,listTag,+ provideInspectorLensTags,provideTransformListTags,provideTransformRunListTags,provideTransformTags,@@ -216,6 +222,31 @@ export const transformApi = Api.injectEndpoints({body: { query: queryString },}),}),+ getInspectorDiscovery: builder.query<+ InspectorDiscoveryResponse,+ TransformId+ >({+ query: (id) => ({+ method: "GET",+ url: `/api/transform/${id}/inspect`,+ }),+ providesTags: (_, _error, id) => [idTag("transform", id)],+ }),+ getInspectorLens: builder.query<InspectorLens, GetInspectorLensRequest>({+ query: ({ transformId, lensId, lensParams }) => ({+ method: "GET",+ url: `/api/transform/${transformId}/inspect/${lensId}`,+ params: lensParams,+ }),+ providesTags: provideInspectorLensTags,+ }),+ runInspectorQuery: builder.query<Dataset, RunInspectorQueryRequest>({+ query: ({ transformId, lensId, query, lensParams }) => ({+ method: "POST",+ url: `/api/transform/${transformId}/inspect/${lensId}/query`,+ body: { query, lens_params: lensParams },+ }),+ }),}),});@@ -224,6 +255,9 @@ export const {useListTransformRunsQuery,useGetTransformQuery,useLazyGetTransformQuery,+ useGetInspectorDiscoveryQuery,+ useGetInspectorLensQuery,+ useLazyGetInspectorLensQuery,useRunTransformMutation,useCancelCurrentTransformRunMutation,useCreateTransformMutation,@@ -232,4 +266,5 @@ export const {useDeleteTransformTargetMutation,useExtractColumnsFromQueryMutation,useLazyCheckQueryComplexityQuery,+ useRunInspectorQueryQuery,} = transformApi;diff --git a/frontend/src/metabase/api/utils/transform-inspector-lens.ts b/frontend/src/metabase/api/utils/transform-inspector-lens.tsnew file mode 100644index 00000000000..b516127503c--- /dev/null+++ b/frontend/src/metabase/api/utils/transform-inspector-lens.ts@@ -0,0 +1,12 @@+import type { LensHandle } from "metabase-types/api";++export const getLensKey = (handle: LensHandle): string => {+ if (!handle.params) {+ return handle.id;+ }+ const searchParams = new URLSearchParams(+ Object.entries(handle.params).map(([key, value]) => [key, String(value)]),+ );+ searchParams.sort();+ return `${handle.id}?${searchParams.toString()}`;+};diff --git a/frontend/src/metabase/data-studio/common/components/PaneHeader/PaneHeader.tsx b/frontend/src/metabase/data-studio/common/components/PaneHeader/PaneHeader.tsxindex 48764dc9242..bda613e1aed 100644--- a/frontend/src/metabase/data-studio/common/components/PaneHeader/PaneHeader.tsx+++ b/frontend/src/metabase/data-studio/common/components/PaneHeader/PaneHeader.tsx@@ -141,7 +141,11 @@ export function PaneHeaderTabs({ tabs, withBackground }: PaneHeaderTabsProps) {return (<Group gap="sm">- {tabs.map(({ label, to, icon, isSelected = to === pathname }) => {+ {tabs.map(({ label, to, icon, isSelected }) => {+ const selected =+ typeof isSelected === "function"+ ? isSelected(pathname)+ : (isSelected ?? to === pathname);return (<Buttonkey={label}@@ -149,8 +153,8 @@ export function PaneHeaderTabs({ tabs, withBackground }: PaneHeaderTabsProps) {to={to}size="sm"radius="xl"- c={isSelected ? "brand" : undefined}- bg={isSelected ? "background-selected" : backgroundColor}+ c={selected ? "brand" : undefined}+ bg={selected ? "background-selected" : backgroundColor}bd="none"leftSection={icon != null ? <FixedSizeIcon name={icon} /> : null}>diff --git a/frontend/src/metabase/data-studio/common/components/PaneHeader/types.ts b/frontend/src/metabase/data-studio/common/components/PaneHeader/types.tsindex 3eebf72c006..a598bfec814 100644--- a/frontend/src/metabase/data-studio/common/components/PaneHeader/types.ts+++ b/frontend/src/metabase/data-studio/common/components/PaneHeader/types.ts@@ -4,5 +4,5 @@ export type PaneHeaderTab = {label: string;to: string;icon?: IconName;- isSelected?: boolean;+ isSelected?: boolean | ((pathname: string) => boolean);};diff --git a/frontend/src/metabase/lib/urls/transforms.ts b/frontend/src/metabase/lib/urls/transforms.tsindex 22d429db4f2..3788e7b8ebc 100644--- a/frontend/src/metabase/lib/urls/transforms.ts+++ b/frontend/src/metabase/lib/urls/transforms.ts@@ -77,6 +77,10 @@ export function transformDependencies(transformId: TransformId) {return `${TRANSFORMS_ROOT_URL}/${transformId}/dependencies`;}+export function transformInspect(transformId: TransformId) {+ return `${TRANSFORMS_ROOT_URL}/${transformId}/inspect`;+}+export function transformJobList() {return JOBS_ROOT_URL;}diff --git a/frontend/src/metabase/transforms/analytics.ts b/frontend/src/metabase/transforms/analytics.tsindex 05dc916a554..c9435bf6851 100644--- a/frontend/src/metabase/transforms/analytics.ts+++ b/frontend/src/metabase/transforms/analytics.ts@@ -1,5 +1,12 @@import { trackSimpleEvent } from "metabase/lib/analytics";-import type { TransformId, TransformJobId } from "metabase-types/api";+import type {+ InspectorCardId,+ InspectorLensId,+ TransformId,+ TransformJobId,+} from "metabase-types/api";++import type { LensKey } from "./pages/TransformInspectPage/types";export function trackTransformTriggerManualRun({transformId,@@ -62,3 +69,82 @@ export function trackTransformRunTagsUpdated({result,});}++export function trackTransformInspectLensLoaded({+ transformId,+ lensKey,+ durationMs,+}: {+ transformId: TransformId;+ lensKey: LensKey;+ durationMs: number;+}) {+ trackSimpleEvent({+ event: "transform_inspect_lens_loaded",+ target_id: transformId,+ event_detail: lensKey,+ duration_ms: durationMs,+ });+}++export function trackTransformInspectDrillLensClicked({+ transformId,+ lensId,+ triggeredFrom,+}: {+ transformId: TransformId;+ lensId: InspectorLensId;+ triggeredFrom: "card_drills" | "join_analysis";+}) {+ trackSimpleEvent({+ event: "transform_inspect_drill_lens_clicked",+ target_id: transformId,+ event_detail: lensId,+ triggered_from: triggeredFrom,+ });+}++export function trackTransformInspectAlertClicked({+ transformId,+ cardId,+}: {+ transformId: TransformId;+ cardId: InspectorCardId;+}) {+ trackSimpleEvent({+ event: "transform_inspect_alert_clicked",+ target_id: transformId,+ event_detail: cardId,+ });+}++export function trackTransformInspectDrillLensClosed({+ transformId,+ lensId,+}: {+ transformId: TransformId;+ lensId: InspectorLensId;+}) {+ trackSimpleEvent({+ event: "transform_inspect_drill_lens_closed",+ target_id: transformId,+ event_detail: lensId,+ });+}++export function trackDependencyDiagnosticsEntitySelected({+ triggeredFrom,+ entityId,+ entityType,+}: {+ entityId: number;+ entityType: string;+ triggeredFrom: "broken" | "unreferenced";+}) {+ trackSimpleEvent({+ event: "dependency_diagnostics_entity_selected",+ triggered_from: triggeredFrom,+ target_id: entityId,+ event_detail: entityType,+ });+}diff --git a/frontend/src/metabase/transforms/components/TransformHeader/TransformTabs.tsx b/frontend/src/metabase/transforms/components/TransformHeader/TransformTabs.tsxindex 9d526be09bc..813b276371c 100644--- a/frontend/src/metabase/transforms/components/TransformHeader/TransformTabs.tsx+++ b/frontend/src/metabase/transforms/components/TransformHeader/TransformTabs.tsx@@ -18,6 +18,7 @@ export const TransformTabs = ({ transform }: TransformTabsProps) => {};function getTabs(id: TransformId): PaneHeaderTab[] {+ const inspectUrl = Urls.transformInspect(id);const tabs: PaneHeaderTab[] = [{label: t`Definition`,@@ -31,6 +32,11 @@ function getTabs(id: TransformId): PaneHeaderTab[] {label: t`Settings`,to: Urls.transformSettings(id),},+ {+ label: t`Inspect`,+ to: inspectUrl,+ isSelected: (pathname: string) => pathname.startsWith(inspectUrl),+ },];if (PLUGIN_DEPENDENCIES.isEnabled) {diff --git a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.tsnew file mode 100644index 00000000000..7dac5342e17--- /dev/null+++ b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts@@ -0,0 +1,35 @@+import { useState } from "react";++import { skipToken, useGetTransformQuery } from "metabase/api";+import type { Transform } from "metabase-types/api";++import { POLLING_INTERVAL } from "../constants";+import {+ isTransformCanceling,+ isTransformRunning,+ isTransformSyncing,+} from "../utils";++const isPollingNeeded = (transform?: Transform) =>+ transform != null &&+ (isTransformRunning(transform) ||+ isTransformCanceling(transform) ||+ isTransformSyncing(transform));++export const useTransformWithPolling = (transformId: number | undefined) => {+ const [isPolling, setIsPolling] = useState(false);++ const {+ data: transform,+ isLoading,+ error,+ } = useGetTransformQuery(transformId ?? skipToken, {+ pollingInterval: isPolling ? POLLING_INTERVAL : undefined,+ });++ if (isPolling !== isPollingNeeded(transform)) {+ setIsPolling(isPollingNeeded(transform));+ }++ return { transform, isLoading, error };+};diff --git a/frontend/src/metabase/transforms/lib/transforms-inspector.ts b/frontend/src/metabase/transforms/lib/transforms-inspector.tsnew file mode 100644index 00000000000..ab91cc9ebc1--- /dev/null+++ b/frontend/src/metabase/transforms/lib/transforms-inspector.ts@@ -0,0 +1,54 @@+import * as INSPECTOR from "cljs/metabase.transforms_inspector.js";+import type {+ InspectorAlertTrigger,+ InspectorCard,+ InspectorCardId,+ InspectorDrillLensTrigger,+ InspectorField,+ InspectorLens,+ InspectorLensId,+ InspectorVisitedFields,+ RowValues,+ VisualizationDisplay,+} from "metabase-types/api";++type TriggerResult = {+ alerts: InspectorAlertTrigger[];+ drill_lenses: InspectorDrillLensTrigger[];+};++export type CardStats = Record<string, unknown>;++export const interestingFields = (+ fields: InspectorField[],+ visitedFields?: InspectorVisitedFields,+ options?: { threshold?: number; limit?: number },+): Array<InspectorField & { interestingness: { score: number } }> =>+ INSPECTOR.interestingFields(+ fields,+ visitedFields ?? null,+ options?.threshold,+ options?.limit,+ );++export const evaluateTriggers = (+ lens: InspectorLens,+ cardsStats: Record<InspectorCardId, CardStats>,+): TriggerResult => INSPECTOR.evaluateTriggers(lens, cardsStats);++export const computeCardStats = (+ lensId: InspectorLensId,+ card: InspectorCard,+ rows: RowValues[] | undefined,+): CardStats | null => INSPECTOR.computeCardResult(lensId, card, rows ?? []);++export type DegeneracyResult = {+ degenerate: boolean;+ reason: string | null;+};++export const isDegenerate = (+ cardId: InspectorCardId,+ displayType: VisualizationDisplay,+ cardsStats: Record<InspectorCardId, CardStats>,+): DegeneracyResult => INSPECTOR.isDegenerate(cardId, displayType, cardsStats);diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/TransformInspectPage.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/TransformInspectPage.tsxnew file mode 100644index 00000000000..04dc857aa26--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/TransformInspectPage.tsx@@ -0,0 +1,56 @@+import type { Location } from "history";+import { t } from "ttag";++import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { PageContainer } from "metabase/data-studio/common/components/PageContainer";+import * as Urls from "metabase/lib/urls";+import { Alert, Center, Icon, Text } from "metabase/ui";++import { TransformHeader } from "../../components/TransformHeader";+import { useTransformWithPolling } from "../../hooks/use-transform-with-polling";+import { RunSection } from "../TransformRunPage/RunSection";++import { InspectorContent } from "./components/InspectorContent";+import type { RouteParams } from "./types";++type TransformInspectPageProps = {+ params: RouteParams;+ location: Location;+};++export const TransformInspectPage = ({+ params,+ location,+}: TransformInspectPageProps) => {+ const transformId = Urls.extractEntityId(params.transformId);++ const { transform, isLoading, error } = useTransformWithPolling(transformId);++ if (isLoading || error || !transform) {+ return (+ <Center h="100%">+ <LoadingAndErrorWrapper loading={isLoading} error={error} />+ </Center>+ );+ }++ return (+ <PageContainer data-testid="transform-inspect-content">+ <TransformHeader transform={transform} />+ {transform.last_run?.status !== "succeeded" ? (+ <>+ <Alert color="brand" icon={<Icon name="info" />}>+ <Text>{t`To inspect the transform you need to run it first.`}</Text>+ </Alert>+ <RunSection transform={transform} noTitle={true} />+ </>+ ) : (+ <InspectorContent+ transform={transform}+ params={params}+ location={location}+ />+ )}+ </PageContainer>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/CardAlerts.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/CardAlerts.tsxnew file mode 100644index 00000000000..d710284c179--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/CardAlerts.tsx@@ -0,0 +1,40 @@+import { match } from "ts-pattern";++import { Alert, Icon, Stack } from "metabase/ui";+import type { InspectorAlertTrigger } from "metabase-types/api";++type CardAlertsProps = {+ alerts: InspectorAlertTrigger[];+ fullWidth?: boolean;+};++export const CardAlerts = ({ alerts, fullWidth }: CardAlertsProps) => {+ if (alerts.length === 0) {+ return null;+ }++ return (+ <Stack gap="xs" w={fullWidth ? "100%" : undefined}>+ {alerts.map((alert) => (+ <Alert+ key={alert.id}+ icon={+ <Icon+ name={match(alert.severity)+ .with("error", () => "warning" as const)+ .with("warning", () => "warning" as const)+ .otherwise(() => "info" as const)}+ />+ }+ color={match(alert.severity)+ .with("error", () => "error" as const)+ .with("warning", () => "warning" as const)+ .otherwise(() => "brand" as const)}+ variant="light"+ >+ {alert.message}+ </Alert>+ ))}+ </Stack>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/CardDrills.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/CardDrills.tsxnew file mode 100644index 00000000000..07674d5ac77--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/CardDrills.tsx@@ -0,0 +1,45 @@+import { t } from "ttag";++import { trackTransformInspectDrillLensClicked } from "metabase/transforms/analytics";+import { Flex } from "metabase/ui";+import type { InspectorDrillLensTrigger } from "metabase-types/api";++import { DrillButton } from "./DrillButton";+import { useLensContentContext } from "./LensContent/LensContentContext";+import { getLensKey, toLensHandle } from "./LensNavigator/utils";++type CardDrillsProps = {+ drillLenses: InspectorDrillLensTrigger[];+};++export const CardDrills = ({ drillLenses }: CardDrillsProps) => {+ const { navigateToLens, transform } = useLensContentContext();++ if (drillLenses.length === 0) {+ return null;+ }++ return (+ <Flex gap="xs" wrap="wrap">+ {drillLenses.map((drillLens) => {+ const title = drillLens.reason ?? drillLens.lens_id;+ const lensHandle = toLensHandle(drillLens);+ return (+ <DrillButton+ key={getLensKey(lensHandle)}+ onClick={() => {+ trackTransformInspectDrillLensClicked({+ transformId: transform.id,+ lensId: drillLens.lens_id,+ triggeredFrom: "card_drills",+ });+ navigateToLens(lensHandle);+ }}+ >+ {t`Inspect`} {title}+ </DrillButton>+ );+ })}+ </Flex>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/DrillButton.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/DrillButton.tsxnew file mode 100644index 00000000000..42f416138b7--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/DrillButton.tsx@@ -0,0 +1,23 @@+import { Flex, Icon, Text, UnstyledButton } from "metabase/ui";++export function DrillButton({+ onClick,+ children,+}: {+ onClick: () => void;+ children: React.ReactNode;+}) {+ return (+ <UnstyledButton+ onClick={(e) => {+ e.stopPropagation();+ onClick();+ }}+ >+ <Flex gap="xs" align="center">+ <Icon name="zoom_in" c="brand" />+ <Text c="brand">{children}</Text>+ </Flex>+ </UnstyledButton>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/InspectorContent.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/InspectorContent.tsxnew file mode 100644index 00000000000..a1345a42314--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/InspectorContent.tsx@@ -0,0 +1,90 @@+import type { Location } from "history";+import { useCallback } from "react";++import { useGetInspectorDiscoveryQuery } from "metabase/api";+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { trackTransformInspectDrillLensClosed } from "metabase/transforms/analytics";+import { Center } from "metabase/ui";+import type { Transform } from "metabase-types/api";++import type { RouteParams } from "../types";++import { LensContent } from "./LensContent/LensContent";+import { LensNavigator, useLensNavigation } from "./LensNavigator";+import { getLensKey } from "./LensNavigator/utils";++type InspectorContentProps = {+ transform: Transform;+ params: RouteParams;+ location: Location;+};++export const InspectorContent = ({+ transform,+ params,+ location,+}: InspectorContentProps) => {+ const {+ data: discovery,+ isLoading: isLoadingDiscovery,+ error: discoveryError,+ } = useGetInspectorDiscoveryQuery(transform.id);++ const {+ tabs,+ activeTabKey,+ currentLensHandle,+ navigateToLens,+ closeTab,+ switchTab,+ markLensAsLoaded,+ updateTabTitle,+ onLensError,+ } = useLensNavigation(discovery?.available_lenses ?? [], params, location);++ const handleCloseTab = useCallback(+ (tabKey: string) => {+ trackTransformInspectDrillLensClosed({+ transformId: transform.id,+ lensId: tabKey,+ });+ closeTab(tabKey);+ },+ [transform.id, closeTab],+ );++ if (isLoadingDiscovery || discoveryError || !discovery) {+ return (+ <Center h="100%" style={{ flex: 1 }}>+ <LoadingAndErrorWrapper+ loading={isLoadingDiscovery}+ error={discoveryError}+ />+ </Center>+ );+ }++ if (!currentLensHandle) {+ return null;+ }++ return (+ <LensNavigator+ tabs={tabs}+ activeTabKey={activeTabKey}+ onSwitchTab={switchTab}+ onCloseTab={handleCloseTab}+ >+ <LensContent+ key={getLensKey(currentLensHandle)}+ transform={transform}+ lensHandle={currentLensHandle}+ discovery={discovery}+ navigateToLens={navigateToLens}+ onAllCardsLoaded={markLensAsLoaded}+ onTitleResolved={updateTabTitle}+ onError={onLensError}+ />+ </LensNavigator>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/LensContent.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/LensContent.tsxnew file mode 100644index 00000000000..8ffd5e81347--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/LensContent.tsx@@ -0,0 +1,139 @@+import { useEffect, useMemo } from "react";+import { match } from "ts-pattern";+import _ from "underscore";++import { useGetInspectorLensQuery } from "metabase/api";+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { Center, Stack } from "metabase/ui";+import type { InspectorDiscoveryResponse, Transform } from "metabase-types/api";++import { useCardLoadingTracker, useTriggerEvaluation } from "../../hooks";+import type { LensHandle } from "../../types";+import { getLensKey } from "../LensNavigator/utils";+import {+ DefaultLensSections,+ GenericSummarySections,+ JoinAnalysisSections,+} from "../LensSections";++import { LensContentProvider } from "./LensContentContext";+import { useLensLoadedTracking } from "./useLensLoadedTracking";++type LensContentProps = {+ transform: Transform;+ lensHandle: LensHandle;+ discovery: InspectorDiscoveryResponse;+ navigateToLens: (lensHandle: LensHandle) => void;+ onAllCardsLoaded: (lensKey: string) => void;+ onTitleResolved: (tabKey: string, title: string) => void;+ onError: (lensHandle: LensHandle, error: unknown) => void;+};++export const LensContent = ({+ transform,+ lensHandle,+ discovery,+ navigateToLens,+ onAllCardsLoaded,+ onTitleResolved,+ onError,+}: LensContentProps) => {+ const trackLensLoaded = useLensLoadedTracking(+ transform.id,+ getLensKey(lensHandle),+ );++ const {+ data: lens,+ isLoading,+ error,+ } = useGetInspectorLensQuery({+ transformId: transform.id,+ lensId: lensHandle.id,+ lensParams: lensHandle.params,+ });++ useEffect(() => {+ if (!lens) {+ return;+ }+ onTitleResolved(getLensKey(lensHandle), lens.display_name);+ }, [lens, lensHandle, onTitleResolved]);++ useEffect(() => {+ if (error && !isLoading) {+ onError(lensHandle, error);+ }+ }, [error, isLoading, lensHandle, onError]);++ const {+ alertsByCardId,+ drillLensesByCardId,+ pushNewStats,+ collectedCardStats,+ } = useTriggerEvaluation(lens);++ const { markCardLoaded, markCardStartedLoading } = useCardLoadingTracker(+ lens,+ () => {+ if (lens) {+ onAllCardsLoaded(lens.id);+ trackLensLoaded();+ }+ },+ );++ const cardsBySection = useMemo(+ () => _.groupBy(lens?.cards ?? [], (c) => c.section_id ?? "default"),+ [lens],+ );++ if (isLoading || error || !lens) {+ return (+ <Center h={200}>+ <LoadingAndErrorWrapper loading={isLoading} error={error} />+ </Center>+ );+ }++ return (+ <LensContentProvider+ transform={transform}+ lensHandle={lensHandle}+ lens={lens}+ alertsByCardId={alertsByCardId}+ drillLensesByCardId={drillLensesByCardId}+ collectedCardStats={collectedCardStats}+ navigateToLens={navigateToLens}+ onStatsReady={pushNewStats}+ onCardStartedLoading={markCardStartedLoading}+ onCardLoaded={markCardLoaded}+ >+ <Stack gap="xl">+ {match(lens.id)+ .with("generic-summary", () => (+ <GenericSummarySections+ sections={lens.sections}+ cardsBySection={cardsBySection}+ sources={discovery.sources}+ target={discovery.target}+ />+ ))+ .with("join-analysis", () => (+ <JoinAnalysisSections+ sections={lens.sections}+ cardsBySection={cardsBySection}+ />+ ))+ .otherwise(() => (+ <DefaultLensSections+ sections={lens.sections}+ cardsBySection={cardsBySection}+ sources={discovery.sources}+ visitedFields={discovery.visited_fields}+ />+ ))}+ </Stack>+ </LensContentProvider>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/LensContentContext.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/LensContentContext.tsxnew file mode 100644index 00000000000..7f287fcf925--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/LensContentContext.tsx@@ -0,0 +1,91 @@+import {+ type PropsWithChildren,+ createContext,+ useContext,+ useMemo,+} from "react";++import type { CardStats } from "metabase/transforms/lib/transforms-inspector";+import type {+ InspectorAlertTrigger,+ InspectorCardId,+ InspectorDrillLensTrigger,+ InspectorLens,+ Transform,+} from "metabase-types/api";++import type { LensHandle } from "../../types";++type LensContentContextValue = {+ transform: Transform;+ lens: InspectorLens;+ lensHandle: LensHandle;+ alertsByCardId: Record<InspectorCardId, InspectorAlertTrigger[]>;+ drillLensesByCardId: Record<InspectorCardId, InspectorDrillLensTrigger[]>;+ collectedCardStats: Record<InspectorCardId, CardStats>;+ navigateToLens: (lensHandle: LensHandle) => void;+ onStatsReady: (cardId: InspectorCardId, stats: CardStats | null) => void;+ onCardStartedLoading: (cardId: InspectorCardId) => void;+ onCardLoaded: (cardId: InspectorCardId) => void;+};++const LensEvaluationContext = createContext<+ LensContentContextValue | undefined+>(undefined);++export const useLensContentContext = (): LensContentContextValue => {+ const context = useContext(LensEvaluationContext);+ if (!context) {+ throw new Error(+ "useLensEvaluationContext must be used within a LensEvaluationProvider",+ );+ }+ return context;+};++export const LensContentProvider = ({+ transform,+ lens,+ lensHandle,+ alertsByCardId,+ drillLensesByCardId,+ collectedCardStats,+ onStatsReady,+ onCardStartedLoading,+ onCardLoaded,+ navigateToLens,+ children,+}: PropsWithChildren<LensContentContextValue>) => {+ const value = useMemo(+ () => ({+ transform,+ lens,+ lensHandle,+ alertsByCardId,+ drillLensesByCardId,+ collectedCardStats,+ onStatsReady,+ onCardStartedLoading,+ onCardLoaded,+ navigateToLens,+ }),+ [+ transform,+ lens,+ lensHandle,+ alertsByCardId,+ drillLensesByCardId,+ collectedCardStats,+ onStatsReady,+ onCardStartedLoading,+ onCardLoaded,+ navigateToLens,+ ],+ );++ return (+ <LensEvaluationContext.Provider value={value}>+ {children}+ </LensEvaluationContext.Provider>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/useLensLoadedTracking.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/useLensLoadedTracking.tsnew file mode 100644index 00000000000..159bf1b63c6--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensContent/useLensLoadedTracking.ts@@ -0,0 +1,28 @@+import { useCallback, useRef } from "react";++import { trackTransformInspectLensLoaded } from "metabase/transforms/analytics";+import type { TransformId } from "metabase-types/api";++import type { LensKey } from "../../types";++export const useLensLoadedTracking = (+ transformId: TransformId,+ lensKey: LensKey,+) => {+ const lensStartTimeRef = useRef<number>(Date.now());+ const hasCalledRef = useRef(false);++ const handleAllCardsLoaded = useCallback(() => {+ if (hasCalledRef.current) {+ return;+ }+ hasCalledRef.current = true;+ trackTransformInspectLensLoaded({+ transformId,+ lensKey,+ durationMs: Date.now() - lensStartTimeRef.current,+ });+ }, [transformId, lensKey]);++ return handleAllCardsLoaded;+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/LensNavigator.css b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/LensNavigator.cssnew file mode 100644index 00000000000..efe77aebcb8--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/LensNavigator.css@@ -0,0 +1,17 @@+.tabs {+ border-bottom: 1px solid var(--border-color);+}++.tabsList {+ --tab-border-color: none;++ gap: var(--mantine-spacing-sm);+}++.tab {+ max-width: 236px;++ &[data-orientation="horizontal"] {+ padding: 0.625rem 0.625rem 0.625rem var(--mantine-spacing-sm);+ }+}diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/LensNavigator.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/LensNavigator.tsxnew file mode 100644index 00000000000..47ecfea5968--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/LensNavigator.tsx@@ -0,0 +1,93 @@+import type { PropsWithChildren } from "react";+import { match } from "ts-pattern";+import { t } from "ttag";++import { Box, Group, Icon, Stack, Tabs, Text, Tooltip, rem } from "metabase/ui";++import styles from "./LensNavigator.css";+import type { LensTab } from "./types";++type LensNavigatorProps = {+ tabs: LensTab[];+ activeTabKey: string | undefined;+ onSwitchTab: (tabKey: string) => void;+ onCloseTab: (tabKey: string) => void;+};++export const LensNavigator = ({+ tabs,+ activeTabKey,+ onSwitchTab,+ onCloseTab,+ children,+}: PropsWithChildren<LensNavigatorProps>) => (+ <Stack+ gap="0"+ bd="1px solid border"+ bdrs="sm"+ bg="background-primary"+ pt={rem(6)}+ >+ <Tabs+ value={activeTabKey}+ onChange={(value) => value && onSwitchTab(value)}+ className={styles.tabs}+ >+ <Tabs.List px="lg" className={styles.tabsList}>+ {tabs.map((tab) => (+ <Tabs.Tab key={tab.key} value={tab.key} className={styles.tab}>+ <Group gap="xs" wrap="nowrap">+ <Text+ size="md"+ fw={700}+ c={tab.key === activeTabKey ? "text-brand" : "text-primary"}+ truncate+ miw={0}+ >+ {tab.title ?? t`Loading...`}+ </Text>+ {tab.isStatic &&+ !tab.isFullyLoaded &&+ match(tab.complexity?.level)+ .with("slow", "very-slow", (level) => (+ <Tooltip+ label={match(level)+ .with(+ "slow",+ () => t`This analysis may take longer to load`,+ )+ .with(+ "very-slow",+ () =>+ t`This analysis may take significantly longer to load`,+ )+ .exhaustive()}+ >+ <Icon name="clock" size={12} c="text-tertiary" />+ </Tooltip>+ ))+ .otherwise(() => null)}+ {!tab.isStatic && (+ <Box+ component="span"+ onClick={(e) => {+ e.stopPropagation();+ onCloseTab(tab.key);+ }}+ role="button"+ aria-label={t`Close tab`}+ ml={rem("2px")}+ lh={1}+ style={{ cursor: "pointer" }}+ >+ <Icon name="close" size={16} c="text-tertiary" />+ </Box>+ )}+ </Group>+ </Tabs.Tab>+ ))}+ </Tabs.List>+ </Tabs>+ <Box p="lg">{children}</Box>+ </Stack>+);diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/index.tsnew file mode 100644index 00000000000..0967cb340b5--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/index.ts@@ -0,0 +1,2 @@+export { LensNavigator } from "./LensNavigator";+export { useLensNavigation } from "./useLensNavigation";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/types.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/types.tsnew file mode 100644index 00000000000..30c0323578b--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/types.ts@@ -0,0 +1,22 @@+import type { InspectorLensComplexity } from "metabase-types/api";++import type { LensHandle } from "../../types";++export type StaticLensTab = {+ key: string;+ title: string;+ isStatic: true;+ lensHandle: LensHandle;+ isFullyLoaded?: boolean;+ complexity?: InspectorLensComplexity;+};++export type DynamicLensTab = {+ key: string;+ title?: string;+ isStatic: false;+ lensHandle: LensHandle;+ isFullyLoaded?: boolean;+};++export type LensTab = StaticLensTab | DynamicLensTab;diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/useLensNavigation.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/useLensNavigation.tsnew file mode 100644index 00000000000..cfe9dc3ca6a--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/useLensNavigation.ts@@ -0,0 +1,188 @@+import type { Location } from "history";+import { useCallback, useEffect, useMemo, useState } from "react";+import { push, replace } from "react-router-redux";+import { t } from "ttag";++import { useDispatch } from "metabase/lib/redux";+import type { InspectorLensMetadata } from "metabase-types/api";++import type { LensHandle, RouteParams } from "../../types";++import type { LensTab } from "./types";+import {+ createDynamicTab,+ createStaticTab,+ getLensKey,+ parseLocationParams,+} from "./utils";++type UseLensNavigationResult = {+ tabs: LensTab[];+ activeTabKey: string | undefined;+ currentLensHandle: LensHandle | undefined;+ navigateToLens: (lensHandle: LensHandle, isReplace?: boolean) => void;+ closeTab: (tabKey: string) => void;+ switchTab: (tabKey: string) => void;+ markLensAsLoaded: (lensKey: string) => void;+ updateTabTitle: (tabKey: string, title: string) => void;+ onLensError: (lensHandle: LensHandle, error: unknown) => void;+};++export const useLensNavigation = (+ availableLenses: InspectorLensMetadata[],+ params: RouteParams,+ location: Location,+): UseLensNavigationResult => {+ const dispatch = useDispatch();+ const [loadedLenses, setLoadedLenses] = useState<Set<string>>(new Set());++ const [dynamicTabs, setDynamicTabs] = useState<LensTab[]>([]);+ const staticTabs = useMemo(+ () => availableLenses.map(createStaticTab),+ [availableLenses],+ );++ const tabs = useMemo(+ () =>+ [...staticTabs, ...dynamicTabs].map((tab) => ({+ ...tab,+ isFullyLoaded: loadedLenses.has(tab.key),+ })),+ [staticTabs, dynamicTabs, loadedLenses],+ );++ const currentLensHandle = useMemo<LensHandle | undefined>(() => {+ if (!params.lensId) {+ return undefined;+ }+ return { id: params.lensId, params: parseLocationParams(location.search) };+ }, [params.lensId, location.search]);++ const activeTabKey = currentLensHandle+ ? getLensKey(currentLensHandle)+ : undefined;++ const activeTab = useMemo(+ () => tabs.find(({ key }) => key === activeTabKey),+ [activeTabKey, tabs],+ );++ const basePath = useMemo(() => {+ const pathname = location.pathname;+ if (currentLensHandle) {+ const lastSlash = pathname.lastIndexOf("/");+ return pathname.slice(0, lastSlash);+ }+ return pathname;+ }, [location.pathname, currentLensHandle]);++ const addDynamicTab = (handle: LensHandle) =>+ setDynamicTabs((prev) => {+ const key = getLensKey(handle);+ if (prev.some((tab) => tab.key === key)) {+ return prev;+ }+ return [...prev, createDynamicTab(handle)];+ });++ const removeDynamicTab = (handle: LensHandle) =>+ setDynamicTabs((prev) => {+ const key = getLensKey(handle);+ return prev.filter((tab) => tab.key !== key);+ });++ const navigate = useCallback(+ (handle: LensHandle, isReplace: boolean = false) => {+ const action = isReplace ? replace : push;+ const path = `${basePath}/${handle.id}`;+ dispatch(action({ pathname: path, query: handle.params }));+ },+ [basePath, dispatch],+ );++ const navigateToLens = useCallback(+ (handle: LensHandle, isReplace = false) => {+ const key = getLensKey(handle);+ const alreadyExists = tabs.some((tab) => tab.key === key);+ if (!alreadyExists) {+ addDynamicTab(handle);+ }+ navigate(handle, isReplace);+ },+ [tabs, navigate],+ );++ useEffect(() => {+ if (!activeTabKey && staticTabs.length > 0) {+ navigateToLens(staticTabs[0].lensHandle, true);+ }+ }, [activeTabKey, staticTabs, navigateToLens]);++ useEffect(() => {+ if (currentLensHandle && tabs.length > 0 && !activeTab) {+ navigateToLens(currentLensHandle, true);+ }+ }, [activeTab, currentLensHandle, tabs, navigateToLens]);++ const closeTab = useCallback(+ (tabKey: string) => {+ const tabIndex = tabs.findIndex(({ key }) => key === tabKey);+ const tab = tabs[tabIndex];+ if (!tab || tab.isStatic) {+ return;+ }+ removeDynamicTab(tab.lensHandle);+ if (activeTabKey === tabKey) {+ const remainingTabs = tabs.filter(({ key }) => key !== tabKey);+ const newActiveIndex = Math.min(tabIndex, remainingTabs.length - 1);+ const newActiveTab = remainingTabs[newActiveIndex];+ if (newActiveTab) {+ navigate(newActiveTab.lensHandle);+ }+ }+ },+ [tabs, activeTabKey, navigate],+ );++ const markLensAsLoaded = useCallback((lensKey: string) => {+ setLoadedLenses((prev) =>+ prev.has(lensKey) ? prev : new Set([...prev, lensKey]),+ );+ }, []);++ const switchTab = useCallback(+ (tabKey: string) => {+ const tab = tabs.find(({ key }) => key === tabKey);+ if (tab) {+ navigate(tab.lensHandle);+ }+ },+ [navigate, tabs],+ );++ const updateTabTitle = useCallback((tabKey: string, title: string) => {+ setDynamicTabs((prev) =>+ prev.map((tab) => (tab.key === tabKey ? { ...tab, title } : tab)),+ );+ }, []);++ const onLensError = useCallback(+ (lensHandle: LensHandle) => {+ const tabKey = getLensKey(lensHandle);+ updateTabTitle(tabKey, t`Error`);+ },+ [updateTabTitle],+ );++ return {+ tabs,+ activeTabKey,+ currentLensHandle,+ navigateToLens,+ closeTab,+ switchTab,+ markLensAsLoaded,+ updateTabTitle,+ onLensError,+ };+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/utils.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/utils.tsnew file mode 100644index 00000000000..a4ca82d45fe--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensNavigator/utils.ts@@ -0,0 +1,48 @@+import { getLensKey } from "metabase/api/utils/transform-inspector-lens";+import type {+ InspectorDrillLensTrigger,+ InspectorLensMetadata,+ LensParams,+} from "metabase-types/api";++import type { LensHandle } from "../../types";++import type { DynamicLensTab, StaticLensTab } from "./types";++export { getLensKey };++export const toLensHandle = (+ source: InspectorLensMetadata | InspectorDrillLensTrigger,+): LensHandle => {+ if ("lens_id" in source) {+ return {+ id: source.lens_id,+ params: source.params,+ };+ }+ return { id: source.id };+};++export const parseLocationParams = (search: string): LensParams | undefined => {+ const params = new URLSearchParams(search);+ return params.size > 0 ? Object.fromEntries(params) : undefined;+};++export const createStaticTab = (+ lensMetadata: InspectorLensMetadata,+): StaticLensTab => {+ const ref = toLensHandle(lensMetadata);+ return {+ key: getLensKey(ref),+ title: lensMetadata.display_name,+ isStatic: true,+ lensHandle: ref,+ complexity: lensMetadata.complexity,+ };+};++export const createDynamicTab = (handle: LensHandle): DynamicLensTab => ({+ key: getLensKey(handle),+ isStatic: false,+ lensHandle: handle,+});diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/DefaultLensSections.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/DefaultLensSections.tsxnew file mode 100644index 00000000000..80de9c4c5fa--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/DefaultLensSections.tsx@@ -0,0 +1,57 @@+import { match } from "ts-pattern";++import { Box, Stack, Title } from "metabase/ui";+import type {+ InspectorCard,+ InspectorSection,+ InspectorSectionId,+ InspectorSource,+ InspectorVisitedFields,+} from "metabase-types/api";++import { ComparisonLayout } from "./components/ComparisonLayout/ComparisonLayout";+import { FlatLayout } from "./components/FlatLayout";++type DefaultLensSectionsProps = {+ sections: InspectorSection[];+ cardsBySection: Record<InspectorSectionId, InspectorCard[]>;+ sources: InspectorSource[];+ visitedFields?: InspectorVisitedFields;+};++export const DefaultLensSections = ({+ sections,+ cardsBySection,+ sources,+ visitedFields,+}: DefaultLensSectionsProps) => {+ return (+ <Box>+ {sections.map((section) => {+ const cards = cardsBySection[section.id] ?? [];++ if (cards.length === 0) {+ return null;+ }++ const content = match(section.layout ?? "flat")+ .with("comparison", () => (+ <ComparisonLayout+ cards={cards}+ sources={sources}+ visitedFields={visitedFields}+ />+ ))+ .with("flat", () => <FlatLayout cards={cards} />)+ .exhaustive();++ return (+ <Stack key={section.id} gap="md">+ <Title order={3}>{section.title}</Title>+ {content}+ </Stack>+ );+ })}+ </Box>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/DefaultLensSections.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/DefaultLensSections.unit.spec.tsxnew file mode 100644index 00000000000..28d3faa8940--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/DefaultLensSections.unit.spec.tsx@@ -0,0 +1,149 @@+import { setupRunInspectorQueryEndpoint } from "__support__/server-mocks/transform";+import { renderWithProviders, screen } from "__support__/ui";+import type {+ InspectorAlertTrigger,+ InspectorCard,+ InspectorDrillLensTrigger,+ InspectorSection,+} from "metabase-types/api";+import {+ createMockColumn,+ createMockDataset,+ createMockInspectorCard,+ createMockTransform,+ createMockTransformInspectSource,+} from "metabase-types/api/mocks";++import { LensContentProvider } from "../../LensContent/LensContentContext";++import { DefaultLensSections } from "./DefaultLensSections";++jest.mock("metabase/transforms/lib/transforms-inspector", () => ({+ ...jest.requireActual("metabase/transforms/lib/transforms-inspector"),+ computeCardStats: () => null,+}));++const scalarCard1 = createMockInspectorCard({+ id: "card-1",+ title: "Row Count",+ display: "scalar",+});++const scalarCard2 = createMockInspectorCard({+ id: "card-2",+ title: "Column Count",+ display: "scalar",+});++const mockAlerts: InspectorAlertTrigger[] = [+ {+ id: "alert-1",+ severity: "warning",+ message: "Row count dropped significantly",+ condition: { name: "row_count_drop", card_id: "card-1" },+ },+ {+ id: "alert-2",+ severity: "error",+ message: "Missing values detected",+ condition: { name: "missing_values", card_id: "card-2" },+ },+];++const mockDrillLenses: InspectorDrillLensTrigger[] = [+ {+ lens_id: "drill-lens-1",+ reason: "missing values",+ condition: { name: "missing_values_drill", card_id: "card-1" },+ },+ {+ lens_id: "drill-lens-2",+ reason: "distribution anomaly",+ condition: { name: "distribution_drill", card_id: "card-2" },+ },+];++const defaultSections: InspectorSection[] = [+ { id: "section-1", title: "Overview" },+];++function setup({+ sections = defaultSections,+ cardsBySection = { "section-1": [scalarCard1, scalarCard2] },+ alertsByCardId = {} satisfies Record<string, InspectorAlertTrigger[]>,+ drillLensesByCardId = {} satisfies Record<+ string,+ InspectorDrillLensTrigger[]+ >,+}: {+ sections?: InspectorSection[];+ cardsBySection?: Record<string, InspectorCard[]>;+ alertsByCardId?: Record<string, InspectorAlertTrigger[]>;+ drillLensesByCardId?: Record<string, InspectorDrillLensTrigger[]>;+} = {}) {+ const lensId = "lens-1";+ setupRunInspectorQueryEndpoint(+ 1,+ lensId,+ createMockDataset({+ data: {+ rows: [[42]],+ cols: [createMockColumn({ name: "count" })],+ },+ }),+ );++ renderWithProviders(+ <LensContentProvider+ transform={createMockTransform()}+ lens={{+ id: lensId,+ display_name: "Test Lens",+ sections: [],+ cards: [],+ }}+ lensHandle={{ id: lensId }}+ alertsByCardId={alertsByCardId}+ drillLensesByCardId={drillLensesByCardId}+ collectedCardStats={{}}+ navigateToLens={jest.fn()}+ onStatsReady={jest.fn()}+ onCardStartedLoading={jest.fn()}+ onCardLoaded={jest.fn()}+ >+ <DefaultLensSections+ sections={sections}+ cardsBySection={cardsBySection}+ sources={[createMockTransformInspectSource()]}+ />+ </LensContentProvider>,+ );+}++describe("DefaultLensSections", () => {+ it("renders alerts for cards that have triggered alerts", () => {+ setup({+ alertsByCardId: {+ "card-1": [mockAlerts[0]],+ "card-2": [mockAlerts[1]],+ },+ });++ expect(+ screen.getByText("Row count dropped significantly"),+ ).toBeInTheDocument();+ expect(screen.getByText("Missing values detected")).toBeInTheDocument();+ });++ it("renders drill buttons for cards that have triggered drill lenses", () => {+ setup({+ drillLensesByCardId: {+ "card-1": [mockDrillLenses[0]],+ "card-2": [mockDrillLenses[1]],+ },+ });++ expect(screen.getByText(/missing values/)).toBeInTheDocument();+ expect(screen.getByText(/distribution anomaly/)).toBeInTheDocument();+ });+});diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/ComparisonLayout.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/ComparisonLayout.tsxnew file mode 100644index 00000000000..9d0b08d6808--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/ComparisonLayout.tsx@@ -0,0 +1,52 @@+import { useMemo } from "react";+import { match } from "ts-pattern";++import { SimpleGrid, Stack } from "metabase/ui";+import type {+ InspectorCard,+ InspectorSource,+ InspectorVisitedFields,+} from "metabase-types/api";++import { ScalarCard } from "../ScalarCard";+import { VisualizationCard } from "../VisualizationCard";++import { groupCardsBySource, sortGroupsByScore } from "./utils";++type ComparisonLayoutProps = {+ cards: InspectorCard[];+ sources: InspectorSource[];+ visitedFields?: InspectorVisitedFields;+};++export const ComparisonLayout = ({+ cards,+ sources,+ visitedFields,+}: ComparisonLayoutProps) => {+ const groups = useMemo(+ () => groupCardsBySource(cards, sources),+ [cards, sources],+ );++ const sortedGroups = useMemo(+ () => sortGroupsByScore(groups, sources, visitedFields),+ [groups, sources, visitedFields],+ );++ const renderCard = (card: InspectorCard) =>+ match(card.display)+ .with("scalar", () => <ScalarCard card={card} key={card.id} />)+ .otherwise(() => <VisualizationCard card={card} key={card.id} />);++ return (+ <Stack gap="lg">+ {sortedGroups.map((group) => (+ <SimpleGrid key={group.groupId} cols={2} spacing="md">+ <Stack gap="sm">{group.inputCards.map(renderCard)}</Stack>+ <Stack gap="sm">{group.outputCards.map(renderCard)}</Stack>+ </SimpleGrid>+ ))}+ </Stack>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/ComparisonLayout.unit.spec.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/ComparisonLayout.unit.spec.tsnew file mode 100644index 00000000000..c0ca7464af6--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/ComparisonLayout.unit.spec.ts@@ -0,0 +1,98 @@+import type { InspectorField } from "metabase-types/api";+import {+ createMockInspectorCard,+ createMockTransformInspectSource,+} from "metabase-types/api/mocks";++import { type CardGroup, sortGroupsByScore } from "./utils";++jest.mock("metabase/transforms/lib/transforms-inspector", () => ({+ interestingFields: (+ fields: InspectorField[],+ _visitedFields: unknown,+ // we use field ID as the score for testing purposes+ ): Array<InspectorField & { interestingness: { score: number } }> =>+ fields.map((f) => ({ ...f, interestingness: { score: f.id ?? 0 } })),+}));++const makeSource = (+ tableName: string,+ fields: Array<{ name: string; id: number }>,+) =>+ createMockTransformInspectSource({+ table_name: tableName,+ column_count: fields.length,+ fields: fields.map((f) => ({ name: f.name, id: f.id })),+ });++const makeGroup = (+ groupId: string,+ inputFieldIds: number[],+ outputTitles: string[] = [],+): CardGroup => ({+ groupId,+ inputCards: inputFieldIds.map((fieldId, index) =>+ createMockInspectorCard({+ id: `${groupId}-in-${index}`,+ title: `Card ${index}`,+ metadata: { card_type: "table_count", dedup_key: [], field_id: fieldId },+ }),+ ),+ outputCards: outputTitles.map((title, index) =>+ createMockInspectorCard({ id: `${groupId}-out-${index}`, title }),+ ),+});++describe("sortGroupsByScore", () => {+ it("sorts groups by highest field score descending", () => {+ const sources = [+ makeSource("orders", [+ { name: "Revenue", id: 10 },+ { name: "Quantity", id: 5 },+ ]),+ ];+ const groups = [makeGroup("low", [5]), makeGroup("high", [10])];++ const result = sortGroupsByScore(groups, sources);++ expect(result.map((g) => g.groupId)).toEqual(["high", "low"]);+ expect(result[0].topScore).toBe(10);+ expect(result[1].topScore).toBe(5);+ });++ it("uses the maximum score among input cards in a group", () => {+ const sources = [+ makeSource("orders", [+ { name: "Revenue", id: 10 },+ { name: "Quantity", id: 2 },+ { name: "Discount", id: 8 },+ ]),+ ];+ const groups = [makeGroup("mixed", [2, 10, 8])];++ const result = sortGroupsByScore(groups, sources);++ expect(result[0].topScore).toBe(10);+ });++ it("assigns score 0 for cards whose field_id doesn't match any field", () => {+ const sources = [makeSource("orders", [{ name: "Revenue", id: 10 }])];+ const groups = [makeGroup("unknown", [999])];++ const result = sortGroupsByScore(groups, sources);++ expect(result[0].topScore).toBe(0);+ });++ it("preserves group data (inputCards, outputCards) in results", () => {+ const sources = [makeSource("orders", [{ name: "Revenue", id: 5 }])];+ const groups = [makeGroup("g1", [5], ["Output"])];++ const result = sortGroupsByScore(groups, sources);++ expect(result[0].inputCards).toHaveLength(1);+ expect(result[0].outputCards).toHaveLength(1);+ expect(result[0].inputCards[0].title).toBe("Card 0");+ expect(result[0].outputCards[0].title).toBe("Output");+ });+});diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/utils.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/utils.tsnew file mode 100644index 00000000000..1005983733a--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ComparisonLayout/utils.ts@@ -0,0 +1,99 @@+import { interestingFields } from "metabase/transforms/lib/transforms-inspector";+import type {+ InspectorCard,+ InspectorSource,+ InspectorVisitedFields,+} from "metabase-types/api";++export type CardGroup = {+ groupId: string;+ inputCards: InspectorCard[];+ outputCards: InspectorCard[];+};++type CardGroupWithScore = CardGroup & {+ topScore: number;+};+export function sortGroupsByScore(+ groups: CardGroup[],+ sources: InspectorSource[],+ visitedFields?: InspectorVisitedFields,+): CardGroupWithScore[] {+ const allFields = sources.flatMap((s) => s.fields ?? []);+ const scoredFields = interestingFields(allFields, visitedFields);+ const groupsWithScore = groups.map((g) => {+ const topScore = g.inputCards.reduce((maxScore, card) => {+ const scoredField = scoredFields.find(+ (f) => f.id === card.metadata.field_id,+ );+ const score = scoredField?.interestingness.score ?? 0;+ return Math.max(maxScore, score);+ }, 0);+ return { ...g, topScore };+ });++ return groupsWithScore.sort((a, b) => b.topScore - a.topScore);+}++const buildTableOrdersInSources = (sources: InspectorSource[]) => {+ const map = new Map<number | undefined, number>();+ for (let i = 0; i < sources.length; i++) {+ map.set(sources[i].table_id, i);+ }+ return map;+};++const groupCardsByGroupId = (+ cards: InspectorCard[],+): Map<string, CardGroup> => {+ const groupMap = new Map<string, CardGroup>();+ for (const card of cards) {+ const metadata = card.metadata ?? {};+ const groupId = String(metadata.group_id ?? "default");+ const groupRole = metadata.group_role;+ const group = groupMap.get(groupId) ?? {+ groupId,+ inputCards: [],+ outputCards: [],+ };+ if (!groupMap.has(groupId)) {+ groupMap.set(groupId, group);+ }+ if (groupRole === "output") {+ group.outputCards.push(card);+ } else {+ group.inputCards.push(card);+ }+ }+ return groupMap;+};++export const groupCardsBySource = (+ cards: InspectorCard[],+ sources: InspectorSource[],+): CardGroup[] => {+ const cardsByGroupId = groupCardsByGroupId(cards);+ const tableOrdersInSources = buildTableOrdersInSources(sources);+ for (const group of cardsByGroupId.values()) {+ group.inputCards.sort((a, b) => {+ const orderA = getTableOrderInSources(a, tableOrdersInSources);+ const orderB = getTableOrderInSources(b, tableOrdersInSources);+ if (orderA !== orderB) {+ return orderA - orderB;+ }+ return getGroupOrder(a) - getGroupOrder(b);+ });++ group.outputCards.sort((a, b) => getGroupOrder(a) - getGroupOrder(b));+ }++ return Array.from(cardsByGroupId.values());+};++const getGroupOrder = (card: InspectorCard) => card.metadata?.group_order ?? 0;++const getTableOrderInSources = (+ card: InspectorCard,+ tableOrdersInSources: Map<number | undefined, number>,+): number =>+ tableOrdersInSources.get(card.metadata.table_id) ?? Number.POSITIVE_INFINITY;diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/FlatLayout.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/FlatLayout.tsxnew file mode 100644index 00000000000..b34ed202932--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/FlatLayout.tsx@@ -0,0 +1,29 @@+import { SimpleGrid, Stack } from "metabase/ui";+import type { InspectorCard } from "metabase-types/api";++import { ScalarCard } from "./ScalarCard";+import { VisualizationCard } from "./VisualizationCard";++type FlatLayoutProps = {+ cards: InspectorCard[];+};++export const FlatLayout = ({ cards }: FlatLayoutProps) => {+ const scalarCards = cards.filter((c) => c.display === "scalar");+ const otherCards = cards.filter((c) => c.display !== "scalar");++ return (+ <Stack gap="md">+ {scalarCards.length > 0 && (+ <SimpleGrid cols={Math.min(scalarCards.length, 4)} spacing="md">+ {scalarCards.map((card) => (+ <ScalarCard key={card.id} card={card} />+ ))}+ </SimpleGrid>+ )}+ {otherCards.map((card) => (+ <VisualizationCard key={card.id} card={card} />+ ))}+ </Stack>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ScalarCard.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ScalarCard.tsxnew file mode 100644index 00000000000..fbfebfcf506--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/ScalarCard.tsx@@ -0,0 +1,41 @@+import { Card, Loader, Stack, Text } from "metabase/ui";+import type { InspectorCard } from "metabase-types/api";++import { useLensCardLoader } from "../../../../hooks";+import { CardAlerts } from "../../../CardAlerts";+import { CardDrills } from "../../../CardDrills";+import { useLensContentContext } from "../../../LensContent/LensContentContext";++type ScalarCardProps = {+ card: InspectorCard;+};++export const ScalarCard = ({ card }: ScalarCardProps) => {+ const { alertsByCardId, drillLensesByCardId } = useLensContentContext();+ const { data, isLoading } = useLensCardLoader({ card });++ const alerts = alertsByCardId[card.id] ?? [];+ const drillLenses = drillLensesByCardId[card.id] ?? [];+ const value = data?.data?.rows?.[0]?.[0];++ return (+ <Card p="md" shadow="none" withBorder>+ <Stack gap="xs" align="center">+ <Text size="sm" c="text-secondary">+ {card.title}+ </Text>+ {isLoading ? (+ <Loader size="sm" />+ ) : (+ <Text size="xl" fw={700}>+ {value?.toString() ?? "-"}+ </Text>+ )}++ <CardAlerts alerts={alerts} fullWidth />++ <CardDrills drillLenses={drillLenses} />+ </Stack>+ </Card>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/VisualizationCard.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/VisualizationCard.tsxnew file mode 100644index 00000000000..343ac96e170--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/components/VisualizationCard.tsx@@ -0,0 +1,147 @@+import { useGetAdhocQueryMetadataQuery } from "metabase/api";+import { useSelector } from "metabase/lib/redux";+import * as Urls from "metabase/lib/urls";+import { getMetadata } from "metabase/selectors/metadata";+import { Box, Card, Loader, Stack } from "metabase/ui";+import Visualization from "metabase/visualizations/components/Visualization";+import * as Lib from "metabase-lib";+import { defaultDisplay } from "metabase-lib/viz/display";+import type {+ CardDisplayType,+ Dataset,+ InspectorCard,+ InspectorCardDisplayType,+ RawSeries,+ VisualizationSettings,+} from "metabase-types/api";+import { createMockCard } from "metabase-types/api/mocks";++import { useLensCardLoader } from "../../../../hooks";+import { CardAlerts } from "../../../CardAlerts";+import { CardDrills } from "../../../CardDrills";+import { useLensContentContext } from "../../../LensContent/LensContentContext";++type VisualizationCardProps = {+ card: InspectorCard;+ height?: number;+};++const DEFAULT_HEIGHT = 235;++export const VisualizationCard = ({+ card,+ height = DEFAULT_HEIGHT,+}: VisualizationCardProps) => {+ const { alertsByCardId, drillLensesByCardId } = useLensContentContext();+ const { data, isLoading: isDataLoading } = useLensCardLoader({ card });++ const { isLoading: isMetadataLoading } = useGetAdhocQueryMetadataQuery(+ card.dataset_query,+ );+ const metadata = useSelector(getMetadata);++ if (card.display === "hidden") {+ return null;+ }++ const alerts = alertsByCardId[card.id] ?? [];+ const drillLenses = drillLensesByCardId[card.id] ?? [];++ const { displayType, displaySettings } = getDisplayConfig(+ metadata,+ card,+ isMetadataLoading,+ );++ const rawSeries = buildRawSeries(data, card, displayType, displaySettings);+ const isLoading = isMetadataLoading || isDataLoading;++ const questionUrl = rawSeries?.[0]?.card+ ? Urls.serializedQuestion(rawSeries[0].card)+ : undefined;++ const getHref = questionUrl ? () => questionUrl : undefined;++ const onChangeCardAndRun = questionUrl+ ? () => Urls.openInNewTab(questionUrl)+ : undefined;++ return (+ <Card p="md" shadow="none" withBorder>+ <Stack gap="sm">+ {isLoading || !rawSeries ? (+ <Stack gap="sm" align="center" justify="center" h={height}>+ <Loader size="sm" />+ </Stack>+ ) : (+ <Box h={height}>+ <Visualization+ rawSeries={rawSeries}+ showTitle={true}+ // we want to have dashboard like experience: e.g. leaner UI, no column reordering, prevent row detail clicks+ isDashboard={true}+ getHref={getHref}+ onChangeCardAndRun={onChangeCardAndRun}+ />+ </Box>+ )}++ <CardAlerts alerts={alerts} />++ <CardDrills drillLenses={drillLenses} />+ </Stack>+ </Card>+ );+};++function getDisplayConfig(+ metadata: ReturnType<typeof getMetadata>,+ card: InspectorCard,+ isMetadataLoading: boolean,+) {+ if (isMetadataLoading) {+ return { displayType: card.display, displaySettings: {} };+ }++ try {+ const query = Lib.fromJsQueryAndMetadata(metadata, card.dataset_query);+ const { display, settings = {} } = defaultDisplay(query);+ const finalDisplay =+ display === "table" || display === "bar" ? card.display : display;+ return { displayType: finalDisplay, displaySettings: settings };+ } catch {+ return { displayType: card.display, displaySettings: {} };+ }+}++function buildRawSeries(+ dataset: Dataset | undefined,+ card: InspectorCard,+ displayType: InspectorCardDisplayType,+ displaySettings: Partial<VisualizationSettings>,+): RawSeries | undefined {+ if (!dataset) {+ return;+ }++ const vizDisplay: CardDisplayType =+ displayType === "hidden" ? "table" : displayType;++ return [+ {+ card: createMockCard({+ name: card.title,+ display: vizDisplay,+ dataset_query: card.dataset_query,+ visualization_settings: {+ "graph.y_axis.labels_enabled": false,+ "graph.x_axis.labels_enabled": false,+ "table.row_index": false, // hide row index column+ ...displaySettings,+ ...card.visualization_settings,+ },+ }),+ data: dataset.data,+ },+ ];+}diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/index.tsnew file mode 100644index 00000000000..9a9f1458d7a--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/DefaultLensSections/index.ts@@ -0,0 +1 @@+export { DefaultLensSections } from "./DefaultLensSections";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/GenericSummarySection.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/GenericSummarySection.tsxnew file mode 100644index 00000000000..f31eea53e77--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/GenericSummarySection.tsx@@ -0,0 +1,143 @@+import { useMemo } from "react";+import { msgid, ngettext, t } from "ttag";++import {+ Box,+ Card,+ SimpleGrid,+ Stack,+ Text,+ Title,+ TreeTable,+ type TreeTableColumnDef,+ useTreeTableInstance,+} from "metabase/ui";+import type {+ ConcreteTableId,+ InspectorCard,+ InspectorSource,+ InspectorTarget,+} from "metabase-types/api";++import { FieldInfoSection } from "./components/FieldInfoSection/FieldInfoSection";+import { RowCountCard } from "./components/RowCountCard";+import { treeTableStyles } from "./styles";++type GenericSummarySectionProps = {+ cards: InspectorCard[];+ sources: InspectorSource[];+ target?: InspectorTarget;+};++type TableRow = {+ id: string;+ card: InspectorCard;+ columnCount?: number;+};++export const GenericSummarySection = ({+ cards,+ sources,+ target,+}: GenericSummarySectionProps) => {+ const { inputData, outputData } = useMemo(() => {+ const input: TableRow[] = [];+ const output: TableRow[] = [];+ for (const card of cards) {+ const groupRole = card.metadata?.group_role;+ const tableId = card.metadata?.table_id;+ if (groupRole === "output") {+ output.push({ id: card.id, card, columnCount: target?.column_count });+ } else {+ const source = sources.find((source) => source.table_id === tableId);+ input.push({ id: card.id, card, columnCount: source?.column_count });+ }+ }+ return { inputData: input, outputData: output };+ }, [cards, sources, target]);++ const columns = useMemo<TreeTableColumnDef<TableRow>[]>(+ () => [+ {+ id: "table_name",+ header: t`Table`,+ accessorFn: (original) =>+ getTableName(+ original.card.metadata.table_id,+ [...sources, target].filter((t) => t !== undefined),+ ),+ cell: (props) => <Text>{String(props.getValue() ?? "-")}</Text>,+ },+ {+ id: "row_count",+ header: t`Rows`,+ cell: ({ row }) => <RowCountCard card={row.original.card} />,+ },+ {+ id: "column_count",+ header: t`Columns`,+ cell: ({ row }) => (+ <Text ta="right">+ {row.original.columnCount?.toLocaleString() ?? "-"}+ </Text>+ ),+ },+ ],+ [sources, target],+ );++ const inputInstance = useTreeTableInstance({+ data: inputData,+ columns,+ getNodeId: (node) => node.id,+ });++ const outputInstance = useTreeTableInstance({+ data: outputData,+ columns,+ getNodeId: (node) => node.id,+ });++ const inputCount = inputData.length;++ return (+ <Stack gap="xl">+ <Stack gap="md" data-testid="generic-summary-tables">+ <SimpleGrid cols={2} spacing="lg">+ <Title order={4}>+ {ngettext(+ msgid`${inputCount} input table`,+ `${inputCount} input tables`,+ inputCount,+ )}+ </Title>+ {/** we always expect to have one output table */}+ <Title order={4}>{t`1 output table`}</Title>+ </SimpleGrid>++ <Box>+ <SimpleGrid cols={2} spacing="lg">+ <Card p={0} shadow="none" withBorder>+ <TreeTable instance={inputInstance} styles={treeTableStyles} />+ </Card>+ <Card p={0} shadow="none" withBorder>+ <TreeTable instance={outputInstance} styles={treeTableStyles} />+ </Card>+ </SimpleGrid>+ </Box>+ </Stack>+ <FieldInfoSection sources={sources} target={target} />+ </Stack>+ );+};++function getTableName(+ tableId: ConcreteTableId | undefined,+ tables: Array<InspectorSource | InspectorTarget>,+): string | undefined {+ if (!tableId) {+ return undefined;+ }+ const table = tables.find((table) => table.table_id === tableId);+ return table?.table_name;+}diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/GenericSummarySections.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/GenericSummarySections.tsxnew file mode 100644index 00000000000..f6bf7e35f11--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/GenericSummarySections.tsx@@ -0,0 +1,34 @@+import type {+ InspectorCard,+ InspectorSection,+ InspectorSectionId,+ InspectorSource,+ InspectorTarget,+} from "metabase-types/api";++import { GenericSummarySection } from "./GenericSummarySection";++type GenericSummarySectionsProps = {+ sections: InspectorSection[];+ cardsBySection: Record<InspectorSectionId, InspectorCard[]>;+ sources: InspectorSource[];+ target?: InspectorTarget;+};++export const GenericSummarySections = ({+ sections,+ cardsBySection,+ sources,+ target,+}: GenericSummarySectionsProps) =>+ sections.map((section) => {+ const cards = cardsBySection[section.id];+ return (+ <GenericSummarySection+ key={section.id}+ cards={cards}+ sources={sources}+ target={target}+ />+ );+ });diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/FieldInfoSection.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/FieldInfoSection.tsxnew file mode 100644index 00000000000..b32610a7e9c--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/FieldInfoSection.tsx@@ -0,0 +1,98 @@+import type { Row } from "@tanstack/react-table";+import { useCallback, useMemo } from "react";+import { t } from "ttag";++import {+ Alert,+ Card,+ Icon,+ SimpleGrid,+ Stack,+ Title,+ TreeTable,+ type TreeTableColumnDef,+ useTreeTableInstance,+} from "metabase/ui";++import { treeTableStyles } from "../../styles";++import type { FieldInfoSectionProps, FieldTreeNode } from "./types";+import { buildTableNodes, getColumns } from "./utils";++export const FieldInfoSection = ({+ sources,+ target,+}: FieldInfoSectionProps) => {+ const sourceData = useMemo(() => buildTableNodes(sources), [sources]);+ const targetData = useMemo(+ () => (target ? buildTableNodes([target]) : []),+ [target],+ );++ const columns: TreeTableColumnDef<FieldTreeNode>[] = useMemo(+ () => getColumns(),+ [],+ );++ const handleRowClick = useCallback((row: Row<FieldTreeNode>) => {+ if (row.getCanExpand()) {+ row.toggleExpanded();+ }+ }, []);++ const sourceInstance = useTreeTableInstance({+ data: sourceData,+ columns,+ getNodeId: (node) => node.id,+ getSubRows: (node) => node.children,+ defaultExpanded: true,+ });++ const targetInstance = useTreeTableInstance({+ data: targetData,+ columns,+ getNodeId: (node) => node.id,+ getSubRows: (node) => node.children,+ defaultExpanded: true,+ });++ return (+ <SimpleGrid cols={2} spacing="lg" data-testid="generic-summary-fields">+ <Stack gap="md">+ <Title order={4}>{t`Input fields`}</Title>+ {sources.length > 0 ? (+ <Card p={0} shadow="none" withBorder>+ <TreeTable+ instance={sourceInstance}+ onRowClick={handleRowClick}+ styles={treeTableStyles}+ />+ </Card>+ ) : (+ <Alert+ color="warning"+ icon={<Icon name="warning" />}+ >{t`Missing input data`}</Alert>+ )}+ </Stack>++ <Stack gap="md">+ <Title order={4}>{t`Output fields`}</Title>+ {target ? (+ <Card p={0} shadow="none" withBorder>+ <TreeTable+ instance={targetInstance}+ onRowClick={handleRowClick}+ styles={treeTableStyles}+ />+ </Card>+ ) : (+ <Alert+ color="warning"+ icon={<Icon name="warning" />}+ >{t`No output table`}</Alert>+ )}+ </Stack>+ </SimpleGrid>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/FieldInfoSection.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/FieldInfoSection.unit.spec.tsxnew file mode 100644index 00000000000..8f3dd0a79f4--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/FieldInfoSection.unit.spec.tsx@@ -0,0 +1,25 @@+import { renderWithProviders, screen } from "__support__/ui";++import { FieldInfoSection } from "./FieldInfoSection";++function setup() {+ renderWithProviders(+ <FieldInfoSection+ sources={[]}+ target={{+ table_id: 1,+ table_name: "Output table",+ column_count: 0,+ fields: [],+ }}+ />,+ );+}++describe("FieldInfoSection", () => {+ it("shows warning when sources are empty", () => {+ setup();++ expect(screen.getByText("Missing input data")).toBeInTheDocument();+ });+});diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/types.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/types.tsnew file mode 100644index 00000000000..d563019a7cf--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/types.ts@@ -0,0 +1,32 @@+import type {+ ConcreteTableId,+ InspectorField,+ InspectorFieldStats,+ InspectorSource,+ InspectorTarget,+} from "metabase-types/api";++export type FieldInfoSectionProps = {+ sources: InspectorSource[];+ target?: InspectorTarget;+};++export type FieldTreeNode = {+ id: string;+ type: "table" | "field";+ tableName: string;+ fieldName?: string;+ fieldCount?: number;+ baseType?: string;+ stats?: InspectorFieldStats;+ original?: InspectorField;+ children?: FieldTreeNode[];+};++export type TableWithFields = {+ table_id?: ConcreteTableId | null;+ table_name: string;+ fields: InspectorField[];+};++export type StatsColumn = "distinct_count" | "range_and_averages";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/utils.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/utils.tsxnew file mode 100644index 00000000000..d363a7d31af--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/FieldInfoSection/utils.tsx@@ -0,0 +1,168 @@+import type { CellContext } from "@tanstack/react-table";+import { c, t } from "ttag";++import { getFormattedTime } from "metabase/common/components/DateTime";+import { Ellipsified } from "metabase/common/components/Ellipsified";+import { formatNumber, formatPercent } from "metabase/static-viz/lib/numbers";+import {+ EntityNameCell,+ Flex,+ Text,+ type TreeTableColumnDef,+} from "metabase/ui";+import {+ isDate,+ isDateWithoutTime,+ isNumeric,+ isString,+} from "metabase-lib/v1/types/utils/isa";+import type { InspectorField } from "metabase-types/api";++import type { FieldTreeNode, TableWithFields } from "./types";++export function buildTableNodes(tables: TableWithFields[]): FieldTreeNode[] {+ return tables.map((table) => {+ const tableKey = table.table_id ?? table.table_name;+ return {+ id: `table-${tableKey}`,+ type: "table" as const,+ tableName: table.table_name,+ fieldCount: table.fields.length,+ children: table.fields.map((field) => ({+ id: `field-${tableKey}-${field.name}`,+ type: "field" as const,+ tableName: table.table_name,+ fieldName: field.display_name ?? field.name,+ baseType: formatType(field),+ stats: field.stats,+ original: field,+ })),+ };+ });+}++export function formatType(field: InspectorField): string {+ return field.base_type?.replace("type/", "") ?? t`Unknown`;+}++export function getColumns(): TreeTableColumnDef<FieldTreeNode>[] {+ return [+ {+ id: "column",+ header: t`Field`,+ minWidth: "auto",+ accessorFn: (originalRow) =>+ originalRow.type === "table"+ ? `${originalRow.tableName} (${originalRow.fieldCount})`+ : originalRow.fieldName,+ cell: ({ row }) => {+ const node = row.original;+ if (node.type === "table") {+ return (+ <EntityNameCell+ icon="table"+ name={+ <Flex align="center" gap="xs">+ <Text fw="bold">{node.tableName}</Text>{" "}+ <Text c="text-secondary">({node.fieldCount})</Text>+ </Flex>+ }+ />+ );+ }+ return <Ellipsified>{node.fieldName}</Ellipsified>;+ },+ },+ {+ id: "type",+ header: t`Type`,+ width: "auto",+ accessorFn: (originalRow) => originalRow.baseType,+ cell: (props) => {+ const node = props.row.original;+ if (node.type === "field") {+ return <Text ta="right">{String(props.getValue())}</Text>;+ }+ return null;+ },+ },+ {+ id: "distinct_count",+ header: t`Distincts`,+ width: "auto",+ accessorFn: getDistinctCount,+ cell: renderEllipsified,+ },+ {+ id: "nil_percent",+ header: t`Nil %`,+ accessorFn: getNilPercent,+ width: "auto",+ cell: renderEllipsified,+ },+ {+ id: "range_and_averages",+ header: t`Range and averages`,+ width: "auto",+ accessorFn: (row: FieldTreeNode) => gerRangeAndAverages(row),+ cell: renderEllipsified,+ },+ ];+}++function renderEllipsified(props: CellContext<FieldTreeNode, unknown>) {+ const node = props.row.original;+ if (node.type === "table") {+ return null;+ }+ return <Ellipsified>{String(props.getValue())}</Ellipsified>;+}++function getDistinctCount(node: FieldTreeNode) {+ const distinctCount = node.stats?.distinct_count;+ return distinctCount !== undefined ? distinctCount.toLocaleString() : "";+}++function getNilPercent(node: FieldTreeNode) {+ const nilPercent = node.stats?.nil_percent;+ return nilPercent !== undefined ? formatPercent(nilPercent) : "";+}++function gerRangeAndAverages(node: FieldTreeNode) {+ if (!node.stats) {+ return "";+ }++ const { avg, min, max, earliest, latest } = node.stats;++ if (isNumeric(node.original)) {+ if (min === undefined || max === undefined) {+ return "";+ }+ const range = `${formatNumber(min)} – ${formatNumber(max)}`;+ if (avg != null) {+ return c("{0} represents range and {1} average)")+ .t`${range} (${formatNumber(avg)} avg)`;+ }+ return range;+ }++ if (isDate(node.original)) {+ if (earliest === undefined || latest === undefined) {+ return null;+ }+ const isDateOnly = isDateWithoutTime(node.original);+ const unit = isDateOnly ? "day" : undefined;+ return `${getFormattedTime(earliest, unit)} – ${getFormattedTime(latest, unit)}`;+ }++ if (isString(node.original)) {+ if (avg === undefined) {+ return "";+ }+ const avgRounded = Math.round(avg);+ return `${avgRounded} character avg.`;+ }++ return "";+}diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/RowCountCard.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/RowCountCard.tsxnew file mode 100644index 00000000000..33526e6ee8b--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/components/RowCountCard.tsx@@ -0,0 +1,28 @@+import { Box, Loader, Text } from "metabase/ui";+import type { InspectorCard } from "metabase-types/api";++import { useLensCardLoader } from "../../../../hooks";++type RowCountCardProps = {+ card: InspectorCard;+};++export const RowCountCard = ({ card }: RowCountCardProps) => {+ const { data, isLoading } = useLensCardLoader({ card });++ const rowCount = data?.data?.rows?.[0]?.[0];++ if (isLoading) {+ return (+ <Box ta="right">+ <Loader size="xs" />+ </Box>+ );+ }++ return (+ <Text size="sm" ta="right">+ {rowCount?.toString() ?? "-"}+ </Text>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/index.tsnew file mode 100644index 00000000000..2a8bdc7392f--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/index.ts@@ -0,0 +1 @@+export { GenericSummarySections } from "./GenericSummarySections";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/styles.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/styles.tsnew file mode 100644index 00000000000..a5b4ec2a238--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/GenericSummarySections/styles.ts@@ -0,0 +1,5 @@+export const treeTableStyles = {+ headerRow: {+ backgroundColor: "var(--mb-color-background-secondary)",+ },+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/JoinAnalysisSection.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/JoinAnalysisSection.tsxnew file mode 100644index 00000000000..7f8ff7ebeaf--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/JoinAnalysisSection.tsx@@ -0,0 +1,206 @@+import { useMemo } from "react";+import { msgid, ngettext, t } from "ttag";+import _ from "underscore";++import {+ Card,+ Stack,+ Title,+ TreeTable,+ type TreeTableColumnDef,+ useTreeTableInstance,+} from "metabase/ui";+import type { InspectorCard } from "metabase-types/api";++import { useLensContentContext } from "../../LensContent/LensContentContext";++import { AlertSubRow } from "./components/AlertSubRow";+import { DrillLensesCell } from "./components/DrillLensesCell";+import { JoinHeaderCell } from "./components/JoinHeaderCell";+import { JoinNameCell } from "./components/JoinNameCell";+import { JoinStepStatCell } from "./components/JoinStepStatCell";+import { TableCountCard } from "./components/TableCountCard";+import type { JoinTableRow } from "./types";+import { getMaxSeverity } from "./utils";++type JoinAnalysisSectionProps = {+ cards: InspectorCard[];+};++export const JoinAnalysisSection = ({ cards }: JoinAnalysisSectionProps) => {+ const {+ alertsByCardId,+ drillLensesByCardId,+ collectedCardStats,+ navigateToLens,+ transform,+ } = useLensContentContext();++ const { joinStepCards, tableCountCards } = useMemo(+ () => ({+ joinStepCards: cards.filter((c) => c.metadata.card_type === "join_step"),+ tableCountCards: cards.filter(+ (c) => c.metadata.card_type === "table_count",+ ),+ }),+ [cards],+ );++ const treeData = useMemo<JoinTableRow[]>(() => {+ const sortedCards = _.sortBy(+ joinStepCards,+ (card) => card.metadata?.join_step,+ );+ return sortedCards.map((card) => {+ const tableCard = tableCountCards.find(+ ({ metadata }) => metadata?.join_step === card.metadata?.join_step,+ );+ const alerts = alertsByCardId[card.id] ?? [];+ return {+ id: card.id,+ card,+ tableCard,+ joinAlias: card.metadata.join_alias?.toString() ?? t`Unknown`,+ joinStrategy: card.metadata.join_strategy?.toString() ?? "left-join",+ alerts,+ severity: alerts.length > 0 ? getMaxSeverity(alerts) : null,+ drillLenses: drillLensesByCardId[card.id] ?? [],+ cardStats: collectedCardStats[card.id],+ };+ });+ }, [+ joinStepCards,+ tableCountCards,+ alertsByCardId,+ drillLensesByCardId,+ collectedCardStats,+ ]);++ const hasDrills = useMemo(+ () => treeData.some((row) => row.drillLenses.length > 0),+ [treeData],+ );++ const columns = useMemo(+ () =>+ _.compact<(TreeTableColumnDef<JoinTableRow> | false)[]>([+ {+ id: "alert",+ width: 42,+ cell: ({ row }) => (+ <JoinHeaderCell+ card={row.original.card}+ severity={row.original.severity}+ transformId={transform.id}+ onToggleAlerts={() => row.toggleExpanded()}+ />+ ),+ },+ {+ id: "name",+ header: t`Join`,+ width: "auto",+ maxWidth: 436,+ accessorFn: (original) => original.joinAlias,+ cell: ({ row }) => (+ <JoinNameCell+ joinAlias={row.original.joinAlias}+ joinStrategy={row.original.joinStrategy}+ />+ ),+ },+ {+ id: "output",+ header: t`Output`,+ width: "auto",+ cell: ({ row }) => (+ <JoinStepStatCell+ value={row.original.cardStats?.["output_count"]}+ isLoading={!row.original.cardStats}+ />+ ),+ },+ {+ id: "matched",+ header: t`Matched`,+ width: "auto",+ cell: ({ row }) => (+ <JoinStepStatCell+ value={row.original.cardStats?.["matched_count"]}+ isLoading={!row.original.cardStats}+ />+ ),+ },+ {+ id: "table-rows-count",+ header: t`Table rows`,+ width: "auto",+ cell: ({ row }) =>+ row.original.tableCard ? (+ <TableCountCard card={row.original.tableCard} />+ ) : null,+ },+ hasDrills && {+ id: "drills",+ width: "auto",+ header: () => null,+ accessorFn: (original) =>+ original.drillLenses.length > 0 ? original.drillLenses : null,+ cell: ({ row }) => (+ <DrillLensesCell+ drillLenses={row.original.drillLenses}+ transformId={transform.id}+ navigateToLens={navigateToLens}+ />+ ),+ },+ ]),+ [hasDrills, navigateToLens, transform.id],+ );++ const instance = useTreeTableInstance({+ data: treeData,+ columns,+ getNodeId: (node) => node.id,+ enableSorting: false,+ });++ const joinsCount = treeData.length;++ return (+ <Stack gap="md">+ {joinsCount > 0 && (+ <>+ <Title order={4}>+ {ngettext(+ msgid`${joinsCount} join`,+ `${joinsCount} joins`,+ joinsCount,+ )}+ </Title>+ <Card p={0} shadow="none" withBorder>+ <TreeTable+ instance={instance}+ hierarchical={false}+ styles={{+ cell: { padding: "var(--mantine-spacing-sm)" },+ headerCell: { padding: "var(--mantine-spacing-sm)" },+ }}+ renderSubRow={(row) => {+ if (!row.getIsExpanded()) {+ return null;+ }+ return (+ <AlertSubRow+ alerts={row.original.alerts}+ severity={row.original.severity}+ />+ );+ }}+ />+ </Card>+ </>+ )}+ </Stack>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/JoinAnalysisSections.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/JoinAnalysisSections.tsxnew file mode 100644index 00000000000..f186bb9b3c9--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/JoinAnalysisSections.tsx@@ -0,0 +1,21 @@+import type {+ InspectorCard,+ InspectorSection,+ InspectorSectionId,+} from "metabase-types/api";++import { JoinAnalysisSection } from "./JoinAnalysisSection";++type JoinAnalysisSectionsProps = {+ sections: InspectorSection[];+ cardsBySection: Record<InspectorSectionId, InspectorCard[]>;+};++export const JoinAnalysisSections = ({+ sections,+ cardsBySection,+}: JoinAnalysisSectionsProps) =>+ sections.map((section) => {+ const cards = cardsBySection[section.id];+ return <JoinAnalysisSection key={section.id} cards={cards} />;+ });diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/AlertSubRow.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/AlertSubRow.tsxnew file mode 100644index 00000000000..a33981b752f--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/AlertSubRow.tsx@@ -0,0 +1,35 @@+import { match } from "ts-pattern";++import { Box, List } from "metabase/ui";+import type { InspectorAlertTrigger } from "metabase-types/api";++type AlertSubRowProps = {+ alerts: InspectorAlertTrigger[];+ severity: InspectorAlertTrigger["severity"] | null;+};++export const AlertSubRow = ({ alerts, severity }: AlertSubRowProps) => {+ if (!severity || alerts.length === 0) {+ return null;+ }++ return (+ <Box p="sm">+ <Box+ p="sm"+ bdrs="sm"+ bg={match(severity)+ .with("error", () => "background-error" as const)+ .with("warning", () => "background-warning" as const)+ .with("info", () => "background-secondary" as const)+ .exhaustive()}+ >+ <List>+ {alerts.map((alert) => (+ <List.Item key={alert.id}>{alert.message}</List.Item>+ ))}+ </List>+ </Box>+ </Box>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/DrillLensesCell.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/DrillLensesCell.tsxnew file mode 100644index 00000000000..094f380f494--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/DrillLensesCell.tsx@@ -0,0 +1,52 @@+import { c } from "ttag";++import { trackTransformInspectDrillLensClicked } from "metabase/transforms/analytics";+import { Flex } from "metabase/ui";+import type {+ InspectorDrillLensTrigger,+ TransformId,+} from "metabase-types/api";++import type { LensHandle } from "../../../../types";+import { DrillButton } from "../../../DrillButton";+import { getLensKey, toLensHandle } from "../../../LensNavigator/utils";++type DrillLensesCellProps = {+ drillLenses: InspectorDrillLensTrigger[];+ transformId: TransformId;+ navigateToLens: (lensHandle: LensHandle) => void;+};++export const DrillLensesCell = ({+ drillLenses,+ transformId,+ navigateToLens,+}: DrillLensesCellProps) => {+ if (drillLenses.length === 0) {+ return null;+ }++ return (+ <Flex direction="column" gap="xs">+ {drillLenses.map((drillLens) => {+ const lensHandle = toLensHandle(drillLens);+ return (+ <DrillButton+ key={getLensKey(lensHandle)}+ onClick={() => {+ trackTransformInspectDrillLensClicked({+ transformId,+ lensId: drillLens.lens_id,+ triggeredFrom: "join_analysis",+ });+ navigateToLens(lensHandle);+ }}+ >+ {c("Button to inspect a specific aspect of a data join")+ .t`Inspect ${drillLens.reason ?? drillLens.lens_id}`}+ </DrillButton>+ );+ })}+ </Flex>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinHeaderCell.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinHeaderCell.tsxnew file mode 100644index 00000000000..d3a2b843637--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinHeaderCell.tsx@@ -0,0 +1,53 @@+import { match } from "ts-pattern";++import { trackTransformInspectAlertClicked } from "metabase/transforms/analytics";+import { ActionIcon, Icon } from "metabase/ui";+import type {+ InspectorAlertTrigger,+ InspectorCard,+ TransformId,+} from "metabase-types/api";++import { useLensCardLoader } from "../../../../hooks";++type JoinHeaderCellProps = {+ card: InspectorCard;+ severity: InspectorAlertTrigger["severity"] | null;+ transformId: TransformId;+ onToggleAlerts?: () => void;+};++export const JoinHeaderCell = ({+ card,+ severity,+ transformId,+ onToggleAlerts,+}: JoinHeaderCellProps) => {+ useLensCardLoader({ card });+ if (!severity) {+ return null;+ }+ return (+ <ActionIcon+ variant="transparent"+ color={match(severity)+ .with("error", () => "error" as const)+ .with("warning", () => "warning" as const)+ .with("info", () => "brand" as const)+ .exhaustive()}+ size="lg"+ onClick={(e) => {+ e.stopPropagation();+ trackTransformInspectAlertClicked({ transformId, cardId: card.id });+ onToggleAlerts?.();+ }}+ >+ <Icon+ name={match(severity)+ .with("error", "warning", () => "warning_round_filled" as const)+ .with("info", () => "info" as const)+ .exhaustive()}+ />+ </ActionIcon>+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinNameCell.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinNameCell.tsxnew file mode 100644index 00000000000..638a13cf60a--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinNameCell.tsx@@ -0,0 +1,37 @@+import { match } from "ts-pattern";+import { t } from "ttag";++import { Ellipsified } from "metabase/common/components/Ellipsified";+import { FixedSizeIcon, Group, Tooltip } from "metabase/ui";++type JoinNameCellProps = {+ joinAlias: string;+ joinStrategy: string;+};++export const JoinNameCell = ({+ joinAlias,+ joinStrategy,+}: JoinNameCellProps) => (+ <Group gap="sm" wrap="nowrap" miw={0}>+ <Tooltip+ label={match(joinStrategy)+ .with("left-join", () => t`Left Join`)+ .with("right-join", () => t`Right Join`)+ .with("inner-join", () => t`Inner Join`)+ .with("full-join", () => t`Full Join`)+ .otherwise(() => joinStrategy)}+ >+ <FixedSizeIcon+ name={match(joinStrategy)+ .with("left-join", () => "join_left_outer" as const)+ .with("right-join", () => "join_right_outer" as const)+ .with("inner-join", () => "join_inner" as const)+ .with("full-join", () => "join_full_outer" as const)+ .otherwise(() => "join_left_outer" as const)}+ c="brand"+ />+ </Tooltip>+ <Ellipsified>{joinAlias}</Ellipsified>+ </Group>+);diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinStepStatCell.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinStepStatCell.tsxnew file mode 100644index 00000000000..91262c07ed5--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/JoinStepStatCell.tsx@@ -0,0 +1,16 @@+import { Loader, Text } from "metabase/ui";++type JoinStepStatCellProps = {+ value: undefined | unknown;+ isLoading: boolean;+};++export const JoinStepStatCell = ({+ value,+ isLoading,+}: JoinStepStatCellProps) => {+ if (isLoading) {+ return <Loader size="xs" />;+ }+ return <Text fw={500}>{value?.toString() ?? "-"}</Text>;+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/TableCountCard.tsx b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/TableCountCard.tsxnew file mode 100644index 00000000000..fe601795e43--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/components/TableCountCard.tsx@@ -0,0 +1,20 @@+import { Loader, Text } from "metabase/ui";+import type { InspectorCard } from "metabase-types/api";++import { useLensCardLoader } from "../../../../hooks";++type TableCountCardProps = {+ card: InspectorCard;+};++export const TableCountCard = ({ card }: TableCountCardProps) => {+ const { data, isLoading } = useLensCardLoader({ card });++ const tableCount = data?.data?.rows?.[0]?.[0];++ if (isLoading) {+ return <Loader size="xs" />;+ }++ return <Text>{tableCount?.toString() ?? "-"}</Text>;+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/index.tsnew file mode 100644index 00000000000..b4ffb9c46fe--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/index.ts@@ -0,0 +1 @@+export { JoinAnalysisSections } from "./JoinAnalysisSections";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/types.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/types.tsnew file mode 100644index 00000000000..f598dcc01f3--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/types.ts@@ -0,0 +1,18 @@+import type { CardStats } from "metabase/transforms/lib/transforms-inspector";+import type {+ InspectorAlertTrigger,+ InspectorCard,+ InspectorDrillLensTrigger,+} from "metabase-types/api";++export type JoinTableRow = {+ id: string;+ card: InspectorCard;+ tableCard: InspectorCard | undefined;+ joinAlias: string;+ joinStrategy: string;+ alerts: InspectorAlertTrigger[];+ severity: InspectorAlertTrigger["severity"] | null;+ drillLenses: InspectorDrillLensTrigger[];+ cardStats: CardStats | undefined;+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/utils.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/utils.tsnew file mode 100644index 00000000000..564bec0fb0d--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/JoinAnalysisSections/utils.ts@@ -0,0 +1,13 @@+import type { InspectorAlertTrigger } from "metabase-types/api";++export const getMaxSeverity = (+ alerts: InspectorAlertTrigger[],+): InspectorAlertTrigger["severity"] => {+ if (alerts.some((a) => a.severity === "error")) {+ return "error";+ }+ if (alerts.some((a) => a.severity === "warning")) {+ return "warning";+ }+ return "info";+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/index.tsnew file mode 100644index 00000000000..c05f0edd36a--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/LensSections/index.ts@@ -0,0 +1,3 @@+export { DefaultLensSections } from "./DefaultLensSections";+export { GenericSummarySections } from "./GenericSummarySections";+export { JoinAnalysisSections } from "./JoinAnalysisSections";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/components/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/index.tsnew file mode 100644index 00000000000..eddcce7759b--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/components/index.ts@@ -0,0 +1 @@+export { LensContent } from "./LensContent/LensContent";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/index.tsnew file mode 100644index 00000000000..100e53f0423--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/index.ts@@ -0,0 +1,3 @@+export { useCardLoadingTracker } from "./useCardLoadingTracker";+export { useTriggerEvaluation } from "./useTriggerEvaluation";+export { useLensCardLoader } from "./useLensCardLoader";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useCardLoadingTracker.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useCardLoadingTracker.tsnew file mode 100644index 00000000000..9b9c6868436--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useCardLoadingTracker.ts@@ -0,0 +1,46 @@+import { useCallback, useEffect, useMemo, useState } from "react";++import type {+ InspectorCardId,+ InspectorLens,+ InspectorLensId,+} from "metabase-types/api";++type CardState = "loading" | "loaded";++export const useCardLoadingTracker = (+ lens: InspectorLens | undefined,+ onAllCardsLoaded: (lensId: InspectorLensId) => void,+) => {+ const [cardsStates, setCardsStates] = useState<+ Record<InspectorCardId, CardState>+ >({});++ useEffect(() => {+ if (!lens) {+ return;+ }+ const states = Object.values(cardsStates);+ const isSomethingLoading = states.includes("loading");+ if (states.length > 0 && !isSomethingLoading) {+ onAllCardsLoaded(lens.id);+ }+ }, [lens, cardsStates, onAllCardsLoaded]);++ const markCard = useCallback(+ (state: CardState) => (cardId: InspectorCardId) => {+ setCardsStates((prev) =>+ prev[cardId] === state ? prev : { ...prev, [cardId]: state },+ );+ },+ [],+ );++ return useMemo(+ () => ({+ markCardLoaded: markCard("loaded"),+ markCardStartedLoading: markCard("loading"),+ }),+ [markCard],+ );+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useLensCardLoader.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useLensCardLoader.tsnew file mode 100644index 00000000000..68eeb910cdd--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useLensCardLoader.ts@@ -0,0 +1,47 @@+import { useEffect, useState } from "react";++import { useRunInspectorQueryQuery } from "metabase/api";+import {+ type CardStats,+ computeCardStats,+} from "metabase/transforms/lib/transforms-inspector";+import type { InspectorCard } from "metabase-types/api";++import { useLensContentContext } from "../components/LensContent/LensContentContext";++type UseLensCardLoaderOptions = {+ card: InspectorCard;+};++export const useLensCardLoader = ({ card }: UseLensCardLoaderOptions) => {+ const {+ lensHandle,+ transform,+ onStatsReady,+ onCardStartedLoading,+ onCardLoaded,+ } = useLensContentContext();+ const { data, isLoading } = useRunInspectorQueryQuery({+ transformId: transform.id,+ lensId: lensHandle.id,+ query: card.dataset_query,+ lensParams: lensHandle.params,+ });+ const [stats, setStats] = useState<CardStats | null>();++ useEffect(() => {+ onCardStartedLoading(card.id);+ }, [card.id, onCardStartedLoading]);++ useEffect(() => {+ if (isLoading) {+ return;+ }+ const stats = computeCardStats(lensHandle.id, card, data?.data?.rows);+ setStats(stats);+ onStatsReady(card.id, stats);+ onCardLoaded(card.id);+ }, [card, lensHandle, data, isLoading, onStatsReady, onCardLoaded]);++ return { data, isLoading, stats };+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useTriggerEvaluation.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useTriggerEvaluation.tsnew file mode 100644index 00000000000..ebaf461997c--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/hooks/useTriggerEvaluation.ts@@ -0,0 +1,73 @@+import { useCallback, useEffect, useMemo, useState } from "react";+import _ from "underscore";++import { evaluateTriggers } from "metabase/transforms/lib/transforms-inspector";+import type {+ InspectorAlertTrigger,+ InspectorCardId,+ InspectorDrillLensTrigger,+ InspectorLens,+} from "metabase-types/api";++import type { CardStats } from "../types";++type TriggerEvaluationResult = {+ alertsByCardId: Record<InspectorCardId, InspectorAlertTrigger[]>;+ drillLensesByCardId: Record<InspectorCardId, InspectorDrillLensTrigger[]>;+ collectedCardStats: Record<InspectorCardId, CardStats>;+ pushNewStats: (cardId: InspectorCardId, stats: CardStats | null) => void;+};++type TriggerEvaluationState = {+ alerts: InspectorAlertTrigger[];+ drillLenses: InspectorDrillLensTrigger[];+};++export const useTriggerEvaluation = (+ lens: InspectorLens | undefined,+): TriggerEvaluationResult => {+ const [cardsStats, setCardsStats] = useState<+ Record<InspectorCardId, CardStats>+ >({});+ const [state, setState] = useState<TriggerEvaluationState>({+ alerts: [],+ drillLenses: [],+ });++ const pushNewStats = useCallback(+ (cardId: InspectorCardId, stats: CardStats | null) => {+ if (stats) {+ setCardsStats((prev) => ({ ...prev, [cardId]: stats }));+ }+ },+ [],+ );++ useEffect(() => {+ if (!lens) {+ return;+ }+ const result = evaluateTriggers(lens, cardsStats);+ setState({+ alerts: result.alerts,+ drillLenses: result.drill_lenses,+ });+ }, [lens, cardsStats]);++ const alertsByCardId = useMemo(+ () => _.groupBy(state.alerts, ({ condition }) => condition.card_id),+ [state.alerts],+ );++ const drillLensesByCardId = useMemo(+ () => _.groupBy(state.drillLenses, ({ condition }) => condition.card_id),+ [state.drillLenses],+ );++ return {+ collectedCardStats: cardsStats,+ alertsByCardId,+ drillLensesByCardId,+ pushNewStats,+ };+};diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/index.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/index.tsnew file mode 100644index 00000000000..eaca67a4cf9--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/index.ts@@ -0,0 +1 @@+export { TransformInspectPage } from "./TransformInspectPage";diff --git a/frontend/src/metabase/transforms/pages/TransformInspectPage/types.ts b/frontend/src/metabase/transforms/pages/TransformInspectPage/types.tsnew file mode 100644index 00000000000..f350709d8c1--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformInspectPage/types.ts@@ -0,0 +1,9 @@+import type { InspectorLensId } from "metabase-types/api";+export type { LensHandle } from "metabase-types/api";++export type { CardStats } from "metabase/transforms/lib/transforms-inspector";++export type RouteParams = { transformId: string; lensId?: InspectorLensId };++// Lens ID concatenated with params+export type LensKey = string;diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsxindex 4de91bccd66..240466e8722 100644--- a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx@@ -14,7 +14,7 @@ import { useSelector } from "metabase/lib/redux";import * as Urls from "metabase/lib/urls";import { useMetadataToasts } from "metabase/metadata/hooks";import { PLUGIN_REMOTE_SYNC } from "metabase/plugins";-import { Anchor, Box, Divider, Group, Stack } from "metabase/ui";+import { Anchor, Box, Card, Divider, Group, Stack } from "metabase/ui";import type { Transform, TransformTagId } from "metabase-types/api";import {@@ -31,18 +31,16 @@ import { LogOutput } from "./LogOutput";type RunSectionProps = {transform: Transform;readOnly?: boolean;+ noTitle?: boolean;};-export function RunSection({ transform, readOnly }: RunSectionProps) {+export function RunSection({ transform, readOnly, noTitle }: RunSectionProps) {const isRemoteSyncReadOnly = useSelector(PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,);- return (- <TitleSection- label={t`Run this transform`}- description={t`This transform will be run whenever the jobs it belongs are scheduled.`}- >+ const content = (+ <><Stack><Group p="lg" justify="space-between"><RunStatusSection transform={transform} />@@ -61,6 +59,23 @@ export function RunSection({ transform, readOnly }: RunSectionProps) {readOnly={readOnly || isRemoteSyncReadOnly}/></Group>+ </>+ );++ if (noTitle) {+ return (+ <Card p={0} shadow="none" withBorder>+ {content}+ </Card>+ );+ }++ return (+ <TitleSection+ label={t`Run this transform`}+ description={t`This transform will be run whenever the jobs it belongs to are scheduled.`}+ >+ {content}</TitleSection>);}diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsxindex 3573a084b66..10fa3f3cde4 100644--- a/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsx+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsx@@ -1,20 +1,11 @@-import { useState } from "react";--import { skipToken, useGetTransformQuery } from "metabase/api";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";import { PageContainer } from "metabase/data-studio/common/components/PageContainer";import * as Urls from "metabase/lib/urls";import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";+import { useTransformWithPolling } from "metabase/transforms/hooks/use-transform-with-polling";import { Center } from "metabase/ui";-import type { Transform } from "metabase-types/api";import { TransformHeader } from "../../components/TransformHeader";-import { POLLING_INTERVAL } from "../../constants";-import {- isTransformCanceling,- isTransformRunning,- isTransformSyncing,-} from "../../utils";import { RunSection } from "./RunSection";@@ -26,26 +17,19 @@ type TransformRunPageProps = {params: TransformRunPageParams;};-export function TransformRunPage({ params }: TransformRunPageProps) {- const [isPolling, setIsPolling] = useState(false);+export const TransformRunPage = ({ params }: TransformRunPageProps) => {const transformId = Urls.extractEntityId(params.transformId);const {- data: transform,+ transform,isLoading: isLoadingTransform,error: transformError,- } = useGetTransformQuery(transformId ?? skipToken, {- pollingInterval: isPolling ? POLLING_INTERVAL : undefined,- });+ } = useTransformWithPolling(transformId);const { readOnly, isLoadingDatabases, databasesError } =useTransformPermissions({ transform });const isLoading = isLoadingTransform || isLoadingDatabases;const error = transformError || databasesError;- if (isPolling !== isPollingNeeded(transform)) {- setIsPolling(isPollingNeeded(transform));- }-- if (isLoading || error || transform == null) {+ if (isLoading || error || !transform) {return (<Center h="100%"><LoadingAndErrorWrapper loading={isLoading} error={error} />@@ -59,13 +43,4 @@ export function TransformRunPage({ params }: TransformRunPageProps) {<RunSection transform={transform} readOnly={readOnly} /></PageContainer>);-}--function isPollingNeeded(transform?: Transform) {- return (- transform != null &&- (isTransformRunning(transform) ||- isTransformCanceling(transform) ||- isTransformSyncing(transform))- );-}+};diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsxindex 9cbe8e2c22c..d255221ed5e 100644--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsx+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsx@@ -1,20 +1,11 @@-import { useState } from "react";--import { skipToken, useGetTransformQuery } from "metabase/api";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";import { PageContainer } from "metabase/data-studio/common/components/PageContainer";import * as Urls from "metabase/lib/urls";import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";import { Center } from "metabase/ui";-import type { Transform } from "metabase-types/api";import { TransformHeader } from "../../components/TransformHeader";-import { POLLING_INTERVAL } from "../../constants";-import {- isTransformCanceling,- isTransformRunning,- isTransformSyncing,-} from "../../utils";+import { useTransformWithPolling } from "../../hooks/use-transform-with-polling";import { TransformSettingsSection } from "./TransformSettingsSection";@@ -27,24 +18,17 @@ type TransformTargetPageProps = {};export const TransformSettingsPage = ({ params }: TransformTargetPageProps) => {- const [isPolling, setIsPolling] = useState(false);const transformId = Urls.extractEntityId(params.transformId);const {- data: transform,+ transform,isLoading: isLoadingTransform,error: transformError,- } = useGetTransformQuery(transformId ?? skipToken, {- pollingInterval: isPolling ? POLLING_INTERVAL : undefined,- });+ } = useTransformWithPolling(transformId);const { readOnly, isLoadingDatabases, databasesError } =useTransformPermissions({ transform });const isLoading = isLoadingTransform || isLoadingDatabases;const error = transformError || databasesError;- if (isPolling !== isPollingNeeded(transform)) {- setIsPolling(isPollingNeeded(transform));- }-if (isLoading || error || transform == null) {return (<Center h="100%">@@ -60,12 +44,3 @@ export const TransformSettingsPage = ({ params }: TransformTargetPageProps) => {</PageContainer>);};--function isPollingNeeded(transform?: Transform) {- return (- transform != null &&- (isTransformRunning(transform) ||- isTransformCanceling(transform) ||- isTransformSyncing(transform))- );-}diff --git a/frontend/src/metabase/transforms/routes.tsx b/frontend/src/metabase/transforms/routes.tsxindex 1311f75b4e0..e1b01353ab2 100644--- a/frontend/src/metabase/transforms/routes.tsx+++ b/frontend/src/metabase/transforms/routes.tsx@@ -15,6 +15,7 @@ import {} from "./pages/NewTransformPage";import { RunListPage } from "./pages/RunListPage";import { TransformDependenciesPage } from "./pages/TransformDependenciesPage";+import { TransformInspectPage } from "./pages/TransformInspectPage";import { TransformListPage } from "./pages/TransformListPage";import { TransformQueryPage } from "./pages/TransformQueryPage";import { TransformRunPage } from "./pages/TransformRunPage";@@ -39,6 +40,11 @@ export function getDataStudioTransformRoutes() {<Route path=":transformId/edit" component={TransformQueryPage} /><Route path=":transformId/run" component={TransformRunPage} /><Route path=":transformId/settings" component={TransformSettingsPage} />+ <Route path=":transformId/inspect" component={TransformInspectPage} />+ <Route+ path=":transformId/inspect/:lensId"+ component={TransformInspectPage}+ />{PLUGIN_DEPENDENCIES.isEnabled && (<Routepath=":transformId/dependencies"diff --git a/frontend/src/metabase/ui/components/data-display/TreeTable/EntityNameCell/EntityNameCell.tsx b/frontend/src/metabase/ui/components/data-display/TreeTable/EntityNameCell/EntityNameCell.tsxindex db8460faa24..50291708fe0 100644--- a/frontend/src/metabase/ui/components/data-display/TreeTable/EntityNameCell/EntityNameCell.tsx+++ b/frontend/src/metabase/ui/components/data-display/TreeTable/EntityNameCell/EntityNameCell.tsx@@ -8,7 +8,7 @@ import { Group, Icon, Text } from "metabase/ui";interface EntityNameCellProps {icon?: IconName;- name: string;+ name: React.ReactNode;iconColor?: ColorName;wrap?: boolean;ellipsifiedProps?: ComponentProps<typeof Ellipsified>;diff --git a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTable.tsx b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTable.tsxindex 2afd5859002..92a861ec0a2 100644--- a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTable.tsx+++ b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTable.tsx@@ -37,6 +37,8 @@ export function TreeTable<TData extends TreeNodeData>({isRowDisabled,getRowProps,getRowHref,+ renderSubRow,+ hierarchical = true,classNames,styles,ariaLabel,@@ -97,7 +99,7 @@ export function TreeTable<TData extends TreeNodeData>({return () => observer.disconnect();}, [containerRef, setContainerWidth, showCheckboxes]);- const hasExpandableNodes = useMemo(+ const hasExpandableRows = useMemo(() => rows.some((row) => row.getCanExpand()),[rows],);@@ -144,7 +146,7 @@ export function TreeTable<TData extends TreeNodeData>({table={table}columnWidths={columnWidths}showCheckboxes={showCheckboxes}- showExpandButtons={hasExpandableNodes}+ showExpandButtons={hasExpandableRows}indentWidth={indentWidth}activeRowId={activeRowId}selectedRowId={selectedRowId}@@ -161,6 +163,8 @@ export function TreeTable<TData extends TreeNodeData>({styles={styles}getRowProps={getRowProps}href={getRowHref?.(row)}+ renderSubRow={renderSubRow}+ hierarchical={hierarchical}/>);diff --git a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableHeader/TreeTableHeader.tsx b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableHeader/TreeTableHeader.tsxindex 79fbbfb4bfe..2f59961569c 100644--- a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableHeader/TreeTableHeader.tsx+++ b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableHeader/TreeTableHeader.tsx@@ -50,6 +50,7 @@ export function TreeTableHeader<TData extends TreeNodeData>({const isFirstColumn = index === 0;const columnStyle = {+ padding: "0.75rem",...getColumnStyle(columnWidths, column.id, isFirstColumn),...styles?.headerCell,};@@ -64,7 +65,6 @@ export function TreeTableHeader<TData extends TreeNodeData>({key={header.id}className={cx(S.cell, classNames?.headerCell)}align="center"- p="0.75rem"style={columnStyle}/>);@@ -77,7 +77,6 @@ export function TreeTableHeader<TData extends TreeNodeData>({key={header.id}className={cx(S.cell, classNames?.headerCell)}align="center"- p="0.75rem"style={columnStyle}role={isSortable ? "columnheader" : undefined}aria-sort={diff --git a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.module.css b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.module.cssindex e79bbc46b5f..633a73c8722 100644--- a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.module.css+++ b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.module.css@@ -4,7 +4,7 @@left: 0;width: 100%;- &:not(:last-child) .content {+ &:not(:last-child) {border-bottom: 1px solid var(--mb-color-border);}}diff --git a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.tsx b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.tsxindex 27ad9753919..1372e1768f0 100644--- a/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.tsx+++ b/frontend/src/metabase/ui/components/data-display/TreeTable/TreeTableRow/TreeTableRow.tsx@@ -40,6 +40,7 @@ interface TreeTableRowContentProps<TData extends TreeNodeData>onRowClick?: (row: Row<TData>, event: MouseEvent) => void;onRowDoubleClick?: (row: Row<TData>, event: MouseEvent) => void;rowProps?: Record<string, unknown>;+ hierarchical?: boolean;}// Memoized component that does not depend on virtualItem for performance reasons@@ -66,6 +67,7 @@ const TreeTableRowContent = memo(function TreeTableRowContent<classNames,styles,rowProps,+ hierarchical = false,}: TreeTableRowContentProps<TData>) {const isKeyboardFocused = activeRowId === row.id;const isSelected = selectedRowId === row.id;@@ -130,7 +132,7 @@ const TreeTableRowContent = memo(function TreeTableRowContent<const cellContent = flexRender(columnDef?.cell, cell.getContext());- if (colIndex === 0) {+ if (colIndex === 0 && hierarchical) {return (<Flexkey={cell.id}@@ -171,7 +173,6 @@ const TreeTableRowContent = memo(function TreeTableRowContent<</Flex>);}-return (<Flexkey={cell.id}@@ -179,8 +180,8 @@ const TreeTableRowContent = memo(function TreeTableRowContent<className={cx(S.cell, classNames?.cell)}align="center"gap="0.5rem"- p="0.75rem"style={{+ padding: "0.75rem",...getColumnStyle(columnWidths, cell.column.id, false),...styles?.cell,}}@@ -220,6 +221,8 @@ export function TreeTableRow<TData extends TreeNodeData>({styles,getRowProps,href,+ renderSubRow,+ hierarchical = true,}: TreeTableRowProps<TData>) {const rowProps = useMemo(() => getRowProps?.(row), [getRowProps, row]);@@ -245,16 +248,24 @@ export function TreeTableRow<TData extends TreeNodeData>({classNames={classNames}styles={styles}rowProps={rowProps}+ hierarchical={hierarchical}/>);- const renderContent = () =>- href ? (++ const renderContent = () => {+ const subRowContent = renderSubRow?.(row) ?? null;+ return href ? (<Link to={href} className={S.link}>{content}+ {subRowContent}</Link>) : (- content+ <>+ {content}+ {subRowContent}+ </>);+ };if (typeof virtualItemOrPinnedPosition === "string") {return (diff --git a/frontend/src/metabase/ui/components/data-display/TreeTable/types.ts b/frontend/src/metabase/ui/components/data-display/TreeTable/types.tsindex fdd6260f47a..52935ad6bc7 100644--- a/frontend/src/metabase/ui/components/data-display/TreeTable/types.ts+++ b/frontend/src/metabase/ui/components/data-display/TreeTable/types.ts@@ -19,6 +19,8 @@ import type {RefObject,} from "react";+export type RenderSubRow<TData> = (row: Row<TData>) => ReactNode;+/*** Base interface that all tree node data must extend.* IDs can be strings or numbers - getNodeId converts them to strings for TanStack Table.@@ -254,6 +256,11 @@ export interface TreeTableProps<TData extends TreeNodeData>*/getRowHref?: (row: Row<TData>) => string | null;+ renderSubRow?: RenderSubRow<TData>;++ /** When false, renders as a flat table without expand buttons or indentation. Defaults to true. */+ hierarchical?: boolean;+ariaLabel?: string;ariaLabelledBy?: string;}@@ -287,6 +294,8 @@ export interface TreeTableRowProps<TData extends TreeNodeData>getRowProps?: (row: Row<TData>) => Record<string, unknown>;/** When provided, renders the row as a link for Cmd+Click support */href?: string | null;+ renderSubRow?: RenderSubRow<TData>;+ hierarchical?: boolean;}/**diff --git a/frontend/test/__support__/server-mocks/transform.ts b/frontend/test/__support__/server-mocks/transform.tsindex 98592b80f86..28ef4f3efe6 100644--- a/frontend/test/__support__/server-mocks/transform.ts+++ b/frontend/test/__support__/server-mocks/transform.ts@@ -1,8 +1,11 @@import fetchMock from "fetch-mock";import type {+ Dataset,+ InspectorLensId,ListTransformRunsResponse,Transform,+ TransformId,TransformJob,TransformJobId,TransformTag,@@ -72,3 +75,14 @@ export function setupDeleteTransformJobEndpointWithError() {fetchMock.delete(`path:/api/transform-job/${jobId}`, 500);}++export function setupRunInspectorQueryEndpoint(+ transformId: TransformId,+ lensId: InspectorLensId,+ response: Dataset,+) {+ fetchMock.post(+ `path:/api/transform/${transformId}/inspect/${lensId}/query`,+ response,+ );+}diff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/lens_id.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/lens_id.yamlnew file mode 100644index 00000000000..dd2861ab415--- /dev/null+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/lens_id.yaml@@ -0,0 +1,38 @@+name: lens_id+display_name: Lens ID+description: null+created_at: '2026-02-11T16:02:44.430974Z'+active: true+visibility_type: normal+table_id: [Internal Metabase Database, public, v_query_log]+database_type: varchar+base_type: type/Text+effective_type: type/Text+semantic_type: null+database_is_auto_increment: false+database_required: false+fk_target_field_id: null+dimensions: []+json_unfolding: false+parent_id: null+coercion_strategy: null+preview_display: true+position: 20+custom_position: 0+database_position: 20+has_field_values: auto-list+settings: null+caveats: null+points_of_interest: null+nfc_path: null+serdes/meta:+- {id: Internal Metabase Database, model: Database}+- {id: public, model: Schema}+- {id: v_query_log, model: Table}+- {id: lens_id, model: Field}+database_default: null+database_indexed: null+database_is_generated: false+database_is_nullable: true+database_is_pk: false+database_partitioned: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/lens_params.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/lens_params.yamlnew file mode 100644index 00000000000..495edbbd773--- /dev/null+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/lens_params.yaml@@ -0,0 +1,38 @@+name: lens_params+display_name: Lens Params+description: null+created_at: '2026-02-11T16:02:44.430974Z'+active: true+visibility_type: normal+table_id: [Internal Metabase Database, public, v_query_log]+database_type: text+base_type: type/Text+effective_type: type/Text+semantic_type: null+database_is_auto_increment: false+database_required: false+fk_target_field_id: null+dimensions: []+json_unfolding: false+parent_id: null+coercion_strategy: null+preview_display: true+position: 21+custom_position: 0+database_position: 21+has_field_values: auto-list+settings: null+caveats: null+points_of_interest: null+nfc_path: null+serdes/meta:+- {id: Internal Metabase Database, model: Database}+- {id: public, model: Schema}+- {id: v_query_log, model: Table}+- {id: lens_params, model: Field}+database_default: null+database_indexed: null+database_is_generated: false+database_is_nullable: true+database_is_pk: false+database_partitioned: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/query.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/query.yamlindex b8a766aee82..77e939f2edc 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/query.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/query.yaml@@ -17,9 +17,9 @@ json_unfolding: falseparent_id: nullcoercion_strategy: nullpreview_display: false-position: 18+position: 22custom_position: 0-database_position: 18+database_position: 22has_field_values: nullsettings: nullcaveats: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/transform_id.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/transform_id.yamlnew file mode 100644index 00000000000..0654db39172--- /dev/null+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/transform_id.yaml@@ -0,0 +1,38 @@+name: transform_id+display_name: Transform ID+description: null+created_at: '2026-02-11T16:02:44.430974Z'+active: true+visibility_type: normal+table_id: [Internal Metabase Database, public, v_query_log]+database_type: int4+base_type: type/Integer+effective_type: type/Integer+semantic_type: null+database_is_auto_increment: false+database_required: false+fk_target_field_id: null+dimensions: []+json_unfolding: false+parent_id: null+coercion_strategy: null+preview_display: true+position: 18+custom_position: 0+database_position: 18+has_field_values: auto-list+settings: null+caveats: null+points_of_interest: null+nfc_path: null+serdes/meta:+- {id: Internal Metabase Database, model: Database}+- {id: public, model: Schema}+- {id: v_query_log, model: Table}+- {id: transform_id, model: Field}+database_default: null+database_indexed: null+database_is_generated: false+database_is_nullable: true+database_is_pk: false+database_partitioned: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/transform_qualified_id.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/transform_qualified_id.yamlnew file mode 100644index 00000000000..0ac95cfbcef--- /dev/null+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/fields/transform_qualified_id.yaml@@ -0,0 +1,38 @@+name: transform_qualified_id+display_name: Transform Qualified ID+description: null+created_at: '2026-02-11T16:02:44.430974Z'+active: true+visibility_type: normal+table_id: [Internal Metabase Database, public, v_query_log]+database_type: text+base_type: type/Text+effective_type: type/Text+semantic_type: type/Category+database_is_auto_increment: false+database_required: false+fk_target_field_id: null+dimensions: []+json_unfolding: false+parent_id: null+coercion_strategy: null+preview_display: true+position: 19+custom_position: 0+database_position: 19+has_field_values: auto-list+settings: null+caveats: null+points_of_interest: null+nfc_path: null+serdes/meta:+- {id: Internal Metabase Database, model: Database}+- {id: public, model: Schema}+- {id: v_query_log, model: Table}+- {id: transform_qualified_id, model: Field}+database_default: null+database_indexed: null+database_is_generated: false+database_is_nullable: true+database_is_pk: false+database_partitioned: nulldiff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 95e1c9938d8..2fdc07d8deb 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -3492,6 +3492,88 @@ databaseChangeLog:tableName: metabase_databasecolumnName: write_data_details+ - changeSet:+ id: v59.2026-02-17T12:00:00+ author: bronsa+ comment: Add query_execution.transform_id for inspector query attribution+ changes:+ - addColumn:+ tableName: query_execution+ columns:+ - column:+ name: transform_id+ type: int+ remarks: The ID of the Transform associated with this query execution, if any.++ - changeSet:+ id: v59.2026-02-17T12:00:01+ author: bronsa+ comment: Index query_execution.transform_id+ changes:+ - createIndex:+ tableName: query_execution+ indexName: idx_query_execution_transform_id+ columns:+ - column:+ name: transform_id++ - changeSet:+ id: v59.2026-02-17T12:01:02+ author: bronsa+ comment: Add query_execution.lens_id for inspector query attribution+ changes:+ - addColumn:+ tableName: query_execution+ columns:+ - column:+ name: lens_id+ type: varchar(254)+ remarks: The ID of the inspector lens associated with this query execution, if any.++ - changeSet:+ id: v59.2026-02-17T12:01:03+ author: bronsa+ comment: Add query_execution.lens_params for inspector query attribution+ changes:+ - addColumn:+ tableName: query_execution+ columns:+ - column:+ name: lens_params+ type: ${text.type}+ remarks: JSON-encoded parameters of the inspector lens associated with this query execution, if any.++ - changeSet:+ id: v59.2026-02-17T12:01:04+ author: bronsa+ comment: Updated view v_query_log v4 - add transform_id, lens_id, lens_params+ changes:+ - sqlFile:+ dbms: postgresql+ path: instance_analytics_views/query_log/v4/postgres-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: mysql,mariadb+ path: instance_analytics_views/query_log/v4/mysql-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: h2+ path: instance_analytics_views/query_log/v4/h2-query_log.sql+ relativeToChangelogFile: true+ rollback:+ - sqlFile:+ dbms: postgresql+ path: instance_analytics_views/query_log/v2/postgres-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: mysql,mariadb+ path: instance_analytics_views/query_log/v3/mysql-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: h2+ path: instance_analytics_views/query_log/v2/h2-query_log.sql+ relativeToChangelogFile: true+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/resources/migrations/instance_analytics_views/query_log/v4/h2-query_log.sql b/resources/migrations/instance_analytics_views/query_log/v4/h2-query_log.sqlnew file mode 100644index 00000000000..071200c222e--- /dev/null+++ b/resources/migrations/instance_analytics_views/query_log/v4/h2-query_log.sql@@ -0,0 +1,29 @@+DROP VIEW IF EXISTS v_query_log;+++CREATE OR REPLACE VIEW v_query_log AS+SELECT id AS entity_id,+ started_at,+ cast(running_time AS DOUBLE) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ 'card_' || card_id AS card_qualified_id,+ dashboard_id,+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,+ pulse_id,+ database_id,+ 'database_' || database_id AS database_qualified_id,+ cache_hit,+ action_id,+ 'action_' || action_id AS action_qualified_id,+ transform_id,+ 'transform_' || transform_id AS transform_qualified_id,+ lens_id,+ lens_params,+ query+FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;diff --git a/resources/migrations/instance_analytics_views/query_log/v4/mysql-query_log.sql b/resources/migrations/instance_analytics_views/query_log/v4/mysql-query_log.sqlnew file mode 100644index 00000000000..a561ee6bc7b--- /dev/null+++ b/resources/migrations/instance_analytics_views/query_log/v4/mysql-query_log.sql@@ -0,0 +1,31 @@+DROP VIEW IF EXISTS v_query_log;+++CREATE OR REPLACE+SQL SECURITY INVOKER+VIEW v_query_log AS+SELECT id AS entity_id,+ started_at,+ cast(running_time AS DOUBLE) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ concat('card_', card_id) AS card_qualified_id,+ dashboard_id,+ concat('dashboard_', dashboard_id) AS dashboard_qualified_id,+ pulse_id,+ database_id,+ concat('database_', database_id) AS database_qualified_id,+ cache_hit,+ action_id,+ concat('action_', action_id) AS action_qualified_id,+ transform_id,+ concat('transform_', transform_id) AS transform_qualified_id,+ lens_id,+ lens_params,+ query+FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;diff --git a/resources/migrations/instance_analytics_views/query_log/v4/postgres-query_log.sql b/resources/migrations/instance_analytics_views/query_log/v4/postgres-query_log.sqlnew file mode 100644index 00000000000..2afdb396a40--- /dev/null+++ b/resources/migrations/instance_analytics_views/query_log/v4/postgres-query_log.sql@@ -0,0 +1,29 @@+DROP VIEW IF EXISTS v_query_log;+++CREATE OR REPLACE VIEW v_query_log AS+SELECT id AS entity_id,+ started_at,+ cast(running_time AS double precision) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ 'card_' || card_id AS card_qualified_id,+ dashboard_id,+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,+ pulse_id,+ database_id,+ 'database_' || database_id AS database_qualified_id,+ cache_hit,+ action_id,+ 'action_' || action_id AS action_qualified_id,+ transform_id,+ 'transform_' || transform_id AS transform_qualified_id,+ lens_id,+ lens_params,+ query+FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;diff --git a/shadow-cljs.edn b/shadow-cljs.ednindex 8e0a3b16950..2ab681b914e 100644--- a/shadow-cljs.edn+++ b/shadow-cljs.edn@@ -22,6 +22,7 @@metabase.lib.column-groupmetabase.lib.jsmetabase.lib.limit+ metabase.transforms-inspector.jsmetabase.lib.types.isametabase.parameters.sharedmetabase.pivot.corediff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex bf497b73286..e7b4ca95da3 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -54,9 +54,9 @@[metabase.testing-api.core][metabase.tiles.api][metabase.timeline.api]- [metabase.transforms.api.transform]- [metabase.transforms.api.transform-job]- [metabase.transforms.api.transform-tag]+ [metabase.transforms-rest.api.transform]+ [metabase.transforms-rest.api.transform-job]+ [metabase.transforms-rest.api.transform-tag][metabase.upload.api][metabase.user-key-value.api][metabase.users-rest.api]@@ -99,9 +99,9 @@metabase.task-history.api/keep-memetabase.testing-api.api/keep-memetabase.tiles.api/keep-me- metabase.transforms.api.transform/keep-me- metabase.transforms.api.transform-job/keep-me- metabase.transforms.api.transform-tag/keep-me+ metabase.transforms-rest.api.transform/keep-me+ metabase.transforms-rest.api.transform-job/keep-me+ metabase.transforms-rest.api.transform-tag/keep-memetabase.upload.api/keep-memetabase.user-key-value.api/keep-memetabase.users-rest.api/keep-me@@ -200,9 +200,9 @@"/tiles" (+auth 'metabase.tiles.api)"/timeline" (+auth metabase.timeline.api/timeline-routes)"/timeline-event" (+auth metabase.timeline.api/timeline-event-routes)- "/transform" (+auth metabase.transforms.api.transform/routes)- "/transform-job" (+auth metabase.transforms.api.transform/transform-job-routes)- "/transform-tag" (+auth metabase.transforms.api.transform/transform-tag-routes)+ "/transform" (+auth metabase.transforms-rest.api.transform/routes)+ "/transform-job" (+auth metabase.transforms-rest.api.transform/transform-job-routes)+ "/transform-tag" (+auth metabase.transforms-rest.api.transform/transform-tag-routes)"/upload" (+auth 'metabase.upload.api)"/user" (+auth 'metabase.users-rest.api)"/user-key-value" (+auth 'metabase.user-key-value.api)diff --git a/src/metabase/audit_app/events/audit_log.clj b/src/metabase/audit_app/events/audit_log.cljindex 16181ca1d89..915b0f8e810 100644--- a/src/metabase/audit_app/events/audit_log.clj+++ b/src/metabase/audit_app/events/audit_log.clj@@ -328,6 +328,8 @@(derive :event/update-transform ::transform-event)(derive :event/transform-delete ::transform-event)(derive :event/transform-run-start ::transform-event)+(derive :event/transform-inspect-discover ::transform-event)+(derive :event/transform-inspect-lens ::transform-event)(methodical/defmethod events/publish-event! ::transform-event[topic event]diff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljcindex 0e9a99c346e..edc0949fcdb 100644--- a/src/metabase/lib/schema/info.cljc+++ b/src/metabase/lib/schema/info.cljc@@ -36,7 +36,8 @@:embedded-csv-download:embedded-xlsx-download:embedded-json-download- :table-grid])+ :table-grid+ :transform-inspector])(mr/def ::hash#?(:clj bytes?@@ -62,6 +63,9 @@[:card-id {:optional true} [:maybe ::lib.schema.id/card]][:card-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]][:dashboard-id {:optional true} [:maybe ::lib.schema.id/dashboard]]+ [:transform-id {:optional true} [:maybe ::lib.schema.id/transform]]+ [:lens-id {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:lens-params {:optional true} [:maybe [:map-of :keyword :any]]][:pulse-id {:optional true} [:maybe ::lib.schema.id/pulse]];; Metadata for datasets when querying the dataset. This ensures that user edits to dataset metadata are blended in;; with runtime computed metadata so that edits are saved.diff --git a/src/metabase/queries/models/query_execution.clj b/src/metabase/queries/models/query_execution.cljindex 2d2bcce4d89..b9a65e76dfb 100644--- a/src/metabase/queries/models/query_execution.clj+++ b/src/metabase/queries/models/query_execution.clj@@ -20,6 +20,7 @@(t2/deftransforms :model/QueryExecution{:json_query mi/transform-json+ :lens_params mi/transform-json:status mi/transform-keyword:context mi/transform-keyword})diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.cljindex 9afcfb5364b..94672a35f6c 100644--- a/src/metabase/query_processor/middleware/process_userland_query.clj+++ b/src/metabase/query_processor/middleware/process_userland_query.clj@@ -89,7 +89,7 @@(merge(-> query-executionadd-running-time- (dissoc :error :hash :executor_id :action_id :is_sandboxed :card_id :dashboard_id :pulse_id :result_rows :native+ (dissoc :error :hash :executor_id :action_id :is_sandboxed :card_id :dashboard_id :transform_id :lens_id :lens_params :pulse_id :result_rows :native:parameterized))(dissoc result :cache/details){:cached (when (:cached cache) (:updated_at cache))@@ -123,7 +123,7 @@(mu/defn- query-execution-info"Return the info for the QueryExecution entry for this `query`."{:arglists '([query])}- [{{:keys [executed-by query-hash context action-id card-id dashboard-id pulse-id]+ [{{:keys [executed-by query-hash context action-id card-id dashboard-id transform-id lens-id lens-params pulse-id]:pivot/keys [original-query]} :infodatabase-id :databasequery-type :type@@ -142,6 +142,9 @@:action_id action-id:card_id card-id:dashboard_id dashboard-id+ :transform_id transform-id+ :lens_id lens-id+ :lens_params lens-params:pulse_id pulse-id:context context:hash query-hashdiff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 26ff5792cf6..ce6ca9432be 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -1,11 +1,22 @@(ns metabase.transforms.core"API namespace for the `metabase.transforms` module."(:require+ [java-time.api :as t][metabase.models.transforms.transform]+ [metabase.models.transforms.transform-job][metabase.models.transforms.transform-run]+ [metabase.models.transforms.transform-run-cancelation]+ [metabase.models.transforms.transform-tag]+ [metabase.transforms.canceling]+ [metabase.transforms.crud]+ [metabase.transforms.execute]+ [metabase.transforms.jobs]+ [metabase.transforms.ordering]+ [metabase.transforms.schedule][metabase.transforms.settings][metabase.transforms.util]- [potemkin :as p]))+ [potemkin :as p]+ [toucan2.core :as t2]))(p/import-vars[metabase.transforms.settings@@ -19,7 +30,67 @@transform-source-typetransform-typeis-temp-transform-table?]- [metabase.models.transforms.transform-run- timeout-run!]+ [metabase.transforms.crud+ python-source-table-ref->table-id+ check-database-feature+ check-feature-enabled!+ extract-all-columns-from-query+ extract-incremental-filter-columns-from-query+ validate-incremental-column-type!+ validate-transform-query!+ get-transforms+ get-transform+ create-transform!+ update-transform!+ delete-transform!]+ [metabase.transforms.canceling+ cancel-run!]+ [metabase.transforms.execute+ execute!]+ [metabase.transforms.ordering+ transform-ordering+ get-transform-cycle]+ [metabase.transforms.jobs+ run-job!+ job-transforms]+ [metabase.transforms.schedule+ validate-cron-expression+ initialize-job!+ update-job!+ delete-job!+ existing-trigger][metabase.models.transforms.transform- update-transform-tags!])+ update-transform-tags!]+ [metabase.models.transforms.transform-run+ timeout-run!+ paged-runs+ running-run-for-transform-id]+ [metabase.models.transforms.transform-run-cancelation+ mark-cancel-started-run!]+ [metabase.models.transforms.transform-job+ update-job-tags!]+ [metabase.models.transforms.transform-tag+ tag-name-exists?+ tag-name-exists-excluding?])++(defn transform-stats+ "Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.+ Rolling stats are included under :transform-rolling (up to date totals for today)"+ []+ (let [today-utc (t/offset-date-time (t/zone-offset "+00"))+ yesterday-utc (t/minus today-utc (t/days 1))+ count-runs (fn [source-types date]+ (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]+ :from [[:transform_run :r]]+ :join [[:transform :t] [:= :t.id :r.transform_id]]+ :where [:and+ [:= :r.status "succeeded"]+ [:in :t.source_type source-types]+ [:= [:cast :r.end_time :date] [:cast date :date]]]}))+ 0))]+ {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)+ :transform-python-runs (count-runs ["python"] yesterday-utc)+ :transform-usage-date (str (t/local-date yesterday-utc))+ :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)+ :transform-rolling-python-runs (count-runs ["python"] today-utc)+ :transform-rolling-usage-date (str (t/local-date today-utc))}))diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljnew file mode 100644index 00000000000..4da9e2ef79d--- /dev/null+++ b/src/metabase/transforms/crud.clj@@ -0,0 +1,243 @@+(ns metabase.transforms.crud+ "CRUD operations for transforms. Extracted from `metabase.transforms-rest.api.transform`+ so that non-REST modules (e.g. metabot-v3, workspaces) can use them without depending+ on the `-rest` module."+ (:require+ [metabase.api.common :as api]+ [metabase.database-routing.core :as database-routing]+ [metabase.driver :as driver]+ [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]+ [metabase.driver.util :as driver.u]+ [metabase.events.core :as events]+ [metabase.lib.core :as lib]+ [metabase.models.interface :as mi]+ [metabase.models.transforms.transform :as transform.model]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.transforms.interface :as transforms.i]+ [metabase.transforms.ordering :as transforms.ordering]+ [metabase.transforms.util :as transforms.util]+ [metabase.util :as u]+ [metabase.util.i18n :refer [deferred-tru]]+ [metabase.util.log :as log]+ [toucan2.core :as t2])+ (:import+ (java.sql PreparedStatement)))++(set! *warn-on-reflection* true)++(defn python-source-table-ref->table-id+ "Change source of python transform from name->table-ref to name->table-id.++ We now supported table-ref as source but since FE is still expecting table-id we need to temporarily do this.+ Should update FE to fully use table-ref"+ [transform]+ (if (transforms.util/python-transform? transform)+ (update-in transform [:source :source-tables]+ (fn [source-tables]+ (update-vals source-tables #(if (int? %) % (:table_id %)))))+ transform))++(defn check-database-feature+ "Check that the target database supports the required features for this transform."+ [transform]+ (let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))+ (deferred-tru "The target database cannot be found."))+ features (transforms.util/required-database-features transform)]+ (api/check-400 (not (:is_sample database))+ (deferred-tru "Cannot run transforms on the sample database."))+ (api/check-400 (not (:is_audit database))+ (deferred-tru "Cannot run transforms on audit databases."))+ (api/check-400 (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)+ (deferred-tru "The database does not support the requested transform features."))+ (api/check-400 (not (database-routing/db-routing-enabled? database))+ (deferred-tru "Transforms are not supported on databases with DB routing enabled."))))++(defn check-feature-enabled!+ "Check that the premium features required for this transform type are enabled."+ [transform]+ (api/check (transforms.util/check-feature-enabled transform)+ [402 (deferred-tru "Premium features required for this transform type are not enabled.")]))++(defn validate-transform-query!+ "Validate that the transform query is valid. Throws a 400 error if validation fails."+ [transform]+ (when-let [error (transforms.util/validate-transform-query transform)]+ (throw (ex-info (:error error)+ (assoc error+ :status-code 400)))))++(defn extract-all-columns-from-query+ "Extracts column metadata (name and type) from a query.++ Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.++ The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()+ to inspect the query structure. This works for most modern JDBC drivers but may not+ be supported by all drivers or for all query types."+ [driver database-id query]+ (try+ (let [{:keys [query]} (qp.compile/compile query)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database-id+ {}+ (fn [conn]+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]+ (when-let [rsmeta (.getMetaData stmt)]+ (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))+ (catch Exception e+ (log/debugf e "Failed to extract columns from query: %s" (ex-message e))+ nil)))++(defn extract-incremental-filter-columns-from-query+ "Extracts column names suitable for incremental transform checkpoint filtering.++ This function is specifically for incremental transform checkpoint column selection.+ It only returns columns with types supported for checkpoint filtering:+ - Temporal types (timestamp, timestamp with timezone)+ - Numeric types (integer, float, decimal)++ Text, boolean, and other types are filtered out as they are not supported for+ incremental checkpointing.++ Returns a vector of column names (as strings), or nil if extraction fails.++ The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()+ to inspect the query structure. This works for most modern JDBC drivers but may not+ be supported by all drivers or for all query types."+ [driver database-id query]+ (some->> (extract-all-columns-from-query driver database-id query)+ (filter (comp transforms.util/supported-incremental-filter-type? :base_type))+ (mapv :name)))++(defn validate-incremental-column-type!+ "Validates that the checkpoint column for an incremental transform has a supported type.++ For MBQL/Python transforms, resolves the column from the query using the unique key.+ For native queries, extracts columns from the query and checks the checkpoint-filter column.++ Throws a 400 error if the column type is not supported or cannot be resolved."+ [{:keys [source]}]+ (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}+ (:source-incremental-strategy source)]+ (when (and (= :query (:type source)) (= "checkpoint" strategy-type))+ (let [{:keys [query]} source+ database-id (:database query)+ database (api/check-404 (t2/select-one :model/Database :id database-id))+ driver-name (driver/the-initialized-driver (:engine database))]+ (cond+ ;; For MBQL, resolve column from query metadata+ checkpoint-filter-unique-key+ (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]+ (api/check-400 column (deferred-tru "Checkpoint column not found in query."))+ (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))+ (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."+ (pr-str (:base-type column)))))++ ;; For native query with checkpoint-filter, validate type if we can extract the column metadata+ checkpoint-filter+ (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]+ (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]+ (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."+ checkpoint-filter+ (pr-str (:base_type column)))))))))))++(defn get-transforms+ "Get a list of transforms."+ [& {:keys [last-run-start-time last-run-statuses tag-ids]}]+ (let [enabled-types (transforms.util/enabled-source-types-for-user)]+ (api/check-403 (seq enabled-types))+ (let [transforms (t2/select :model/Transform {:where [:in :source_type enabled-types]+ :order-by [[:id :asc]]})]+ (->> (t2/hydrate transforms :last_run :transform_tag_ids :creator :owner)+ (into []+ (comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last-run-start-time)+ (transforms.util/->status-filter-xf [:last_run :status] last-run-statuses)+ (transforms.util/->tag-filter-xf [:tag_ids] tag-ids)+ (map #(update % :last_run transforms.util/localize-run-timestamps))+ (map python-source-table-ref->table-id)))+ transforms.util/add-source-readable))))++(defn get-transform+ "Get a specific transform."+ [id]+ (let [{:keys [target] :as transform} (api/read-check :model/Transform id)+ target-table (transforms.util/target-table (transforms.i/target-db-id transform) target :active true)]+ (-> transform+ (t2/hydrate :last_run :transform_tag_ids :creator :owner)+ (u/update-some :last_run transforms.util/localize-run-timestamps)+ (assoc :table target-table)+ python-source-table-ref->table-id+ transforms.util/add-source-readable)))++(defn create-transform!+ "Create new transform in the appdb.+ Optionally accepts a creator-id to use instead of the current user (for workspace merges)."+ ([body]+ (create-transform! body nil))+ ([body creator-id]+ (when (transforms.util/query-transform? body)+ (validate-transform-query! body))+ (let [creator-id (or creator-id api/*current-user-id*)+ transform (t2/with-transaction [_]+ (let [tag-ids (:tag_ids body)+ ;; Set owner_user_id to current user if not explicitly provided+ owner-user-id (when-not (:owner_email body)+ (or (:owner_user_id body) creator-id))+ transform (t2/insert-returning-instance!+ :model/Transform+ (assoc (select-keys body [:name :description :source :target :run_trigger+ :collection_id :owner_email])+ :creator_id creator-id+ :owner_user_id owner-user-id))]+ ;; Add tag associations if provided+ (when (seq tag-ids)+ (transform.model/update-transform-tags! (:id transform) tag-ids))+ ;; Return with hydrated tag_ids+ (t2/hydrate transform :transform_tag_ids :creator :owner)))]+ (events/publish-event! :event/transform-create {:object transform :user-id creator-id})+ transform)))++(defn update-transform!+ "Update a transform. Validates features, database support, cycles, and target conflicts.+ Returns the updated transform with hydrated associations."+ [id body]+ (let [transform (t2/with-transaction [_]+ ;; Cycle detection should occur within the transaction to avoid race+ (let [old (t2/select-one :model/Transform id)+ new (merge old body)+ target-fields #(-> % :target (select-keys [:schema :name]))]+ (api/check-403 (and (mi/can-write? old) (mi/can-write? new)))++ ;; we must validate on a full transform object+ (check-feature-enabled! new)+ (check-database-feature new)+ (validate-incremental-column-type! new)+ (when (transforms.util/query-transform? old)+ (validate-transform-query! new)+ (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]+ (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)+ {:status-code 400}))))+ (api/check (not (and (not= (target-fields old) (target-fields new))+ (transforms.util/target-table-exists? new)))+ 403+ (deferred-tru "A table with that name already exists.")))+ (t2/update! :model/Transform id (dissoc body :tag_ids))+ ;; Update tag associations if provided+ (when (contains? body :tag_ids)+ (transform.model/update-transform-tags! id (:tag_ids body)))+ (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids :creator :owner))]+ (events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})+ (-> transform+ python-source-table-ref->table-id+ transforms.util/add-source-readable)))++(defn delete-transform!+ "Delete a transform and publish the delete event."+ [transform]+ (t2/delete! :model/Transform (:id transform))+ (events/publish-event! :event/transform-delete+ {:object transform+ :user-id api/*current-user-id*})+ nil)diff --git a/src/metabase/transforms_inspector/card_result.cljc b/src/metabase/transforms_inspector/card_result.cljcnew file mode 100644index 00000000000..04d66db4d3b--- /dev/null+++ b/src/metabase/transforms_inspector/card_result.cljc@@ -0,0 +1,48 @@+(ns metabase.transforms-inspector.card-result+ "Compute derived fields from raw query results for inspector cards.++ This is a cljc namespace so it can be used by both:+ - Backend: to compute card results after query execution+ - Frontend: to compute results client-side++ The multimethod dispatches on [lens-id card-type] to allow+ lens-specific result computation for different card types.")++;;; -------------------------------------------------- Card Result Computation --------------------------------------------------++(defmulti compute-card-result+ "Compute derived fields from first row of query result for a card.+ Dispatches on [lens-id card-type]. Returns a map of field-name -> value, or nil.++ Arguments:+ - lens-id: keyword like :join-analysis+ - card: the card map with :id, :metadata, etc.+ - row: the first row of query result (vector)"+ {:arglists '([lens-id card row])}+ (fn [lens-id card _row]+ [lens-id (some-> (get-in card [:metadata :card_type]) keyword)]))++(defmethod compute-card-result :default+ [_ _ row]+ (when (nil? row)+ {"no_data" true}))++(defmethod compute-card-result [:join-analysis :join_step]+ [_ _card row]+ (if (nil? row)+ {"no_data" true+ "output_count" 0+ "matched_count" 0+ "null_count" 0+ "null_rate" nil}+ (let [output-count (nth row 0 nil)+ ;; Non-outer joins return only COUNT(*); matched = output by definition+ matched-count (or (nth row 1 nil) output-count)+ null-count (when (and output-count matched-count)+ (- output-count matched-count))+ null-rate (when (and null-count output-count (pos? output-count))+ (/ null-count output-count))]+ {"output_count" output-count+ "matched_count" matched-count+ "null_count" null-count+ "null_rate" null-rate})))diff --git a/src/metabase/transforms_inspector/context.clj b/src/metabase/transforms_inspector/context.cljnew file mode 100644index 00000000000..28672555bf7--- /dev/null+++ b/src/metabase/transforms_inspector/context.clj@@ -0,0 +1,279 @@+(ns metabase.transforms-inspector.context+ "Context building for Transform Inspector."+ (:require+ [clojure.set :as set]+ [clojure.string :as str]+ [metabase.driver :as driver]+ [metabase.lib.core :as lib]+ [metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.transforms-inspector.query-analysis :as query-analysis]+ [metabase.transforms-inspector.schema :as transforms-inspector.schema]+ [metabase.transforms.interface :as transforms.i]+ [metabase.transforms.util :as transforms.util]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Source Extraction --------------------------------------------------++(defn- table-ids->source-info+ "Fetch tables by ID and build source info maps."+ [table-ids]+ (when (seq table-ids)+ (t2/select [:model/Table [:id :table-id] [:name :table-name] :schema [:db_id :db-id]] :id [:in table-ids])))++(defmulti extract-sources+ "Extract source table information for a transform.+ Returns a seq of maps with :table-id, :table-name, :schema, and :db-id."+ {:arglists '([transform])}+ (fn [transform] (transforms.util/transform-source-type (:source transform))))++(defmethod extract-sources :mbql+ [{:keys [source]}]+ (try+ (let [query (-> (:query source)+ transforms.util/massage-sql-query+ qp.preprocess/preprocess)+ table-ids (lib/all-source-table-ids query)]+ (table-ids->source-info table-ids))+ (catch Exception e+ (log/warn e "Failed to extract sources from MBQL transform")+ nil)))++(defmethod extract-sources :native+ [{:keys [source] :as transform}]+ (try+ (let [query (-> (:query source)+ transforms.util/massage-sql-query+ qp.preprocess/preprocess)+ db-id (transforms.util/transform-source-database transform)+ driver (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)+ deps (driver/native-query-deps driver query)+ table-ids (keep :table deps)]+ (table-ids->source-info table-ids))+ (catch Exception e+ (log/warn e "Failed to extract sources from native transform")+ nil)))++(defmethod extract-sources :python+ [transform]+ (try+ (let [source-tables (get-in transform [:source :source-tables])+ normalized (transforms.util/normalize-source-tables source-tables)+ table-ids (keep (fn [[_ v]] (:table_id v)) normalized)]+ (table-ids->source-info table-ids))+ (catch Exception e+ (log/warn e "Failed to extract sources from Python transform")+ nil)))++(defmethod extract-sources :default+ [_transform]+ nil)++;;; -------------------------------------------------- Target Table --------------------------------------------------++(defn- get-target-table+ "Get the target table for a transform. Returns nil if the target doesn't exist."+ [{:keys [target] :as transform}]+ (let [db-id (transforms.i/target-db-id transform)]+ (transforms.util/target-table db-id target)))++;;; -------------------------------------------------- Field Metadata --------------------------------------------------++(mu/defn- get-field-stats :- [:maybe ::transforms-inspector.schema/field-stats]+ "Extract fingerprint stats for a field."+ [field]+ (let [fp (:fingerprint field)]+ (not-empty+ (into {}+ (remove (comp nil? val))+ (cond-> (merge (when-let [dc (get-in fp [:global :distinct-count])]+ {:distinct_count dc})+ (select-keys (get-in fp [:type :type/Number]) [:min :max :avg :q1 :q3])+ (select-keys (get-in fp [:type :type/DateTime]) [:earliest :latest]))+ (some? (get-in fp [:global :nil%]))+ (assoc :nil_percent (get-in fp [:global :nil%])))))))++(mu/defn- collect-field-metadata :- [:sequential ::transforms-inspector.schema/field]+ "Collect metadata for fields in a table."+ [table-id]+ (let [fields (t2/select :model/Field :table_id table-id :active true)]+ (mapv (fn [field]+ (cond-> (select-keys field [:id :name :display_name :base_type :semantic_type])+ (get-field-stats field)+ (assoc :stats (get-field-stats field))))+ fields)))++(mu/defn- build-table-info :- ::transforms-inspector.schema/table+ "Build table info map with fields."+ [{:keys [table-id table-name schema db-id]}]+ (let [fields (collect-field-metadata table-id)]+ {:table_id table-id+ :table_name table-name+ :schema schema+ :db_id db-id+ :column_count (count fields)+ :fields fields}))++;;; -------------------------------------------------- Column Matching --------------------------------------------------++(mr/def ::input-column+ "A matched input column reference."+ [:map+ [:source-table-id {:optional true} [:maybe pos-int?]]+ [:source-table-name {:optional true} [:maybe :string]]+ [:name :string]+ [:id {:optional true} [:maybe pos-int?]]])++(mr/def ::column-match+ "A match between output and input columns."+ [:map+ [:output-field ::transforms-inspector.schema/field]+ [:input-columns [:sequential ::input-column]]])++(defn- normalize-column-name+ "Normalize a column name for comparison."+ [name]+ (some-> name u/lower-case-en (str/replace #"[-_]" "")))++(defn- parse-joined-column-name+ "Parse a joined column name like 'Table__field_name'."+ [name]+ (when name+ (when-let [[_ alias field-name] (re-find #"^(.+)__(.+)$" name)]+ {:alias alias :field-name field-name})))++(defn- match-columns-by-name+ "Find columns that relate between input and output tables using name-based matching.+ Used for native queries where we don't have field IDs."+ [sources target join-structure]+ (let [alias->source-table (into {} (map (juxt :alias :source-table) join-structure))+ source-fields (for [{:keys [table_id table_name fields]} sources+ field fields]+ (assoc field :source-table-id table_id :source-table-name table_name))]+ (for [target-field (:fields target)+ :let [col-name (:name target-field)+ parsed (parse-joined-column-name col-name)+ matching (if-let [src-table (some-> parsed :alias alias->source-table)]+ (filter #(and (= (:source-table-id %) src-table)+ (= (normalize-column-name (:field-name parsed))+ (normalize-column-name (:name %))))+ source-fields)+ (filter #(= (normalize-column-name (or (:field-name parsed) col-name))+ (normalize-column-name (:name %)))+ source-fields))]+ :when (seq matching)]+ {:output-field target-field+ :input-columns (mapv #(select-keys % [:source-table-id :source-table-name :name :id])+ matching)})))++(defn- match-columns-mbql+ "Match columns using lib/returned-columns from the preprocessed MBQL query."+ [preprocessed-query sources target]+ (let [table-id->name (into {} (map (juxt :table_id :table_name) sources))+ target-field-by-name (into {} (map (juxt :name identity) (:fields target)))]+ (for [returned-col (lib/returned-columns preprocessed-query)+ :let [col-name (or (:lib/desired-column-alias returned-col)+ (:name returned-col))+ target-field (target-field-by-name col-name)+ source-table-id (:table-id returned-col)+ source-table-name (table-id->name source-table-id)]+ :when (and target-field source-table-id)]+ {:output-field target-field+ :input-columns [{:source-table-id source-table-id+ :source-table-name source-table-name+ :name (:name returned-col)+ :id (:id returned-col)}]})))++(mu/defn- match-columns :- [:maybe [:sequential ::column-match]]+ "Find columns that relate between input and output tables.+ Uses field ID-based matching for MBQL queries (more accurate),+ falls back to name-based matching for native queries."+ [sources target {:keys [preprocessed-query join-structure]}]+ (if preprocessed-query+ (match-columns-mbql preprocessed-query sources target)+ (match-columns-by-name sources target join-structure)))++;;; -------------------------------------------------- Context Building --------------------------------------------------++(mr/def ::mbql-join-structure-entry+ "A single join in an MBQL query structure."+ [:map+ [:strategy :keyword]+ [:alias [:maybe :string]]+ [:source-table {:optional true} [:maybe pos-int?]]+ [:conditions :any]])++(mr/def ::native-join-structure-entry+ "A single join in a native query structure (HoneySQL forms)."+ [:map+ [:strategy :keyword]+ [:alias [:maybe :string]]+ [:source-table {:optional true} [:maybe pos-int?]]+ [:join-table :any]+ [:join-condition :any]+ [:lhs-column :any]+ [:rhs-column :any]])++(mr/def ::mbql-context+ "MBQL-specific context."+ [:map+ [:preprocessed-query :map]+ [:join-structure [:maybe [:sequential ::mbql-join-structure-entry]]]])++(mr/def ::native-context+ "Native-specific context."+ [:map+ [:from-table :any]+ [:join-structure [:maybe [:sequential ::native-join-structure-entry]]]])++(mr/def ::context+ "Context built for lens discovery and generation."+ [:map+ [:source-type [:enum :mbql :native :python]]+ [:sources [:sequential ::transforms-inspector.schema/table]]+ [:target [:maybe ::transforms-inspector.schema/table]]+ [:db-id pos-int?]+ [:driver :keyword]+ [:from-table-id [:maybe pos-int?]]+ [:has-joins? :boolean]+ [:mbql-context {:optional true} [:maybe ::mbql-context]]+ [:native-context {:optional true} [:maybe ::native-context]]+ [:visited-fields [:maybe ::transforms-inspector.schema/visited-fields]]+ [:has-column-matches? :boolean]+ [:column-matches [:maybe [:sequential ::column-match]]]])++(mu/defn build-context :- ::context+ "Build context for lens discovery and generation."+ [transform]+ (let [source-type (transforms.util/transform-source-type (:source transform))+ sources-info (mapv build-table-info (extract-sources transform))+ target-table (get-target-table transform)+ target-info (when target-table+ (build-table-info (set/rename-keys target-table {:id :table-id :name :table-name :db_id :db-id})))+ query-info (query-analysis/analyze-query transform source-type sources-info)+ join-structure (:join-structure query-info)+ column-matches (when (and (seq sources-info) target-info)+ (seq (match-columns sources-info target-info query-info)))+ db-id (transforms.util/transform-source-database transform)]+ (cond-> {:source-type source-type+ :sources sources-info+ :target target-info+ :db-id db-id+ :driver (or (:driver query-info)+ (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id))+ :from-table-id (:from-table-id query-info)+ :has-joins? (boolean (seq join-structure))+ :visited-fields (:visited-fields query-info)+ :has-column-matches? (boolean column-matches)+ :column-matches column-matches}+ (= source-type :mbql)+ (assoc :mbql-context {:preprocessed-query (:preprocessed-query query-info)+ :join-structure join-structure})+ (= source-type :native)+ (assoc :native-context {:from-table (:from-table query-info)+ :join-structure join-structure}))))diff --git a/src/metabase/transforms_inspector/core.cljc b/src/metabase/transforms_inspector/core.cljcnew file mode 100644index 00000000000..3c382663233--- /dev/null+++ b/src/metabase/transforms_inspector/core.cljc@@ -0,0 +1,112 @@+(ns metabase.transforms-inspector.core+ "Public API for Transform Inspector utilities."+ (:require+ #?@(:clj [[metabase.transforms-inspector.context :as context]+ [metabase.transforms-inspector.lens.column-comparison]+ [metabase.transforms-inspector.lens.core :as lens.core]+ [metabase.transforms-inspector.lens.generic]+ [metabase.transforms-inspector.lens.join-analysis]+ [metabase.transforms-inspector.lens.unmatched-rows]+ [metabase.transforms-inspector.schema]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.malli :as mu]])+ [metabase.transforms-inspector.card-result :as card-result]+ [metabase.transforms-inspector.degeneracy :as degeneracy]+ [metabase.transforms-inspector.interestingness :as interestingness]+ [metabase.transforms-inspector.triggers :as triggers]))++#?(:clj+ (comment+ metabase.transforms-inspector.lens.generic/keep-me+ metabase.transforms-inspector.lens.column-comparison/keep-me+ metabase.transforms-inspector.lens.join-analysis/keep-me+ metabase.transforms-inspector.lens.unmatched-rows/keep-me))++;;; -------------------------------------------------- Card Results --------------------------------------------------++(defn compute-card-result+ "Compute derived fields from first row of query result for a card.+ Dispatches on [lens-id card-type]. Returns a map of field-name -> value, or nil."+ [lens-id card row]+ (card-result/compute-card-result lens-id card row))++;;; -------------------------------------------------- Triggers --------------------------------------------------++(defn triggered-alerts+ "Return alerts whose conditions are met."+ [card-results alert-triggers]+ (filterv #(triggers/evaluate-condition (:condition %) card-results) alert-triggers))++(defn triggered-drill-lenses+ "Return full drill lens trigger objects whose conditions are met."+ [card-results drill-lens-triggers]+ (filterv #(triggers/evaluate-condition (:condition %) card-results) drill-lens-triggers))++(defn evaluate-triggers+ "Evaluate all triggers for a lens against card results.+ Returns {:alerts [...] :drill_lenses [...]}."+ [lens card-results]+ {:alerts (triggered-alerts card-results (:alert_triggers lens))+ :drill_lenses (triggered-drill-lenses card-results (:drill_lens_triggers lens))})++;;; -------------------------------------------------- Interestingness --------------------------------------------------++(defn interesting-fields+ "Filter and sort fields by interestingness.+ Returns fields with score above threshold, sorted by score descending.++ Options:+ - :visited_fields - map of field sets used in query clauses (ignored for now)+ - :threshold - minimum score to include (default 0.3)+ - :limit - maximum number of fields to return (default nil = all)"+ [fields opts]+ (interestingness/interesting-fields fields opts))++;;; -------------------------------------------------- Degeneracy --------------------------------------------------++(defn degenerate?+ "Check if a card result is degenerate and shouldn't be displayed.++ Arguments:+ - card-id: the card's ID string+ - display-type: keyword like :bar, :line, :scalar, etc.+ - card-results: map of all card results (card-id -> result map)++ Returns {:degenerate? bool, :reason keyword-or-nil}."+ [card-id display-type card-results]+ (degeneracy/degenerate-for-display? card-id display-type card-results))++;;; -------------------------------------------------- Lens Discovery API --------------------------------------------------++#?(:clj+ (mu/defn discover-lenses :- :metabase.transforms-inspector.schema/discovery-response+ "Discover available lenses for a transform.+ Returns structural metadata and available lens types.+ This is a cheap operation - no query execution."+ [transform :- :map]+ (let [{:keys [sources target] :as ctx} (context/build-context transform)]+ (if-not target+ {:name (tru "Transform Inspector: {0}" (:name transform))+ :description (tru "Transform has not been run yet.")+ :status :not-run+ :sources sources+ :target nil+ :available_lenses []}+ {:name (tru "Transform Inspector: {0}" (:name transform))+ :description (tru "Analysis of transform inputs, outputs, and joins")+ :status :ready+ :sources sources+ :target target+ :visited_fields (:visited-fields ctx)+ :available_lenses (lens.core/available-lenses ctx)}))))++#?(:clj+ (mu/defn get-lens :- :metabase.transforms-inspector.schema/lens+ "Get full lens contents for a transform.+ Returns sections, cards, and trigger definitions.+ Optional params can filter/customize drill lens output."+ [transform :- :map+ lens-id :- :string+ params :- [:maybe :map]]+ (let [ctx (context/build-context transform)]+ (lens.core/get-lens ctx lens-id params))))diff --git a/src/metabase/transforms_inspector/degeneracy.cljc b/src/metabase/transforms_inspector/degeneracy.cljcnew file mode 100644index 00000000000..ade5b2dfb6c--- /dev/null+++ b/src/metabase/transforms_inspector/degeneracy.cljc@@ -0,0 +1,39 @@+(ns metabase.transforms-inspector.degeneracy+ "Multimethod for detecting degenerate card results.+ A degenerate result is one that doesn't provide useful information+ and should be hidden or deprioritized in the UI.")++;;; -------------------------------------------------- Basic Checks --------------------------------------------------++(defn- no-data?+ "Check if the result has no data at all."+ [card-result]+ (or (get card-result "no_data")+ (= 0 (get card-result "row_count"))))++;;; -------------------------------------------------- Display-specific Checks --------------------------------------------------++(defmulti degenerate-for-display?+ "Check if a card result is degenerate for a given display type.++ Arguments:+ - card-id: the card's ID (for looking up in card-results)+ - display-type: keyword like :bar, :line, etc.+ - card-results: map of all card results (card-id -> result map)++ Returns {:degenerate? bool :reason keyword}."+ {:arglists '([card-id display-type card-results])}+ (fn [_card-id display-type _card-results] display-type))++(defmethod degenerate-for-display? :default+ [card-id _display-type card-results]+ (cond+ (no-data? (get card-results card-id))+ {:degenerate? true :reason :no-data}++ :else+ {:degenerate? false}))++(defmethod degenerate-for-display? :hidden+ [_card-id _display-type _card-results]+ {:degenerate? true})diff --git a/src/metabase/transforms_inspector/interestingness.cljc b/src/metabase/transforms_inspector/interestingness.cljcnew file mode 100644index 00000000000..4169498d6a1--- /dev/null+++ b/src/metabase/transforms_inspector/interestingness.cljc@@ -0,0 +1,145 @@+(ns metabase.transforms-inspector.interestingness+ "Column scoring for visualization worthiness.+ Determines which columns are interesting to display in inspector cards.++ This is a cljc namespace so it can be used by both:+ - Backend: to score columns when generating cards+ - Frontend: to filter/sort cards client-side++ Scoring criteria:+ - Dominated columns (PK, FK, UUID) get score 0.0+ - High cardinality columns (>1000 distinct values) get low scores+ - Mostly null columns (>90% null) get low scores+ - Temporal, categorical, and numeric columns with variance get high scores")++(def ^:private dominated-semantic-types+ "Semantic types that are never interesting to visualize."+ #{:type/PK :type/FK :type/UUID :type/SerializedJSON})++(def ^:private high-interest-semantic-types+ "Semantic types that are typically interesting to visualize."+ #{:type/Category :type/State :type/Country :type/City+ :type/CreationTimestamp :type/UpdatedTimestamp+ :type/CreationDate :type/UpdatedDate+ :type/Price :type/Cost :type/Quantity :type/Score+ :type/Percentage :type/Duration})++(def ^:private temporal-base-types+ "Base types that represent temporal data."+ #{:type/DateTime :type/Date :type/Time+ :type/DateTimeWithTZ :type/DateTimeWithLocalTZ+ :type/DateTimeWithZoneID :type/DateTimeWithZoneOffset})++(defn dominated-column?+ "Returns true if the column should never be visualized.+ These are structural columns (PKs, FKs, UUIDs) that don't provide+ meaningful distribution insights."+ [field]+ (let [semantic-type (:semantic_type field)+ field-name (some-> (:name field) str)]+ (boolean+ (or+ ;; Explicit semantic type indicates dominated column+ (contains? dominated-semantic-types semantic-type)+ ;; Name pattern matching for common ID columns+ (and field-name+ (re-matches #"(?i).*_id$|^id$|.*_uuid$|^uuid$" field-name))))))++(defn- high-cardinality?+ "Returns true if the column has very high cardinality (>1000 distinct values).+ High cardinality columns don't make good bar/pie charts."+ [field]+ (let [distinct-count (get-in field [:stats :distinct_count])]+ (and distinct-count (> distinct-count 1000))))++(defn- mostly-null?+ "Returns true if the column is mostly null (>90% null)."+ [field]+ (let [nil-percent (get-in field [:stats :nil_percent])]+ (and nil-percent (> nil-percent 0.9))))++(defn- temporal-column?+ "Returns true if the column represents temporal data."+ [field]+ (or (contains? temporal-base-types (:base_type field))+ (contains? #{:type/CreationTimestamp :type/UpdatedTimestamp+ :type/CreationDate :type/UpdatedDate}+ (:semantic_type field))))++(defn- categorical-column?+ "Returns true if the column appears to be categorical (low cardinality text/enum)."+ [field]+ (let [distinct-count (get-in field [:stats :distinct_count])+ base-type (:base_type field)]+ (and distinct-count+ (<= distinct-count 20)+ (contains? #{:type/Text :type/TextLike} base-type))))++(defn- numeric-with-variance?+ "Returns true if the column is numeric and has meaningful variance."+ [field]+ (let [base-type (:base_type field)+ min-val (get-in field [:stats :min])+ max-val (get-in field [:stats :max])]+ (and (contains? #{:type/Integer :type/Float :type/Decimal :type/Number} base-type)+ min-val max-val+ (not= min-val max-val))))++(defn score-field+ "Score a field from 0.0 to 1.0 for visualization worthiness.+ Higher scores indicate more interesting columns.++ Returns a map with:+ - :score - numeric score 0.0-1.0+ - :dominated? - true if column should never be shown+ - :reasons - vector of keywords explaining the score"+ [field]+ (cond+ ;; Dominated columns - never show+ (dominated-column? field)+ {:score 0.0 :dominated? true :reasons [:dominated-column]}++ ;; Mostly null - rarely useful+ (mostly-null? field)+ {:score 0.1 :dominated? false :reasons [:mostly-null]}++ ;; High cardinality without semantic type - poor for visualization+ (and (high-cardinality? field)+ (not (contains? high-interest-semantic-types (:semantic_type field))))+ {:score 0.2 :dominated? false :reasons [:high-cardinality]}++ :else+ (let [reasons (cond-> []+ (temporal-column? field) (conj :temporal)+ (categorical-column? field) (conj :categorical)+ (numeric-with-variance? field) (conj :numeric-variance)+ (contains? high-interest-semantic-types+ (:semantic_type field)) (conj :high-interest-semantic-type))+ base-score (cond+ (temporal-column? field) 0.9+ (contains? high-interest-semantic-types+ (:semantic_type field)) 0.85+ (categorical-column? field) 0.8+ (numeric-with-variance? field) 0.75+ :else 0.5)]+ {:score base-score+ :dominated? false+ :reasons (if (seq reasons) reasons [:default])})))++(defn interesting-fields+ "Filter and sort fields by interestingness.+ Returns fields with score above threshold, sorted by score descending.++ Options:+ - :visited_fields - map of field sets used in query clauses (ignored for now)+ - :threshold - minimum score to include (default 0.3)+ - :limit - maximum number of fields to return (default nil = all)"+ [fields {:keys [#_visited_fields threshold limit] :or {threshold 0.3}}]+ (let [scored (->> fields+ (map #(assoc % :interestingness (score-field %)))+ (remove #(get-in % [:interestingness :dominated?]))+ (filter #(> (get-in % [:interestingness :score]) threshold))+ (sort-by #(get-in % [:interestingness :score]) >))]+ (if limit+ (take limit scored)+ scored)))diff --git a/src/metabase/transforms_inspector/js.cljs b/src/metabase/transforms_inspector/js.cljsnew file mode 100644index 00000000000..095f663d0c9--- /dev/null+++ b/src/metabase/transforms_inspector/js.cljs@@ -0,0 +1,51 @@+(ns metabase.transforms-inspector.js+ "Javascript-facing interface for transform inspector. Wraps functions in metabase.transforms-inspector.core"+ (:require+ [metabase.transforms-inspector.core :as inspector]))++(def ^:private empty-triggers #js {:alerts #js [] :drill_lenses #js []})++(defn ^:export evaluateTriggers+ "Evaluate all triggers. Returns {alerts: [...], drill_lenses: [...]}."+ [lens-js card-results-js]+ (if (and (empty? (unchecked-get lens-js "alert_triggers"))+ (empty? (unchecked-get lens-js "drill_lens_triggers")))+ empty-triggers+ (let [lens (-> (js->clj lens-js :keywordize-keys true)+ (update :alert_triggers #(mapv (fn [t] (update t :condition (fn [c] (update c :name keyword)))) %))+ (update :drill_lens_triggers #(mapv (fn [t] (update t :condition (fn [c] (update c :name keyword)))) %)))+ card-results (js->clj card-results-js)+ result (inspector/evaluate-triggers lens card-results)]+ (clj->js result))))++(defn ^:export computeCardResult+ "Compute derived fields from first row of query result for a card.+ Returns a JS object with computed fields, or null if no computation needed."+ [lens-id card-js rows-js]+ (let [lens-kw (keyword lens-id)+ card (js->clj card-js :keywordize-keys true)+ row (js->clj (aget rows-js 0))]+ (when-let [result (inspector/compute-card-result lens-kw card row)]+ (clj->js result))))++(defn ^:export interestingFields+ "Filter and sort fields by interestingness.+ Returns fields with score above threshold, sorted by score descending.+ Options: visited_fields (ignored for now), threshold (default 0.3), limit (default nil = all)."+ [fields-js visited-fields-js threshold limit]+ (let [fields (js->clj fields-js :keywordize-keys true)+ visited-fields (js->clj visited-fields-js :keywordize-keys true)]+ (clj->js (inspector/interesting-fields fields+ {:visited_fields visited-fields+ :threshold (or threshold 0.3)+ :limit limit}))))++(defn ^:export isDegenerate+ "Check if a card result is degenerate and shouldn't be displayed.+ Returns {degenerate: bool, reason: string|null}."+ [card-id display-type card-results-js]+ (let [card-results (js->clj card-results-js)+ display-kw (keyword display-type)+ result (inspector/degenerate? card-id display-kw card-results)]+ (clj->js {:degenerate (:degenerate? result)+ :reason (some-> (:reason result) name)})))diff --git a/src/metabase/transforms_inspector/lens/column_comparison.clj b/src/metabase/transforms_inspector/lens/column_comparison.cljnew file mode 100644index 00000000000..a695816870f--- /dev/null+++ b/src/metabase/transforms_inspector/lens/column_comparison.clj@@ -0,0 +1,116 @@+(ns metabase.transforms-inspector.lens.column-comparison+ "Column Comparison lens - compare distributions for matched columns.++ This lens is applicable when there are matched columns between+ input and output tables. It shows distribution comparisons grouped+ by output column.++ Layout: :comparison"+ (:require+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.transforms-inspector.lens.core :as lens.core]+ [metabase.util.i18n :refer [tru trun]]))++(set! *warn-on-reflection* true)++(lens.core/register-lens! :column-comparison 20)++;;; -------------------------------------------------- Query Building --------------------------------------------------++(defn- make-distribution-query+ "Generate a pMBQL query for field distribution."+ [db-id table-id field-id]+ (let [mp (lib-be/application-database-metadata-provider db-id)+ table-metadata (lib.metadata/table mp table-id)+ field-metadata (lib.metadata/field mp field-id)]+ (-> (lib/query mp table-metadata)+ (lib/aggregate (lib/count))+ (lib/breakout (lib/ref field-metadata)))))++;;; -------------------------------------------------- Display Type Selection --------------------------------------------------++(defn- viz-type-for-field+ "Determine appropriate visualization type for a field."+ [field]+ (let [base-type (:base_type field)+ distinct-count (get-in field [:stats :distinct_count])]+ (cond+ (contains? #{:type/DateTime :type/Date :type/Time+ :type/DateTimeWithTZ :type/DateTimeWithLocalTZ}+ base-type)+ :line++ (= base-type :type/Boolean)+ :pie++ (and distinct-count (<= distinct-count 5))+ :pie++ (and distinct-count (<= distinct-count 20))+ :row++ :else+ :bar)))++;;; -------------------------------------------------- Card Generation --------------------------------------------------++(defn- distribution-card+ "Generate a distribution card for a field."+ [db-id table-id table-name field output-column role order params]+ {:id (lens.core/make-card-id (str table-name "-" (:name field) "-distribution") params)+ :section_id "comparisons"+ :title (tru "{0} ({1})" (:name field) table-name)+ :display (viz-type-for-field field)+ :dataset_query (make-distribution-query db-id table-id (:id field))+ :metadata {:group_id output-column+ :group_role role+ :group_order order+ :table_id table-id+ :field_id (:id field)}})++(defn- comparison-cards-for-match+ "Generate comparison cards for a single column match."+ [{:keys [output-field input-columns]} target params]+ (let [valid-inputs (filter :source-table-id input-columns)+ output-column (:name output-field)]+ (when (seq valid-inputs)+ (concat+ ;; Input cards+ (map-indexed (fn [i {:keys [source-table-id source-table-name id] col-name :name}]+ (distribution-card (:db_id target)+ source-table-id source-table-name+ {:id id :name col-name+ :base_type (:base_type output-field)+ :stats (:stats output-field)}+ output-column :input i params))+ valid-inputs)+ ;; Output card+ [(distribution-card (:db_id target) (:table_id target) (:table_name target)+ output-field output-column :output 0 params)]))))++;;; -------------------------------------------------- Lens Implementation --------------------------------------------------++(defmethod lens.core/lens-applicable? :column-comparison+ [_ ctx]+ (:has-column-matches? ctx))++(defmethod lens.core/lens-metadata :column-comparison+ [_ _ctx]+ {:id "column-comparison"+ :display_name (tru "Column Distributions")+ :description (tru "Compare input/output column distributions")+ :complexity {:level :slow}})++(defmethod lens.core/make-lens :column-comparison+ [lens-type ctx params]+ (let [{:keys [column-matches target]} ctx+ match-count (count column-matches)]+ (lens.core/with-metadata lens-type ctx+ {:summary {:text (tru "Compare value distributions for columns that match between input and output")+ :highlights [{:label (tru "Matched Columns") :value match-count}]}+ :sections [{:id "comparisons"+ :title (trun "{0} matched column" "{0} matched columns" match-count)+ :layout :comparison}]+ :cards (vec (mapcat #(comparison-cards-for-match % target params) column-matches))})))diff --git a/src/metabase/transforms_inspector/lens/core.clj b/src/metabase/transforms_inspector/lens/core.cljnew file mode 100644index 00000000000..3bde1dfe517--- /dev/null+++ b/src/metabase/transforms_inspector/lens/core.clj@@ -0,0 +1,174 @@+(ns metabase.transforms-inspector.lens.core+ "Lens orchestration for Transform Inspector.++ A lens is a 'view' into the transform data with:+ - Metadata (id, display-name, description)+ - Sections grouping cards+ - Summary with highlights+ - Cards (generated queries)+ - Alert and drill-lens triggers++ Each lens type is implemented via multimethods. Adding a new lens+ requires implementing: lens-applicable?, lens-metadata, make-lens."+ (:require+ [clojure.string :as str]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Multimethods --------------------------------------------------++(defmulti lens-applicable?+ "Check if a lens applies to the given context.+ Should be a cheap check - no query execution.++ Arguments:+ - lens-type: keyword like :generic-summary, :join-analysis+ - ctx: context map with :has-joins?, :column-matches, etc.++ Returns true if the lens can be generated for this transform."+ {:arglists '([lens-type ctx])}+ (fn [lens-type _ctx] lens-type))++(defmethod lens-applicable? :default+ [_lens-type _ctx]+ false)++(defmulti lens-metadata+ "Return lens metadata for discovery (Phase 1).+ This is the minimal info shown in the lens selector.++ Returns:+ {:id \"lens-id\"+ :display_name \"Human Name\"+ :description \"Optional description\"}"+ {:arglists '([lens-type ctx])}+ (fn [lens-type _ctx] lens-type))++(defmethod lens-metadata :default+ [lens-type _ctx]+ {:id (name lens-type)+ :display_name (str lens-type)+ :description nil})++(defmulti make-lens+ "Generate full lens contents (Phase 2).++ Arguments:+ - lens-type: keyword like :generic-summary, :join-analysis+ - ctx: context map with sources, target, joins, etc.+ - params: optional map with drill lens parameters (e.g., {:join_step 1})++ Returns a complete lens map with sections, cards, triggers, etc.++ Returns:+ {:id \"lens-id\"+ :display_name \"Human Name\"+ :summary {:text \"...\" :highlights [...] :alerts []}+ :sections [{:id :title :description :layout} ...]+ :cards [{:id :title :display :dataset_query :metadata {...}} ...]+ :drill_lenses [{:id :display_name :description} ...]+ :alert_triggers [...]+ :drill_lens_triggers [...]}"+ {:arglists '([lens-type ctx params])}+ (fn [lens-type _ctx _params] lens-type))++(defmethod make-lens :default+ [lens-type _ctx _params]+ (log/warnf "No lens implementation for type: %s" lens-type)+ {:id (name lens-type)+ :display_name (str lens-type)+ :summary {:text nil :highlights [] :alerts []}+ :sections []+ :cards []})++(defn with-metadata+ "Merge lens-metadata into a lens map. Use this in make-lens implementations+ to avoid duplicating id, display_name, description, and complexity."+ [lens-type ctx lens-map]+ (merge (lens-metadata lens-type ctx) lens-map))++;;; -------------------------------------------------- Lens Registry --------------------------------------------------++(defonce ^:private ^{:doc "Atom containing registered lens types.+ Each entry is {:lens-type keyword :priority number :drill? boolean}.+ Lower priority = shown first in discovery."} registry+ (atom []))++(defn register-lens!+ "Register a lens type. Call this at namespace load time.+ Priority controls ordering - lower numbers appear first.+ Set drill? true for lenses only available via triggers."+ ([lens-type priority]+ (register-lens! lens-type priority false))+ ([lens-type priority drill?]+ (swap! registry (fn [lenses]+ (-> (remove #(= (:lens-type %) lens-type) lenses)+ (concat [{:lens-type lens-type :priority priority :drill? drill?}])+ vec)))+ nil))++(defn- registered-lens-types+ "Return registered lens types in priority order, optionally filtering drill lenses."+ ([]+ (registered-lens-types false))+ ([include-drill?]+ (->> @registry+ (filter #(or include-drill? (not (:drill? %))))+ (sort-by :priority)+ (mapv :lens-type))))++;;; -------------------------------------------------- Public API --------------------------------------------------++(defn available-lenses+ "Return metadata for applicable lenses (Phase 1).+ Filters to only lenses that apply to the given context.+ Order is by priority - first lens is the default."+ [ctx]+ (->> (registered-lens-types)+ (filter #(lens-applicable? % ctx))+ (mapv #(lens-metadata % ctx))))++(defn lens-id->type+ "Map lens ID string to lens type keyword."+ [lens-id]+ (keyword lens-id))++(defn- filter-applicable-drill-triggers+ "Filter drill_lens_triggers to only include sublenses that are applicable to ctx."+ [ctx drill-lens-triggers]+ (filterv #(lens-applicable? (lens-id->type (:lens_id %)) ctx) drill-lens-triggers))++(defn params->id-suffix+ "Generate a suffix for card IDs based on params to ensure uniqueness across+ different parameterizations of the same lens. Returns empty string if no params."+ [params]+ (if (seq params)+ (str "@" (->> params+ (sort-by key)+ (map (fn [[k v]] (str (name k) "=" v)))+ (str/join ",")))+ ""))++(defn make-card-id+ "Create a card ID with optional params suffix for uniqueness in parametric lenses.+ Use this everywhere you create or reference card IDs in parametric lenses."+ [base-id params]+ (str base-id (params->id-suffix params)))++(defn get-lens+ "Generate a lens by ID (Phase 2).+ Returns the full lens with sections, cards, and triggers.+ Filters drill_lens_triggers to only include applicable sublenses.+ Optional params can filter/customize drill lens output."+ ([ctx lens-id]+ (get-lens ctx lens-id nil))+ ([ctx lens-id params]+ (let [lens-type (lens-id->type lens-id)]+ (if (lens-applicable? lens-type ctx)+ (-> (make-lens lens-type ctx params)+ (update :drill_lens_triggers #(filter-applicable-drill-triggers ctx %)))+ (throw (ex-info "Lens data not available"+ {:lens_id lens-id+ :lens_type lens-type+ :status-code 404}))))))diff --git a/src/metabase/transforms_inspector/lens/generic.clj b/src/metabase/transforms_inspector/lens/generic.cljnew file mode 100644index 00000000000..5f1c833af78--- /dev/null+++ b/src/metabase/transforms_inspector/lens/generic.clj@@ -0,0 +1,78 @@+(ns metabase.transforms-inspector.lens.generic+ "Generic Summary lens - basic overview for any transform.++ This lens is always applicable and provides:+ - Row count comparison (inputs vs output)++ Layout: :comparison"+ (:require+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.transforms-inspector.lens.core :as lens.core]+ [metabase.util.i18n :refer [tru]]))++(set! *warn-on-reflection* true)++(lens.core/register-lens! :generic-summary 0)++;;; -------------------------------------------------- Query Building --------------------------------------------------++(defn- make-count-query+ "Generate a pMBQL COUNT(*) query."+ [db-id table-id]+ (let [mp (lib-be/application-database-metadata-provider db-id)+ table-metadata (lib.metadata/table mp table-id)]+ (-> (lib/query mp table-metadata)+ (lib/aggregate (lib/count)))))++;;; -------------------------------------------------- Card Generation --------------------------------------------------++(defn- row-count-card+ "Generate a row count card."+ [db-id table-id table-name role order params]+ {:id (lens.core/make-card-id (str table-name "-row-count") params)+ :section_id "row-counts"+ :title (tru "{0} Row Count" table-name)+ :display :scalar+ :dataset_query (make-count-query db-id table-id)+ :metadata {:dedup_key [:table_count table-id]+ :group_id "row-count"+ :group_role role+ :group_order order+ :table_id table-id}})++;;; -------------------------------------------------- Lens Implementation --------------------------------------------------++(defmethod lens.core/lens-applicable? :generic-summary+ [_ _ctx]+ true)++(defmethod lens.core/lens-metadata :generic-summary+ [_ _ctx]+ {:id "generic-summary"+ :display_name (tru "Data Summary")+ :description (tru "Overview of input and output tables")+ :complexity {:level :fast}})++(defmethod lens.core/make-lens :generic-summary+ [lens-type ctx params]+ (let [{:keys [sources target]} ctx]+ (lens.core/with-metadata lens-type ctx+ {:summary {:text (tru "Compare row counts before and after transformation")+ :highlights (cond-> [{:label (tru "Input Tables") :value (count sources)}]+ target+ (conj {:label (tru "Output Columns") :value (:column_count target)}))}+ :sections [{:id "row-counts"+ :title (tru "Row Counts")+ :layout :comparison}]+ :cards (vec+ (concat+ ;; Input row counts+ (map-indexed (fn [i {:keys [table_id table_name db_id]}]+ (row-count-card db_id table_id table_name :input i params))+ sources)+ ;; Output row count+ (when target+ [(row-count-card (:db_id target) (:table_id target)+ (:table_name target) :output 0 params)])))})))diff --git a/src/metabase/transforms_inspector/lens/join_analysis.clj b/src/metabase/transforms_inspector/lens/join_analysis.cljnew file mode 100644index 00000000000..3c6032b4ea3--- /dev/null+++ b/src/metabase/transforms_inspector/lens/join_analysis.clj@@ -0,0 +1,236 @@+(ns metabase.transforms-inspector.lens.join-analysis+ "Join Analysis lens - analyze join quality and data flow.++ Cards returned:+ - base-count: COUNT(*) with 0 joins+ - join-step-N: [COUNT(*), COUNT(nullable_field)] for each join step+ - table-N-count: COUNT(*) for each joined table (right-row-count)++ FE derives from card results:+ - null-count = output-count - matched-count+ - left-row-count = previous step's output-count+ - match-rate = matched-count / left-row-count++ Layout: :flat (FE renders as table based on lens type)"+ (:require+ [clojure.string :as str]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.transforms-inspector.lens.core :as lens.core]+ [metabase.transforms-inspector.lens.query-util :as query-util]+ [metabase.util.i18n :refer [tru trun]]))++(set! *warn-on-reflection* true)++(lens.core/register-lens! :join-analysis 10)++(defn- get-join-structure+ "Get join-structure from whichever sub-context is present."+ [ctx]+ (:join-structure (or (:mbql-context ctx) (:native-context ctx))))++;;; -------------------------------------------------- MBQL Query Building --------------------------------------------------++(defn- make-count-query+ [query]+ (lib/aggregate query (lib/count)))++(defn- nullable-side-field-info+ "For outer joins, return the field info for the side that becomes NULL on non-match.+ LEFT → RHS is nullable, RIGHT → LHS is nullable, FULL → RHS (detects left-unmatched)."+ [join]+ (let [strategy (or (:strategy join) :left-join)]+ (case strategy+ (:left-join :full-join) (query-util/get-rhs-field-info (:conditions join))+ :right-join (query-util/get-lhs-field-info (:conditions join))+ nil)))++(defn- make-join-step-query-mbql+ "Query returning [COUNT(*), COUNT(nullable_field)] for outer joins, [COUNT(*)] otherwise."+ [query step join]+ (let [field-info (nullable-side-field-info join)+ step-query (-> query (query-util/bare-query-with-n-joins step) make-count-query)]+ (if field-info+ (let [mp (lib-be/application-database-metadata-provider (:database query))+ field-meta (-> (lib.metadata/field mp (:field-id field-info))+ (lib/with-join-alias (:join-alias field-info)))]+ (lib/aggregate step-query (lib/count field-meta)))+ step-query)))++(defn- make-table-count-query+ [db-id table-id]+ (let [mp (lib-be/application-database-metadata-provider db-id)+ table-metadata (lib.metadata/table mp table-id)]+ (-> (lib/query mp table-metadata)+ (lib/aggregate (lib/count)))))++;;; -------------------------------------------------- Native SQL Building --------------------------------------------------++(defn- nullable-side-column+ "For outer joins, return the HoneySQL column ref for the side that becomes NULL on non-match.+ LEFT/FULL → rhs-column, RIGHT → lhs-column."+ [{:keys [strategy rhs-column lhs-column]}]+ (case strategy+ (:left-join :full-join) rhs-column+ :right-join lhs-column+ nil))++(def ^:private strategy->hsql-join-type+ "Map our join strategy keywords to HoneySQL :join-by type keywords.+ HoneySQL uses :join for INNER JOIN, not :inner-join."+ {:inner-join :join+ :left-join :left-join+ :right-join :right-join+ :full-join :full-join+ :cross-join :cross-join})++(defn- build-native-join-step-hsql+ "Build a HoneySQL map for [COUNT(*), COUNT(nullable_field)] with joins.+ Uses :join-by to preserve original join ordering across different join types."+ [from-table joins]+ (let [col (nullable-side-column (last joins))+ join-by (into [] (mapcat (fn [{:keys [strategy join-table join-condition]}]+ [(strategy->hsql-join-type strategy) [join-table join-condition]]))+ joins)]+ {:select (if col+ [[[:count :*]] [[:count col]]]+ [[[:count :*]]])+ :from [from-table]+ :join-by join-by}))++(defn- format-native-query+ "Format a HoneySQL map into a native query for the given driver and db-id."+ [driver db-id hsql]+ (let [mp (lib-be/application-database-metadata-provider db-id)]+ (lib/native-query mp (first (sql.qp/format-honeysql driver hsql)))))++;;; -------------------------------------------------- Card Generation --------------------------------------------------++(defn- base-count-card+ [ctx params]+ (let [{:keys [source-type from-table-id driver db-id]} ctx+ source-table-id from-table-id]+ {:id (lens.core/make-card-id "base-count" params)+ :section_id "join-stats"+ :title (tru "Base Row Count")+ :display :scalar+ :dataset_query+ (case source-type+ :mbql (-> (:preprocessed-query (:mbql-context ctx)) (query-util/bare-query-with-n-joins 0) make-count-query)+ :native (format-native-query driver db-id+ {:select [[[:count :*]]]+ :from [(:from-table (:native-context ctx))]}))+ :metadata {:dedup_key [:table_count source-table-id]+ :card_type :base_count}}))++(defn- join-step-card+ [ctx step params]+ (let [{:keys [source-type driver db-id]} ctx+ join-structure (get-join-structure ctx)+ join (nth join-structure (dec step))+ {:keys [strategy alias]} join]+ {:id (lens.core/make-card-id (str "join-step-" step) params)+ :section_id "join-stats"+ :title (tru "Join {0}: {1}" step alias)+ :display :table+ :dataset_query+ (case source-type+ :mbql (let [preprocessed-query (:preprocessed-query (:mbql-context ctx))]+ (make-join-step-query-mbql preprocessed-query step+ (nth (lib/joins preprocessed-query 0) (dec step))))+ :native (format-native-query driver db-id+ (build-native-join-step-hsql (:from-table (:native-context ctx))+ (take step join-structure))))+ :metadata {:card_type :join_step+ :join_step step+ :join_alias alias+ :join_strategy strategy}}))++(defn- table-count-card+ [ctx step params]+ (let [{:keys [sources db-id]} ctx+ join-structure (get-join-structure ctx)+ join (nth join-structure (dec step))+ table-id (:source-table join)+ table (some #(when (= (:table_id %) table-id) %) sources)]+ (when table+ {:id (lens.core/make-card-id (str "table-" step "-count") params)+ :section_id "join-stats"+ :title (tru "{0} Row Count" (:table_name table))+ :display :scalar+ :dataset_query (make-table-count-query (or (:db_id table) db-id) table-id)+ :metadata {:dedup_key [:table_count table-id]+ :card_type :table_count+ :join_step step+ :table_id table-id}})))++(defn- all-cards+ [ctx params]+ (let [join-count (count (get-join-structure ctx))]+ (into [(base-count-card ctx params)]+ (mapcat (fn [step]+ (let [step-card (join-step-card ctx step params)+ table-card (table-count-card ctx step params)]+ (if table-card+ [step-card table-card]+ [step-card])))+ (range 1 (inc join-count))))))++;;; -------------------------------------------------- Lens Implementation --------------------------------------------------++(defmethod lens.core/lens-applicable? :join-analysis+ [_ ctx]+ (:has-joins? ctx))++(defmethod lens.core/lens-metadata :join-analysis+ [_ _ctx]+ {:id "join-analysis"+ :display_name (tru "Join Analysis")+ :description (tru "Analyze join quality and match rates")+ :complexity {:level :very-slow}})++(defn- make-triggers+ "Generate alert and drill-lens triggers for join steps."+ [join-structure params]+ (let [outer-joins (filter #(contains? #{:left-join :right-join :full-join}+ (:strategy %))+ (map-indexed #(assoc %2 :step (inc %1)) join-structure))]+ {:alert_triggers+ (for [{:keys [step alias strategy]} outer-joins]+ {:id (str "high-null-rate-" step)+ :condition {:name :high-null-rate+ :card_id (lens.core/make-card-id (str "join-step-" step) params)}+ :severity :warning+ :message (tru "Join ''{0}'' has >20% unmatched rows" alias)+ :metadata {:join_step step+ :join_alias alias+ :join_strategy strategy}})++ :drill_lens_triggers+ (for [{:keys [step alias strategy]} outer-joins]+ {:lens_id "unmatched-rows"+ :condition {:name :has-unmatched-rows+ :card_id (lens.core/make-card-id (str "join-step-" step) params)}+ :params {:join_step step}+ :reason (tru "Unmatched rows in {0}" alias)+ :metadata {:join_step step+ :join_alias alias+ :join_strategy strategy}})}))++(defmethod lens.core/make-lens :join-analysis+ [lens-type ctx params]+ (let [join-structure (get-join-structure ctx)+ join-count (count join-structure)+ strategies (distinct (map :strategy join-structure))+ triggers (make-triggers join-structure params)]+ (lens.core/with-metadata lens-type ctx+ {:summary {:text (str (trun "{0} join" "{0} joins" join-count) ": " (str/join ", " (map name strategies)))+ :highlights [{:label (tru "Joins") :value join-count}]}+ :sections [{:id "join-stats"+ :title (tru "Join Statistics")+ :layout :flat}]+ :cards (all-cards ctx params)+ :alert_triggers (vec (:alert_triggers triggers))+ :drill_lens_triggers (vec (:drill_lens_triggers triggers))})))diff --git a/src/metabase/transforms_inspector/lens/query_util.clj b/src/metabase/transforms_inspector/lens/query_util.cljnew file mode 100644index 00000000000..f6d525e8423--- /dev/null+++ b/src/metabase/transforms_inspector/lens/query_util.clj@@ -0,0 +1,44 @@+(ns metabase.transforms-inspector.lens.query-util+ "Shared utilities for manipulating preprocessed pMBQL queries in lens code."+ (:require+ [metabase.lib.core :as lib]))++(set! *warn-on-reflection* true)++(defn bare-query-with-n-joins+ "Stripped-down copy of `query` retaining only `:source-table` and the first `n` joins.+ All other stage keys (`:fields`, `:filters`, `:breakout`, etc.) are removed."+ [query n]+ (lib/update-query-stage query 0+ (fn [stage]+ (cond-> (select-keys stage [:lib/type :source-table :joins])+ (zero? n) (dissoc :joins)+ (pos? n) (update :joins #(vec (take n %)))))))++(defn extract-field-info+ "Extract `{:field-id ... :join-alias ...}` from a `:field` ref clause.+ `:join-alias` will be nil for base-table fields."+ [[tag opts id-or-name :as field-ref]]+ (when (and (vector? field-ref)+ (= :field tag)+ (map? opts))+ {:field-id id-or-name+ :join-alias (:join-alias opts)}))++(defn get-rhs-field-info+ "Field info for the RHS (joined-table side) of a single-condition join.+ Returns nil for multi-condition joins (we don't try to analyze compound conditions)."+ [conditions]+ (when (= 1 (count conditions))+ (when-let [[_op _opts _lhs rhs] (first conditions)]+ (let [info (extract-field-info rhs)]+ (when (:join-alias info)+ info)))))++(defn get-lhs-field-info+ "Field info for the LHS (base or previously-joined side) of a single-condition join.+ Returns nil for multi-condition joins (we don't try to analyze compound conditions)."+ [conditions]+ (when (= 1 (count conditions))+ (when-let [[_op _opts lhs _rhs] (first conditions)]+ (extract-field-info lhs))))diff --git a/src/metabase/transforms_inspector/lens/unmatched_rows.clj b/src/metabase/transforms_inspector/lens/unmatched_rows.cljnew file mode 100644index 00000000000..1e0d051b4d7--- /dev/null+++ b/src/metabase/transforms_inspector/lens/unmatched_rows.clj@@ -0,0 +1,272 @@+(ns metabase.transforms-inspector.lens.unmatched-rows+ "Drill-down lens triggered from join-analysis when null counts are significant.++ For each outer join, produces up to four sample cards:++ 1. Truly unmatched -- LHS key IS NOT NULL, RHS key IS NULL+ 2. Null source key -- LHS key IS NULL (can never match)+ 3. Orphan RHS -- RHS rows whose key doesn't exist in LHS (LEFT/FULL only)+ 4. Orphan LHS -- LHS rows whose key doesn't exist in RHS (RIGHT/FULL only)++ Trigger threshold: >5% null rate. Alert threshold: >20%.+ Accepts `{:join_step N}` params to scope to a single join."+ (:require+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.transforms-inspector.lens.core :as lens.core]+ [metabase.transforms-inspector.lens.query-util :as query-util]+ [metabase.util.i18n :refer [tru trun]]))++(set! *warn-on-reflection* true)++(lens.core/register-lens! :unmatched-rows 100 true)++;;; -------------------------------------------------- Query Building --------------------------------------------------++(defn- field-meta+ "Field metadata for `field-id`, optionally scoped to `join-alias`."+ [mp field-id join-alias]+ (cond-> (lib.metadata/field mp field-id)+ join-alias (lib/with-join-alias join-alias)))++(defn- get-table-field-metas+ "All column metadata for `table-id`, optionally scoped to `join-alias`."+ [mp table-id join-alias]+ (let [fields (lib.metadata/fields mp table-id)]+ (mapv (fn [f]+ (cond-> f+ join-alias (lib/with-join-alias join-alias)))+ fields)))++(defn- find-table-for-join-alias+ "Look up the `:source-table` for the join with the given `alias`."+ [join-structure alias]+ (some #(when (= (:alias %) alias) (:source-table %)) join-structure))++(defn- resolve-join-sides+ "Resolve LHS/RHS field and table info for join `step`. Returns nil when the join condition can't be parsed."+ [ctx step]+ (let [{:keys [from-table-id]} ctx+ {:keys [join-structure]} (:mbql-context ctx)+ join-entry (nth join-structure (dec step))+ rhs-info (query-util/get-rhs-field-info (:conditions join-entry))+ lhs-info (query-util/get-lhs-field-info (:conditions join-entry))+ rhs-table-id (:source-table join-entry)+ lhs-table-id (if (:join-alias lhs-info)+ (find-table-for-join-alias join-structure (:join-alias lhs-info))+ from-table-id)]+ (when (and rhs-info lhs-info rhs-table-id lhs-table-id)+ {:lhs-table-id lhs-table-id+ :rhs-table-id rhs-table-id+ :lhs-field-id (:field-id lhs-info)+ :rhs-field-id (:field-id rhs-info)+ :lhs-join-alias (:join-alias lhs-info)+ :rhs-join-alias (:join-alias rhs-info)})))++(defn- make-base-unmatched-query+ "Stripped-down query with joins up to `step`, plus field metadata for both sides.+ Returns nil when the join condition can't be parsed."+ [ctx step]+ (when-let [{:keys [lhs-table-id lhs-field-id rhs-field-id+ lhs-join-alias rhs-join-alias]} (resolve-join-sides ctx step)]+ (let [{:keys [from-table-id db-id]} ctx+ {:keys [preprocessed-query]} (:mbql-context ctx)+ mp (lib-be/application-database-metadata-provider db-id)+ base-table-id from-table-id+ lhs-field-metas (get-table-field-metas mp lhs-table-id lhs-join-alias)+ base-field-metas (get-table-field-metas mp base-table-id nil)]+ (when (seq lhs-field-metas)+ {:base-query (query-util/bare-query-with-n-joins preprocessed-query step)+ :lhs-field-meta (field-meta mp lhs-field-id lhs-join-alias)+ :rhs-field-meta (field-meta mp rhs-field-id rhs-join-alias)+ :lhs-field-metas lhs-field-metas+ :base-field-metas base-field-metas}))))++(defn- make-truly-unmatched-query+ "Query for rows where LHS key exists but no RHS match was found (LHS NOT NULL, RHS NULL)."+ [ctx step]+ (when-let [{:keys [base-query lhs-field-meta rhs-field-meta lhs-field-metas]} (make-base-unmatched-query ctx step)]+ (-> base-query+ (lib/with-fields lhs-field-metas)+ (lib/filter (lib/not-null lhs-field-meta))+ (lib/filter (lib/is-null rhs-field-meta))+ (lib/limit 100))))++(defn- make-null-source-key-query+ "Query for rows where LHS key is NULL (can never match). Shows base-table columns."+ [ctx step]+ (when-let [{:keys [base-query lhs-field-meta base-field-metas]} (make-base-unmatched-query ctx step)]+ (-> base-query+ (lib/with-fields base-field-metas)+ (lib/filter (lib/is-null lhs-field-meta))+ (lib/limit 100))))++(defn- make-orphan-query+ "Query for rows in `source-table` whose key has no match in `target-table`.+ LEFT JOINs target, then filters source key NOT NULL / target key IS NULL."+ [ctx source-table-id target-table-id source-field-id target-field-id]+ (let [{:keys [db-id]} ctx+ mp (lib-be/application-database-metadata-provider db-id)+ source-table-meta (lib.metadata/table mp source-table-id)+ target-table-meta (lib.metadata/table mp target-table-id)+ source-field-meta (lib.metadata/field mp source-field-id)+ target-field-meta (lib.metadata/field mp target-field-id)+ source-field-metas (lib.metadata/fields mp source-table-id)+ check-alias "__orphan_check__"+ target-field-joined (lib/with-join-alias target-field-meta check-alias)]+ (when (seq source-field-metas)+ (-> (lib/query mp source-table-meta)+ (lib/join (-> (lib/join-clause target-table-meta+ [(lib/= source-field-meta target-field-meta)]+ :left-join)+ (lib/with-join-alias check-alias)))+ (lib/with-fields source-field-metas)+ (lib/filter (lib/not-null source-field-meta))+ (lib/filter (lib/is-null target-field-joined))+ (lib/limit 100)))))++(defn- make-orphan-rhs-query+ "Orphan check: RHS rows with no matching LHS row."+ [ctx step]+ (when-let [{:keys [rhs-table-id lhs-table-id rhs-field-id lhs-field-id]} (resolve-join-sides ctx step)]+ (make-orphan-query ctx rhs-table-id lhs-table-id rhs-field-id lhs-field-id)))++(defn- make-orphan-lhs-query+ "Orphan check: LHS rows with no matching RHS row."+ [ctx step]+ (when-let [{:keys [rhs-table-id lhs-table-id rhs-field-id lhs-field-id]} (resolve-join-sides ctx step)]+ (make-orphan-query ctx lhs-table-id rhs-table-id lhs-field-id rhs-field-id)))++;;; -------------------------------------------------- Card Generation --------------------------------------------------++(defn- truly-unmatched-card+ "Card: rows where LHS key exists but RHS didn't match."+ [ctx step params]+ (let [{:keys [join-structure]} (:mbql-context ctx)+ join (nth join-structure (dec step))+ {:keys [alias strategy]} join+ is-outer? (contains? #{:left-join :right-join :full-join} strategy)]+ (when is-outer?+ (when-let [query (make-truly-unmatched-query ctx step)]+ {:id (lens.core/make-card-id (str "truly-unmatched-" step) params)+ :section_id "samples"+ :title (tru "{0}: Rows with key but no match" alias)+ :display :table+ :dataset_query query+ :metadata {:card_type :truly_unmatched+ :join_step step+ :join_alias alias+ :join_strategy strategy}}))))++(defn- null-source-key-card+ "Card: rows where LHS key is NULL."+ [ctx step params]+ (let [{:keys [join-structure]} (:mbql-context ctx)+ join (nth join-structure (dec step))+ {:keys [alias strategy]} join+ is-outer? (contains? #{:left-join :right-join :full-join} strategy)]+ (when is-outer?+ (when-let [query (make-null-source-key-query ctx step)]+ {:id (lens.core/make-card-id (str "null-source-key-" step) params)+ :section_id "samples"+ :title (tru "{0}: Rows with NULL source key" alias)+ :display :table+ :dataset_query query+ :metadata {:card_type :null_source_key+ :join_step step+ :join_alias alias+ :join_strategy strategy}}))))++(defn- orphan-card+ "Card: rows on `side` (:rhs or :lhs) with no match on the other side.+ `strategies` is the set of join strategies for which this card applies."+ [ctx step params side strategies make-query-fn]+ (let [{:keys [join-structure]} (:mbql-context ctx)+ join (nth join-structure (dec step))+ {:keys [alias strategy]} join+ [side-label other-label] (if (= side :rhs) ["RHS" "LHS"] ["LHS" "RHS"])]+ (when (contains? strategies strategy)+ (when-let [query (make-query-fn ctx step)]+ {:id (lens.core/make-card-id (str "orphan-" (name side) "-" step) params)+ :section_id "samples"+ :title (tru "{0}: {1} rows with no {2} match" alias side-label other-label)+ :display :table+ :dataset_query query+ :metadata {:card_type (keyword (str "orphan_" (name side)))+ :join_step step+ :join_alias alias+ :join_strategy strategy}}))))++(defn- orphan-rhs-card+ "Card: RHS rows with no LHS match (LEFT/FULL joins only)."+ [ctx step params]+ (orphan-card ctx step params :rhs #{:left-join :full-join} make-orphan-rhs-query))++(defn- orphan-lhs-card+ "Card: LHS rows with no RHS match (RIGHT/FULL joins only)."+ [ctx step params]+ (orphan-card ctx step params :lhs #{:right-join :full-join} make-orphan-lhs-query))++(defn- cards-for-join+ "All unmatched-row cards for a single join step."+ [ctx step params]+ (keep identity [(truly-unmatched-card ctx step params)+ (null-source-key-card ctx step params)+ (orphan-rhs-card ctx step params)+ (orphan-lhs-card ctx step params)]))++(defn- all-cards+ "Sample cards for all outer joins, optionally filtered by `:join_step` param."+ [ctx params]+ (let [{:keys [join-structure]} (:mbql-context ctx)+ ;; Parse to int - may be string from query params+ requested-step (some-> (:join_step params) str parse-long)+ join-count (count join-structure)]+ (into []+ (mapcat (fn [step]+ (when (or (nil? requested-step) (= step requested-step))+ (cards-for-join ctx step params))))+ (range 1 (inc join-count)))))++;;; -------------------------------------------------- Lens Implementation --------------------------------------------------++(defmethod lens.core/lens-applicable? :unmatched-rows+ [_ ctx]+ ;; Only applicable for MBQL with outer joins+ (let [{:keys [join-structure preprocessed-query]} (:mbql-context ctx)]+ (and (= (:source-type ctx) :mbql)+ (:has-joins? ctx)+ preprocessed-query+ (some #(contains? #{:left-join :right-join :full-join} (:strategy %))+ join-structure))))++(defmethod lens.core/lens-metadata :unmatched-rows+ [_ _ctx]+ {:id "unmatched-rows"+ :display_name (tru "Unmatched Rows")+ :description (tru "Sample rows that failed to join")+ :complexity {:level :slow}})++(defmethod lens.core/make-lens :unmatched-rows+ [lens-type ctx params]+ (let [cards (all-cards ctx params)+ outer-join-count (count (filter #(contains? #{:left-join :right-join :full-join} (:strategy %))+ (:join-structure (:mbql-context ctx))))+ requested-step (:join_step params)+ title (when requested-step+ (tru "Unmatched Rows - Join {0}" requested-step))]+ (cond-> (lens.core/with-metadata lens-type ctx+ {:summary (if (seq cards)+ {:text (trun "Analyzing unmatched rows for {0} outer join"+ "Analyzing unmatched rows for {0} outer joins"+ outer-join-count)+ :highlights [{:label (tru "Outer Joins") :value outer-join-count}+ {:label (tru "Sample Cards") :value (count cards)}]}+ {:text (tru "No outer joins with detectable join conditions")+ :highlights []})+ :sections [{:id "samples"+ :title (tru "Unmatched Row Samples")+ :layout :flat}]+ :cards cards})+ title (assoc :display_name title))))diff --git a/src/metabase/transforms_inspector/query_analysis.clj b/src/metabase/transforms_inspector/query_analysis.cljnew file mode 100644index 00000000000..43054069abc--- /dev/null+++ b/src/metabase/transforms_inspector/query_analysis.clj@@ -0,0 +1,277 @@+(ns metabase.transforms-inspector.query-analysis+ "Query analysis for Transform Inspector.++ Extracts structural information from MBQL and native SQL queries:+ - Join structure (strategy, alias, source table, conditions)+ - Visited fields (fields used in WHERE, JOIN, GROUP BY, ORDER BY)++ For native queries, HoneySQL data structures are produced during analysis+ to keep macaw AST details isolated to this namespace. Consumers format+ these with [[metabase.driver.sql.query-processor/format-honeysql]]."+ (:require+ ;; TODO (Bronsa 16/02/26): get rid of macaw+ [macaw.ast :as macaw.ast]+ [macaw.core :as macaw]+ [metabase.driver :as driver]+ [metabase.driver.sql.normalize :as sql.normalize]+ [metabase.lib.core :as lib]+ [metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.sql-tools.core :as sql-tools]+ [metabase.transforms.util :as transforms.util]+ [metabase.util :as u]+ [metabase.util.honey-sql-2 :as h2x]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;; TODO (Bronsa 16/02/26): copied from metabase.sql-tools.macaw.core, until we can port everything to sql-tools++(def ^:private considered-drivers+ "The set of drivers for which we configure non-trivial macaw options."+ #{:h2 :mysql :postgres :redshift :sqlite :sqlserver})++(defn- macaw-options+ "Generate the options expected by Macaw based on the nature of the given driver."+ [driver]+ (merge+ (when (contains? considered-drivers driver)+ {:case-insensitive :agnostic+ :quotes-preserve-case? (not (contains? #{:mysql :sqlserver} driver))+ :features {:postgres-syntax (isa? driver/hierarchy driver :postgres)+ :square-bracket-quotes (= :sqlserver driver)+ :unsupported-statements false+ :backslash-escape-char true+ :complex-parsing true}+ :timeout 10000})+ {:non-reserved-words (vec (remove nil? [(when-not (contains? #{:clickhouse} driver)+ :final)]))}))++(defn- macaw-parsed-query+ "Parse SQL using Macaw with driver-specific options."+ [sql driver & {:as opts}]+ (macaw/parsed-query sql (merge (macaw-options driver) opts)))++;;; -------------------------------------------------- MBQL Analysis --------------------------------------------------++(defn- extract-mbql-join-structure+ "Extract join structure from preprocessed MBQL query."+ [preprocessed-query]+ (when-let [joins (lib/joins preprocessed-query 0)]+ (mapv (fn [join]+ {:strategy (or (:strategy join) :left-join)+ :alias (:alias join)+ :source-table (lib/source-table-id join)+ :conditions (:conditions join)})+ joins)))++(defn- extract-mbql-visited-fields+ "Extract field IDs from semantically important MBQL clauses."+ [query]+ (let [extract-ids (fn [clauses] (into #{} (mapcat lib/all-field-ids) clauses))+ filter-fields (some-> (lib/filters query 0) extract-ids)+ group-by-fields (some-> (lib/breakouts query 0) extract-ids)+ order-by-fields (some-> (lib/order-bys query 0) extract-ids)+ join-fields (some->> (lib/joins query 0)+ (into #{} (mapcat #(mapcat lib/all-field-ids (:conditions %)))))]+ {:all (into #{} cat [join-fields filter-fields group-by-fields order-by-fields])}))++(defn analyze-mbql-query+ "Analyze an MBQL query for join structure and visited fields.+ Returns {:join-structure [...] :visited-fields {...}} or nil on failure."+ [transform]+ (let [preprocessed (-> transform :source :query+ transforms.util/massage-sql-query+ qp.preprocess/preprocess)]+ (when (<= (count (:stages preprocessed)) 1)+ {:preprocessed-query preprocessed+ :from-table-id (lib/source-table-id preprocessed)+ :join-structure (extract-mbql-join-structure preprocessed)+ :visited-fields (extract-mbql-visited-fields preprocessed)})))++;;; -------------------------------------------------- Macaw AST → HoneySQL --------------------------------------------------+;;; These functions convert macaw AST nodes to HoneySQL data structures,+;;; isolating macaw details to this namespace.++(defn- macaw-table->hsql+ "Convert a macaw table node to a HoneySQL table reference.+ Returns `[table-identifier :alias]` when aliased, or `table-identifier` when not.+ The alias is a plain keyword since HoneySQL expects that in FROM/JOIN position."+ [driver {:keys [schema table table-alias]}]+ (let [table-id (apply h2x/identifier :table+ (remove nil? [(some->> schema (sql.normalize/normalize-name driver))+ (sql.normalize/normalize-name driver table)]))]+ (if table-alias+ [table-id (keyword (sql.normalize/normalize-name driver table-alias))]+ table-id)))++(defn- macaw-column->hsql+ "Convert a macaw column node to a HoneySQL column identifier."+ [driver {:keys [schema table column]}]+ (apply h2x/identifier :field+ (remove nil? [(some->> schema (sql.normalize/normalize-name driver))+ (some->> table (sql.normalize/normalize-name driver))+ (sql.normalize/normalize-name driver column)])))++(defn- macaw-expr->hsql+ "Convert a macaw AST expression node to a HoneySQL form."+ [driver node]+ (case (:type node)+ :macaw.ast/column (macaw-column->hsql driver node)+ :macaw.ast/literal (let [{:keys [value]} node]+ (cond+ (nil? value) nil+ :else [:inline value]))+ nil))++(defn- macaw-condition->hsql+ "Convert a macaw AST condition to a HoneySQL condition form.+ Handles AND/OR compound conditions recursively."+ [driver condition]+ (when (= (:type condition) :macaw.ast/binary-expression)+ (let [{:keys [operator left right]} condition+ op-upper (u/upper-case-en (str operator))]+ (cond+ (contains? #{"AND" "OR"} op-upper)+ (let [left-hsql (macaw-condition->hsql driver left)+ right-hsql (macaw-condition->hsql driver right)]+ (when (and left-hsql right-hsql)+ [(keyword op-upper) left-hsql right-hsql]))++ :else+ (let [left-hsql (macaw-expr->hsql driver left)+ right-hsql (macaw-expr->hsql driver right)]+ (when (and left-hsql right-hsql)+ [(keyword operator) left-hsql right-hsql]))))))++(defn- column-from-ast+ "Extract the column on `side` (:left or :right) from a single equality join condition.+ Returns nil for compound (AND/OR) or multi-condition joins — we only analyze+ simple single-condition joins."+ [side conditions]+ (let [conditions-list (if (sequential? conditions) conditions [conditions])+ condition (when (= 1 (count conditions-list)) (first conditions-list))]+ (when (and condition+ (= (:type condition) :macaw.ast/binary-expression)+ (not (contains? #{"AND" "OR"} (u/upper-case-en (str (:operator condition))))))+ (let [node (case side :left (:left condition) :right (:right condition))]+ (when (= (:type node) :macaw.ast/column)+ node)))))++(defn- build-join-condition-hsql+ "Build a HoneySQL condition form from a macaw join node's conditions."+ [driver ast-node]+ (let [conditions-raw (:condition ast-node)+ conditions-list (if (sequential? conditions-raw) conditions-raw [conditions-raw])+ parts (keep (partial macaw-condition->hsql driver) conditions-list)]+ (case (count parts)+ 0 nil+ 1 (first parts)+ (into [:and] parts))))++;;; -------------------------------------------------- Native SQL Analysis --------------------------------------------------++(defn- ast-join-type->strategy+ "Map macaw AST join-type to metabase join strategy keyword."+ [join-type]+ (case join-type+ :left :left-join+ :right :right-join+ :full :full-join+ :cross :cross-join+ :natural :inner-join+ :inner :inner-join+ :inner-join))++(defn- sources->table-id-lookup+ "Build a normalized table name → table_id lookup map from sources."+ [driver sources]+ (into {} (map (fn [{:keys [table_name table_id]}]+ [(sql.normalize/normalize-name driver table_name) table_id]))+ sources))++(defn- extract-native-join-structure+ "Extract join structure from macaw AST as HoneySQL data.+ Returns join info with :join-table, :join-condition, :lhs-column, :rhs-column as HoneySQL forms."+ [ast table->id driver]+ (when-let [join-nodes (:join ast)]+ (mapv (fn [join-node]+ (let [strategy (ast-join-type->strategy (:join-type join-node))+ is-outer? (contains? #{:left-join :right-join :full-join} strategy)+ lhs-col (when is-outer? (column-from-ast :left (:condition join-node)))+ rhs-col (when is-outer? (column-from-ast :right (:condition join-node)))]+ {:strategy strategy+ :alias (sql.normalize/normalize-name+ driver+ (or (get-in join-node [:source :table-alias])+ (get-in join-node [:source :table])))+ :source-table (table->id (sql.normalize/normalize-name+ driver+ (get-in join-node [:source :table])))+ :join-table (macaw-table->hsql driver (:source join-node))+ :join-condition (build-join-condition-hsql driver join-node)+ :lhs-column (when lhs-col (macaw-column->hsql driver lhs-col))+ :rhs-column (when rhs-col (macaw-column->hsql driver rhs-col))}))+ join-nodes)))++(defn- resolve-native-visited-fields+ "Resolve native SQL column references to Metabase field IDs using sql-tools.+ Returns a visited-fields map with all referenced field IDs in :all.+ Per-clause breakdown (filter/group-by/order-by/join) is not available through+ sql-tools, but is currently unused by consumers."+ [driver-kw native-query]+ (let [referenced (sql-tools/referenced-fields driver-kw native-query)+ all-ids (into #{} (keep :id) referenced)]+ {:all all-ids}))++(defn analyze-native-query+ "Analyze a native SQL query for join structure and visited fields.+ Requires sources (with field info) for column resolution.+ Returns HoneySQL data structures for table/column/condition references.+ Uses sql-tools for visited field resolution, macaw directly for join structure."+ [transform sources]+ (try+ (let [native-query (-> (get-in transform [:source :query])+ transforms.util/massage-sql-query+ qp.preprocess/preprocess)+ sql (lib/raw-native-query native-query)+ db-id (transforms.util/transform-source-database transform)+ driver (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)+ parsed (macaw-parsed-query sql driver)+ ast (macaw.ast/->ast parsed {:with-instance? false})]+ (when (and ast (= (:type ast) :macaw.ast/select))+ (let [table->id (sources->table-id-lookup driver sources)]+ {:driver driver+ :from-table-id (table->id (sql.normalize/normalize-name+ driver (:table (:from ast))))+ :from-table (macaw-table->hsql driver (:from ast))+ :join-structure (extract-native-join-structure ast table->id driver)+ :visited-fields (resolve-native-visited-fields driver native-query)})))+ (catch Exception e+ (log/warn e "Failed to analyze native SQL query")+ nil)))++;;; -------------------------------------------------- Unified Analysis --------------------------------------------------++(defn analyze-query+ "Analyze a transform's query for join structure and visited fields.+ Dispatches to MBQL or native analysis based on source type.++ Arguments:+ - transform: the transform map+ - source-type: :mbql, :native, or :python+ - sources: source table info (with fields) for native column resolution++ Returns:+ {:preprocessed-query <pMBQL> (MBQL only)+ :driver <keyword> (native only)+ :from-table-id <int> (native only)+ :from-table <honeysql> (native only)+ :join-structure [{:strategy :alias :source-table+ :conditions (MBQL) or :join-table/:join-condition/:lhs-column/:rhs-column (native)} ...]+ :visited-fields {:all <set of field IDs>}}"+ [transform source-type sources]+ (case source-type+ :mbql (analyze-mbql-query transform)+ :native (analyze-native-query transform sources)+ nil))diff --git a/src/metabase/transforms_inspector/schema.clj b/src/metabase/transforms_inspector/schema.cljnew file mode 100644index 00000000000..9c5ba6e7b6b--- /dev/null+++ b/src/metabase/transforms_inspector/schema.clj@@ -0,0 +1,249 @@+(ns metabase.transforms-inspector.schema+ "Schemas for Transform Inspector.++ Key principles:+ - Cards are generic; lens-specific data goes in :metadata+ - Only two layouts: :flat and :comparison+ - FE interprets rendering based on lens type + card metadata"+ (:require+ [malli.util :as mut]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.util.malli.registry :as mr]))++;;; -------------------------------------------------- Field & Table Schemas --------------------------------------------------++(mr/def ::field-stats+ "Statistics for a field from fingerprints."+ [:map+ [:distinct_count {:optional true} :int]+ [:nil_percent {:optional true} :double]+ ;; Number stats+ [:min {:optional true} :double]+ [:max {:optional true} :double]+ [:avg {:optional true} :double]+ [:q1 {:optional true} :double]+ [:q3 {:optional true} :double]+ ;; Temporal stats+ [:earliest {:optional true} :string]+ [:latest {:optional true} :string]])++(mr/def ::field+ "Field metadata for inspector."+ [:map+ [:id pos-int?]+ [:name :string]+ [:display_name {:optional true} [:maybe :string]]+ [:base_type ::lib.schema.common/base-type]+ [:semantic_type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ [:stats {:optional true} [:maybe ::field-stats]]])++(mr/def ::table+ "Table metadata."+ [:map+ [:table_id pos-int?]+ [:table_name :string]+ [:schema {:optional true} [:maybe :string]]+ [:db_id pos-int?]+ [:column_count :int]+ [:fields [:sequential ::field]]])++;;; -------------------------------------------------- Visited Fields --------------------------------------------------++(mr/def ::visited-fields+ "Field IDs used in semantically important query clauses.+ Used for interestingness scoring and preselection."+ [:map+ [:all {:optional true} [:set pos-int?]]])++;;; -------------------------------------------------- Lens Metadata --------------------------------------------------++(mr/def ::complexity-level+ "Estimated complexity/cost of generating a lens."+ [:enum :fast :slow :very-slow])++(mr/def ::complexity+ "Complexity estimate for a lens."+ [:map+ [:level ::complexity-level]+ [:score {:optional true} pos-int?]])++(mr/def ::lens-metadata+ [:map+ [:id :string]+ [:display_name :string]+ [:description {:optional true} [:maybe :string]]+ [:complexity {:optional true} ::complexity]])++;;; -------------------------------------------------- Cards --------------------------------------------------++(mr/def ::display-type+ "Valid display types for cards."+ [:enum :bar :row :line :area :pie :scalar :gauge :progress :table :hidden])++;;; Common card metadata schemas++(mr/def ::dedup-key+ "Key for deduplicating identical queries across lenses.+ Typically a vector like [:table_count table-id]."+ [:sequential :any])++(mr/def ::group-role+ "Role of a card within a comparison group."+ [:enum :input :output])++(mr/def ::comparison-metadata+ "Metadata for cards in comparison layouts."+ [:map+ [:group_id :string]+ [:group_role ::group-role]+ [:group_order :int]])++(mr/def ::join-metadata+ "Common metadata for join-related cards and triggers."+ [:map+ [:join_step {:optional true} :int]+ [:join_alias {:optional true} :string]+ [:join_strategy {:optional true} :keyword]])++(mr/def ::join-step-metadata+ "Metadata for join analysis cards."+ [:merge+ [:map [:card_type :keyword #_[:enum :base_count :join_step :table_count]]]+ (mut/optional-keys ::join-metadata)])++(mr/def ::card-metadata+ "Optional metadata for cards. An open map that may contain keys from+ comparison-metadata, join-step-metadata, or other lens-specific data."+ (mut/open-schema+ [:merge+ [:map+ [:dedup_key {:optional true} ::dedup-key]+ ;; Card IDs this card depends on (e.g. for degeneracy checks)+ [:depends_on_cards {:optional true} [:set :string]]]+ (mut/optional-keys ::comparison-metadata)+ (mut/optional-keys ::join-step-metadata)]))++(mr/def ::card+ "A visualization card in the inspector output.++ Cards are generic - lens-specific data goes in :metadata.+ The :metadata map is opaque to the core but interpreted by+ lens-specific lib utilities on the frontend. "+ [:map+ [:id :string]+ [:section_id {:optional true} [:maybe :string]]+ [:title :string]+ [:display ::display-type]+ [:dataset_query ::lib.schema/query]+ ;; Lens-specific metadata - opaque to core, interpreted by FE+ [:metadata {:optional true} ::card-metadata]])++;;; -------------------------------------------------- Sections --------------------------------------------------++(mr/def ::layout-type+ "Layout hints for sections.+ - :flat - cards displayed in a grid+ - :comparison - cards grouped for side-by-side comparison"+ [:enum :flat :comparison])++(mr/def ::section+ "A section grouping cards in a lens."+ [:map+ [:id :string]+ [:title :string]+ [:description {:optional true} [:maybe :string]]+ [:layout {:optional true} ::layout-type]])++;;; -------------------------------------------------- Summary & Alerts --------------------------------------------------++(mr/def ::summary-highlight+ "A highlight in the lens summary."+ [:map+ [:label :string]+ [:value {:optional true} [:maybe :any]]+ [:card_id {:optional true} :string]])++(mr/def ::lens-summary+ "Summary section of a lens."+ [:map+ [:text {:optional true} [:maybe :string]]+ [:highlights {:optional true} [:sequential ::summary-highlight]]+ [:alerts {:optional true} [:sequential :map]]])++;;; -------------------------------------------------- Triggers --------------------------------------------------++(mr/def ::trigger-condition+ "A named condition that triggers an alert or drill lens.+ Evaluated by multimethod in lib against card results.+ Only :name is required; other keys are condition-specific."+ [:map {:closed false}+ [:name :keyword]])++(mr/def ::trigger-metadata+ "Metadata for triggers. Uses same join fields as card metadata."+ (mut/open-schema+ (mut/optional-keys ::join-metadata)))++(mr/def ::alert-trigger+ "Definition for conditional alerts.+ FE evaluates condition against card results and shows alert if triggered."+ [:map+ [:id :string]+ [:condition ::trigger-condition]+ [:severity [:enum :info :warning :error]]+ [:message :string]+ [:metadata {:optional true} ::trigger-metadata]])++(mr/def ::drill-lens-trigger+ "Definition for conditional drill lens availability.+ FE evaluates condition and shows drill lens option if triggered.+ :params is passed to the drill lens when triggered."+ [:map+ [:lens_id :string]+ [:condition ::trigger-condition]+ [:params {:optional true} [:map-of :keyword :any]]+ [:reason {:optional true} :string]+ [:metadata {:optional true} ::trigger-metadata]])++;;; -------------------------------------------------- Drill Lenses --------------------------------------------------++(mr/def ::drill-lens+ "A drill-down lens available from the current lens."+ [:map+ [:id :string]+ [:display_name :string]+ [:description {:optional true} [:maybe :string]]])++;;; -------------------------------------------------- Lens Response --------------------------------------------------++(mr/def ::lens+ "Full lens contents and drill lens/alert triggers."+ [:map+ [:id :string]+ [:display_name :string]+ [:complexity {:optional true} ::complexity]+ [:summary {:optional true} ::lens-summary]+ [:sections [:sequential ::section]]+ [:cards [:sequential ::card]]+ [:drill_lenses {:optional true} [:sequential ::drill-lens]]+ ;; Trigger definitions - FE evaluates conditions against card results+ [:alert_triggers {:optional true} [:sequential ::alert-trigger]]+ [:drill_lens_triggers {:optional true} [:sequential ::drill-lens-trigger]]])++;;; -------------------------------------------------- Discovery Response --------------------------------------------------++(mr/def ::inspector-status+ "Status of the inspector result."+ [:enum :not-run :ready])++(mr/def ::discovery-response+ "Response from lens discovery."+ [:map+ [:name :string]+ [:description {:optional true} [:maybe :string]]+ [:status ::inspector-status]+ [:sources [:sequential ::table]]+ [:target {:optional true} [:maybe ::table]]+ [:visited_fields {:optional true} [:maybe ::visited-fields]]+ [:available_lenses [:sequential ::lens-metadata]]])diff --git a/src/metabase/transforms_inspector/triggers.cljc b/src/metabase/transforms_inspector/triggers.cljcnew file mode 100644index 00000000000..bae8f5031c4--- /dev/null+++ b/src/metabase/transforms_inspector/triggers.cljc@@ -0,0 +1,29 @@+(ns metabase.transforms-inspector.triggers+ "Multimethod for evaluating trigger conditions against inspector card results.++ Triggers control when alerts and drill-down lenses appear+ based on computed card results (e.g., high null rate).")++(defmulti evaluate-condition+ "Evaluate a named condition against card results.+ Dispatches on condition name. Returns truthy if condition is met.++ Arguments:+ - condition: map with :name and any other keys the condition needs+ - card-results: map of card-id (string) -> result map (string keys)"+ {:arglists '([condition card-results])}+ (fn [condition _card-results] (:name condition)))++(defmethod evaluate-condition :default+ [_ _]+ false)++(defmethod evaluate-condition :high-null-rate+ [{:keys [card_id]} card-results]+ (when-let [result (get card-results card_id)]+ (> (get result "null_rate" 0) 0.2)))++(defmethod evaluate-condition :has-unmatched-rows+ [{:keys [card_id]} card-results]+ (when-let [result (get card-results card_id)]+ (> (get result "null_rate" 0) 0.05)))diff --git a/src/metabase/transforms/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljsimilarity index 50%rename from src/metabase/transforms/api/transform.cljrename to src/metabase/transforms_rest/api/transform.cljindex 8efb7fafdbe..9a2ed58498c 100644--- a/src/metabase/transforms/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -1,44 +1,34 @@-(ns metabase.transforms.api.transform+(ns metabase.transforms-rest.api.transform(:require[metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.api.util.handlers :as handlers]- [metabase.database-routing.core :as database-routing][metabase.driver :as driver]- [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]- [metabase.driver.util :as driver.u][metabase.events.core :as events]- [metabase.lib.core :as lib]- [metabase.models.interface :as mi]- [metabase.models.transforms.transform :as transform.model]- [metabase.models.transforms.transform-run :as transform-run]- [metabase.models.transforms.transform-run-cancelation :as transform-run-cancelation]- [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor :as qp]+ [metabase.query-processor.middleware.constraints :as qp.constraints][metabase.query-processor.schema :as qp.schema]+ [metabase.query-processor.streaming :as qp.streaming][metabase.request.core :as request]+ [metabase.server.core :as server][metabase.sql-tools.core :as sql-tools]- [metabase.transforms.api.transform-job]- [metabase.transforms.api.transform-tag]- [metabase.transforms.canceling :as transforms.canceling]- [metabase.transforms.execute :as transforms.execute]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.ordering :as transforms.ordering]+ [metabase.transforms-inspector.core :as inspector]+ [metabase.transforms-inspector.schema :as inspector.schema]+ [metabase.transforms-rest.api.transform-job]+ [metabase.transforms-rest.api.transform-tag]+ [metabase.transforms.core :as transforms.core][metabase.transforms.schema :as transforms.schema][metabase.transforms.util :as transforms.util]- [metabase.util :as u][metabase.util.i18n :refer [deferred-tru]][metabase.util.jvm :as u.jvm]- [metabase.util.log :as log][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][ring.util.response :as response]- [toucan2.core :as t2])- (:import- (java.sql PreparedStatement)))+ [toucan2.core :as t2]))-(comment metabase.transforms.api.transform-job/keep-me- metabase.transforms.api.transform-tag/keep-me)+(comment metabase.transforms-rest.api.transform-job/keep-me+ metabase.transforms-rest.api.transform-tag/keep-me)(set! *warn-on-reflection* true)@@ -133,18 +123,6 @@[:collection {:optional true} [:maybe :map]][:tag_ids {:optional true} [:sequential pos-int?]]]]]])-(defn- python-source-table-ref->table-id- "Change source of python transform from name->table-ref to name->table-id.-- We now supported table-ref as source but since FE is still expecting table-id we need to temporarily do this.- Should update FE to fully use table-ref"- [transform]- (if (transforms.util/python-transform? transform)- (update-in transform [:source :source-tables]- (fn [source-tables]- (update-vals source-tables #(if (int? %) % (:table_id %)))))- transform))-(comment;; Examples[{:id 1@@ -158,48 +136,6 @@:schema "transforms":name "gadget_products"}}])-(defn- check-database-feature- [transform]- (let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))- (deferred-tru "The target database cannot be found."))- features (transforms.util/required-database-features transform)]- (api/check-400 (not (:is_sample database))- (deferred-tru "Cannot run transforms on the sample database."))- (api/check-400 (not (:is_audit database))- (deferred-tru "Cannot run transforms on audit databases."))- (api/check-400 (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)- (deferred-tru "The database does not support the requested transform features."))- (api/check-400 (not (database-routing/db-routing-enabled? database))- (deferred-tru "Transforms are not supported on databases with DB routing enabled."))))--(defn- check-feature-enabled!- [transform]- (api/check (transforms.util/check-feature-enabled transform)- [402 (deferred-tru "Premium features required for this transform type are not enabled.")]))--(defn- validate-transform-query!- [transform]- (when-let [error (transforms.util/validate-transform-query transform)]- (throw (ex-info (:error error)- (assoc error- :status-code 400)))))--(defn get-transforms- "Get a list of transforms."- [& {:keys [last-run-start-time last-run-statuses tag-ids]}]- (let [enabled-types (transforms.util/enabled-source-types-for-user)]- (api/check-403 (seq enabled-types))- (let [transforms (t2/select :model/Transform {:where [:in :source_type enabled-types]- :order-by [[:id :asc]]})]- (->> (t2/hydrate transforms :last_run :transform_tag_ids :creator :owner)- (into []- (comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last-run-start-time)- (transforms.util/->status-filter-xf [:last_run :status] last-run-statuses)- (transforms.util/->tag-filter-xf [:tag_ids] tag-ids)- (map #(update % :last_run transforms.util/localize-run-timestamps))- (map python-source-table-ref->table-id)))- transforms.util/add-source-readable))))-(api.macros/defendpoint :get "/" :- [:sequential TransformResponse]"Get a list of transforms."[_route-params@@ -209,112 +145,7 @@[:last-run-statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]][:tag-ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]][:database-id {:optional true} [:maybe ms/PositiveInt]]]]- (get-transforms query-params))--(defn- extract-all-columns-from-query- "Extracts column metadata (name and type) from a query.-- Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.-- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()- to inspect the query structure. This works for most modern JDBC drivers but may not- be supported by all drivers or for all query types."- [driver database-id query]- (try- (let [{:keys [query]} (qp.compile/compile query)]- (sql-jdbc.execute/do-with-connection-with-options- driver- database-id- {}- (fn [conn]- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]- (when-let [rsmeta (.getMetaData stmt)]- (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))- (catch Exception e- (log/debugf e "Failed to extract columns from query: %s" (ex-message e))- nil)))--(defn- extract-incremental-filter-columns-from-query- "Extracts column names suitable for incremental transform checkpoint filtering.-- This function is specifically for incremental transform checkpoint column selection.- It only returns columns with types supported for checkpoint filtering:- - Temporal types (timestamp, timestamp with timezone)- - Numeric types (integer, float, decimal)-- Text, boolean, and other types are filtered out as they are not supported for- incremental checkpointing.-- Returns a vector of column names (as strings), or nil if extraction fails.-- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()- to inspect the query structure. This works for most modern JDBC drivers but may not- be supported by all drivers or for all query types."- [driver database-id query]- (some->> (extract-all-columns-from-query driver database-id query)- (filter (comp transforms.util/supported-incremental-filter-type? :base_type))- (mapv :name)))--(defn- validate-incremental-column-type!- "Validates that the checkpoint column for an incremental transform has a supported type.-- For MBQL/Python transforms, resolves the column from the query using the unique key.- For native queries, extracts columns from the query and checks the checkpoint-filter column.-- Throws a 400 error if the column type is not supported or cannot be resolved."- [{:keys [source]}]- (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}- (:source-incremental-strategy source)]- (when (and (= :query (:type source)) (= "checkpoint" strategy-type))- (let [{:keys [query]} source- database-id (:database query)- database (api/check-404 (t2/select-one :model/Database :id database-id))- driver-name (driver/the-initialized-driver (:engine database))]- (cond- ;; For MBQL, resolve column from query metadata- checkpoint-filter-unique-key- (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]- (api/check-400 column (deferred-tru "Checkpoint column not found in query."))- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))- (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."- (pr-str (:base-type column)))))-- ;; For native query with checkpoint-filter, validate type if we can extract the column metadata- checkpoint-filter- (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]- (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."- checkpoint-filter- (pr-str (:base_type column)))))))))))--(defn create-transform!- "Create new transform in the appdb.- Optionally accepts a creator-id to use instead of the current user (for workspace merges)."- ([body]- (create-transform! body nil))- ([body creator-id]- (when (transforms.util/query-transform? body)- (validate-transform-query! body))- (let [creator-id (or creator-id api/*current-user-id*)- transform (t2/with-transaction [_]- (let [tag-ids (:tag_ids body)- ;; Set owner_user_id to current user if not explicitly provided- owner-user-id (when-not (:owner_email body)- (or (:owner_user_id body) creator-id))- transform (t2/insert-returning-instance!- :model/Transform- (assoc (select-keys body [:name :description :source :target :run_trigger- :collection_id :owner_email])- :creator_id creator-id- :owner_user_id owner-user-id))]- ;; Add tag associations if provided- (when (seq tag-ids)- (transform.model/update-transform-tags! (:id transform) tag-ids))- ;; Return with hydrated tag_ids- (t2/hydrate transform :transform_tag_ids :creator :owner)))]- (events/publish-event! :event/transform-create {:object transform :user-id creator-id})- transform)))+ (transforms.core/get-transforms query-params));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen@@ -334,34 +165,22 @@[:owner_user_id {:optional true} [:maybe ms/PositiveInt]][:owner_email {:optional true} [:maybe :string]]]](api/create-check :model/Transform body)- (check-database-feature body)- (check-feature-enabled! body)- (validate-incremental-column-type! body)+ (transforms.core/check-database-feature body)+ (transforms.core/check-feature-enabled! body)+ (transforms.core/validate-incremental-column-type! body)(api/check (not (transforms.util/target-table-exists? body))403(deferred-tru "A table with that name already exists."))- (-> (create-transform! body)- python-source-table-ref->table-id+ (-> (transforms.core/create-transform! body)+ transforms.core/python-source-table-ref->table-idtransforms.util/add-source-readable))-(defn get-transform- "Get a specific transform."- [id]- (let [{:keys [target] :as transform} (api/read-check :model/Transform id)- target-table (transforms.util/target-table (transforms.i/target-db-id transform) target :active true)]- (-> transform- (t2/hydrate :last_run :transform_tag_ids :creator :owner)- (u/update-some :last_run transforms.util/localize-run-timestamps)- (assoc :table target-table)- python-source-table-ref->table-id- transforms.util/add-source-readable)))-(api.macros/defendpoint :get "/:id" :- TransformResponse"Get a specific transform."[{:keys [id]} :- [:map[:id ms/PositiveInt]]]- (get-transform id))+ (transforms.core/get-transform id))(api.macros/defendpoint :get "/:id/dependencies" :- [:sequential TransformResponse]"Get the dependencies of a specific transform."@@ -369,11 +188,11 @@[:id ms/PositiveInt]]](api/read-check :model/Transform id)(let [id->transform (t2/select-pk->fn identity :model/Transform)- global-ordering (transforms.ordering/transform-ordering (vals id->transform))+ global-ordering (transforms.core/transform-ordering (vals id->transform))dep-ids (get global-ordering id)dependencies (map id->transform dep-ids)](->> (t2/hydrate dependencies :creator :owner)- (mapv python-source-table-ref->table-id)+ (mapv transforms.core/python-source-table-ref->table-id)transforms.util/add-source-readable)))(def ^:private MergeHistoryEntry@@ -422,9 +241,9 @@[:end-time {:optional true} [:maybe ms/NonBlankString]][:run-methods {:optional true} [:maybe (ms/QueryVectorOf [:enum "manual" "cron"])]]]](api/check-data-analyst)- (-> (transform-run/paged-runs (assoc query-params- :offset (request/offset)- :limit (request/limit)))+ (-> (transforms.core/paged-runs (assoc query-params+ :offset (request/offset)+ :limit (request/limit)))(update :data #(map transforms.util/localize-run-timestamps %))))(api.macros/defendpoint :get "/run/:run-id" :- TransformRunResponse@@ -436,40 +255,6 @@(-> (t2/hydrate run [:transform :collection :transform_tag_ids])transforms.util/localize-run-timestamps)))-(defn update-transform!- "Update a transform. Validates features, database support, cycles, and target conflicts.- Returns the updated transform with hydrated associations."- [id body]- (let [transform (t2/with-transaction [_]- ;; Cycle detection should occur within the transaction to avoid race- (let [old (t2/select-one :model/Transform id)- new (merge old body)- target-fields #(-> % :target (select-keys [:schema :name]))]- (api/check-403 (and (mi/can-write? old) (mi/can-write? new)))-- ;; we must validate on a full transform object- (check-feature-enabled! new)- (check-database-feature new)- (validate-incremental-column-type! new)- (when (transforms.util/query-transform? old)- (validate-transform-query! new)- (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]- (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)- {:status-code 400}))))- (api/check (not (and (not= (target-fields old) (target-fields new))- (transforms.util/target-table-exists? new)))- 403- (deferred-tru "A table with that name already exists.")))- (t2/update! :model/Transform id (dissoc body :tag_ids))- ;; Update tag associations if provided- (when (contains? body :tag_ids)- (transform.model/update-transform-tags! id (:tag_ids body)))- (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids :creator :owner))]- (events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})- (-> transform- python-source-table-ref->table-id- transforms.util/add-source-readable)))-(api.macros/defendpoint :put "/:id" :- TransformResponse"Update a transform."[{:keys [id]} :- [:map@@ -486,22 +271,13 @@[:owner_user_id {:optional true} [:maybe ms/PositiveInt]][:owner_email {:optional true} [:maybe :string]]]](api/write-check :model/Transform id)- (update-transform! id body))--(defn delete-transform!- "Delete a transform and publish the delete event."- [transform]- (t2/delete! :model/Transform (:id transform))- (events/publish-event! :event/transform-delete- {:object transform- :user-id api/*current-user-id*})- nil)+ (transforms.core/update-transform! id body))(api.macros/defendpoint :delete "/:id" :- :nil"Delete a transform."[{:keys [id]} :- [:map[:id ms/PositiveInt]]]- (delete-transform! (api/write-check :model/Transform id)))+ (transforms.core/delete-transform! (api/write-check :model/Transform id)))(api.macros/defendpoint :delete "/:id/table" :- :nil"Delete a transform's output table."@@ -516,22 +292,22 @@[{:keys [id]} :- [:map[:id ms/PositiveInt]]](let [transform (api/write-check :model/Transform id)- run (api/check-404 (transform-run/running-run-for-transform-id id))]- (transform-run-cancelation/mark-cancel-started-run! (:id run))+ run (api/check-404 (transforms.core/running-run-for-transform-id id))]+ (transforms.core/mark-cancel-started-run! (:id run))(when (transforms.util/python-transform? transform)- (transforms.canceling/cancel-run! (:id run))))+ (transforms.core/cancel-run! (:id run))))nil)(defn run-transform!"Run a transform. Returns a 202 response with run_id.The transform must already be fetched and validated."[transform]- (check-feature-enabled! transform)+ (transforms.core/check-feature-enabled! transform)(let [start-promise (promise)](u.jvm/in-virtual-thread*- (transforms.execute/execute! transform {:start-promise start-promise- :run-method :manual- :user-id api/*current-user-id*}))+ (transforms.core/execute! transform {:start-promise start-promise+ :run-method :manual+ :user-id api/*current-user-id*}))(when (instance? Throwable @start-promise)(throw @start-promise))(let [result @start-promise@@ -591,9 +367,72 @@(let [database-id (:database query)database (api/check-404 (t2/select-one :model/Database :id database-id))driver-name (driver/the-initialized-driver (:engine database))- columns (extract-incremental-filter-columns-from-query driver-name database-id query)]+ columns (transforms.core/extract-incremental-filter-columns-from-query driver-name database-id query)]{:columns columns}))+;;; -------------------------------------------------- Inspector API --------------------------------------------------++(api.macros/defendpoint :get "/:id/inspect"+ :- ::inspector.schema/discovery-response+ "Phase 1: Discover available lenses for a transform.+ Returns structural metadata and available lens types."+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]]+ (let [transform (api/read-check :model/Transform id)+ result (do (transforms.core/check-feature-enabled! transform)+ (inspector/discover-lenses transform))]+ (events/publish-event! :event/transform-inspect-discover+ {:object transform+ :user-id api/*current-user-id*})+ result))++(api.macros/defendpoint :get "/:id/inspect/:lens-id"+ :- ::inspector.schema/lens+ "Phase 2: Get full lens contents for a transform.+ Returns sections, cards with dataset_query, and trigger definitions.+ Accepts optional params for drill lenses as query params."+ [{:keys [id lens-id]} :- [:map+ [:id ms/PositiveInt]+ [:lens-id ms/NonBlankString]]+ params :- [:map-of :keyword :any]]+ (let [transform (api/read-check :model/Transform id)+ result (do (transforms.core/check-feature-enabled! transform)+ (inspector/get-lens transform lens-id params))]+ (events/publish-event! :event/transform-inspect-lens+ {:object transform+ :user-id api/*current-user-id*+ :details {:lens-id lens-id+ :num-cards (count (:cards result))+ :num-drill-lenses (count (:drill_lenses result))+ :num-alert-triggers (count (:alert_triggers result))}})+ result))++(api.macros/defendpoint :post "/:id/inspect/:lens-id/query"+ :- (server/streaming-response-schema ::qp.schema/query-result)+ "Execute a query in the context of a transform inspector lens."+ [{:keys [id lens-id]} :- [:map+ [:id ms/PositiveInt]+ [:lens-id ms/NonBlankString]]+ _query-params+ {query :query, lens-params :lens_params}+ :- [:map+ [:query [:map [:database {:optional true} [:maybe :int]]]]+ [:lens_params {:optional true} [:maybe [:map-of :keyword :any]]]]]+ (let [transform (api/read-check :model/Transform id)]+ (transforms.core/check-feature-enabled! transform)+ (let [info {:executed-by api/*current-user-id*+ :context :transform-inspector+ :transform-id id+ :lens-id lens-id+ :lens-params lens-params}]+ (qp.streaming/streaming-response [rff :api]+ (qp/process-query+ (-> query+ (update-in [:middleware :js-int-to-string?] (fnil identity true))+ (assoc :constraints (qp.constraints/default-query-constraints))+ (update :info merge info)+ qp/userland-query)+ rff)))))+(def ^{:arglists '([request respond raise])} routes"`/api/transform` routes."(handlers/routes@@ -601,8 +440,8 @@(def ^{:arglists '([request respond raise])} transform-tag-routes"`/api/transform-tag` routes."- (api.macros/ns-handler 'metabase.transforms.api.transform-tag +auth))+ (api.macros/ns-handler 'metabase.transforms-rest.api.transform-tag +auth))(def ^{:arglists '([request respond raise])} transform-job-routes"`/api/transform-job` routes."- (api.macros/ns-handler 'metabase.transforms.api.transform-job +auth))+ (api.macros/ns-handler 'metabase.transforms-rest.api.transform-job +auth))diff --git a/src/metabase/transforms/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.cljsimilarity index 93%rename from src/metabase/transforms/api/transform_job.cljrename to src/metabase/transforms_rest/api/transform_job.cljindex 00df1ae90ab..01758de2184 100644--- a/src/metabase/transforms/api/transform_job.clj+++ b/src/metabase/transforms_rest/api/transform_job.clj@@ -1,13 +1,11 @@-(ns metabase.transforms.api.transform-job+(ns metabase.transforms-rest.api.transform-job(:require[medley.core :as m][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.models.interface :as mi]- [metabase.models.transforms.transform-job :as transform-job]- [metabase.transforms.jobs :as transforms.jobs]- [metabase.transforms.schedule :as transforms.schedule]+ [metabase.transforms.core :as transforms.core][metabase.transforms.util :as transforms.util][metabase.util.i18n :refer [deferred-tru LocalizedString]][metabase.util.jvm :as u.jvm]@@ -17,8 +15,6 @@(set! *warn-on-reflection* true)-(comment transform-job/keep-me)-(def ^:private ui-display-types [:cron/raw :cron/builder])(def ^:private LastRunResponse@@ -99,7 +95,7 @@[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]](log/info "Creating transform job:" name "with schedule:" schedule);; Validate cron expression- (api/check-400 (transforms.schedule/validate-cron-expression schedule)+ (api/check-400 (transforms.core/validate-cron-expression schedule)(deferred-tru "Invalid cron expression: {0}" schedule));; Validate tag IDs exist if provided(when (seq tag_ids)@@ -113,7 +109,7 @@_ (api/check-403 (mi/can-create? :model/TransformJob (assoc job-data :tag_ids tag_ids)))job (t2/insert-returning-instance! :model/TransformJobjob-data)]- (transforms.schedule/initialize-job! job)+ (transforms.core/initialize-job! job);; Add tag associations if provided(when (seq tag_ids)(t2/insert! :model/TransformJobTransformTag@@ -147,7 +143,7 @@(api/write-check (assoc existing-job :tag_ids tag-ids))));; Validate cron expression if provided(when schedule- (api/check-400 (transforms.schedule/validate-cron-expression schedule)+ (api/check-400 (transforms.core/validate-cron-expression schedule)(deferred-tru "Invalid cron expression: {0}" schedule)));; Validate tag IDs if provided(when (seq tag-ids)@@ -162,10 +158,10 @@:ui_display_type ui_display_type)](t2/update! :model/TransformJob job-id updates))(when schedule- (transforms.schedule/update-job! job-id schedule))+ (transforms.core/update-job! job-id schedule));; Update tag associations if provided(when (some? tag-ids)- (transform-job/update-job-tags! job-id tag-ids))+ (transforms.core/update-job-tags! job-id tag-ids));; Return updated job with hydration(-> (t2/select-one :model/TransformJob :id job-id)(t2/hydrate :tag_ids :last_run))))@@ -176,7 +172,7 @@(log/info "Deleting transform job" job-id)(api/write-check (t2/hydrate (t2/select-one :model/TransformJob :id job-id) :tag_ids))(t2/delete! :model/TransformJob :id job-id)- (transforms.schedule/delete-job! job-id)+ (transforms.core/delete-job! job-id)api/generic-204-no-content)(api.macros/defendpoint :post "/:job-id/run" :- [:map {:closed true}@@ -188,7 +184,7 @@(api/write-check (t2/select-one :model/TransformJob :id job-id))(u.jvm/in-virtual-thread*(try- (transforms.jobs/run-job! job-id {:run-method :manual+ (transforms.core/run-job! job-id {:run-method :manual:user-id api/*current-user-id*})(catch Throwable t(log/error "Error executing transform job" job-id)@@ -206,7 +202,7 @@(defn- add-next-run[{id :id :as job}]- (if-let [start-time (-> id transforms.schedule/existing-trigger :next-fire-time)]+ (if-let [start-time (-> id transforms.core/existing-trigger :next-fire-time)](assoc job :next_run {:start_time (str (transforms.util/->instant start-time))})job))@@ -216,7 +212,7 @@[:job-id ms/PositiveInt]]](log/info "Getting the transforms of transform job" job-id)(api/check-404 (t2/select-one-pk :model/TransformJob :id job-id))- (-> (transforms.jobs/job-transforms job-id)+ (-> (transforms.core/job-transforms job-id)(#(do (api/check-403 (every? mi/can-read? %)) %))(t2/hydrate :creator)transforms.util/add-source-readable))diff --git a/src/metabase/transforms/api/transform_tag.clj b/src/metabase/transforms_rest/api/transform_tag.cljsimilarity index 91%rename from src/metabase/transforms/api/transform_tag.cljrename to src/metabase/transforms_rest/api/transform_tag.cljindex ffa7e24752c..b8bd7a9fcff 100644--- a/src/metabase/transforms/api/transform_tag.clj+++ b/src/metabase/transforms_rest/api/transform_tag.clj@@ -1,10 +1,10 @@-(ns metabase.transforms.api.transform-tag+(ns metabase.transforms-rest.api.transform-tag(:require[metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.models.interface :as mi]- [metabase.models.transforms.transform-tag :as transform-tag]+ [metabase.transforms.core :as transforms.core][metabase.util.i18n :refer [deferred-tru LocalizedString]][metabase.util.log :as log][metabase.util.malli.schema :as ms]@@ -30,7 +30,7 @@[:name ms/NonBlankString]]](log/info "Creating transform tag:" name)(api/check-403 (mi/can-create? :model/TransformTag {:name name}))- (api/check-400 (not (transform-tag/tag-name-exists? name))+ (api/check-400 (not (transforms.core/tag-name-exists? name))(deferred-tru "A tag with the name ''{0}'' already exists." name))(t2/insert-returning-instance! :model/TransformTag {:name name}))@@ -43,7 +43,7 @@[:name ms/NonBlankString]]](log/info "Updating transform tag" tag-id "with name:" name)(api/write-check (t2/select-one :model/TransformTag :id tag-id))- (api/check-400 (not (transform-tag/tag-name-exists-excluding? name tag-id))+ (api/check-400 (not (transforms.core/tag-name-exists-excluding? name tag-id))(deferred-tru "A tag with the name ''{0}'' already exists." name))(t2/update! :model/TransformTag tag-id {:name name})(t2/select-one :model/TransformTag :id tag-id))diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 23c8745c133..bf2b27b8b0c 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -115,6 +115,9 @@:card_id nil:is_sandboxed false:dashboard_id nil+ :transform_id nil+ :lens_id nil+ :lens_params nil:error nil:id true:action_id nildiff --git a/test/metabase/transforms_inspector/card_result_test.clj b/test/metabase/transforms_inspector/card_result_test.cljnew file mode 100644index 00000000000..6e74ea39c22--- /dev/null+++ b/test/metabase/transforms_inspector/card_result_test.clj@@ -0,0 +1,46 @@+(ns metabase.transforms-inspector.card-result-test+ (:require+ [clojure.test :refer :all]+ [metabase.transforms-inspector.card-result :as card-result]))++(deftest compute-card-result-default-test+ (testing "default method returns nil for non-nil rows"+ (is (nil? (card-result/compute-card-result :unknown-lens {:metadata {:card_type "something"}} [1 2 3]))))+ (testing "default method returns no_data for nil row"+ (is (= {"no_data" true}+ (card-result/compute-card-result :unknown-lens {:metadata {:card_type "something"}} nil)))))++(deftest compute-card-result-join-step-test+ (testing "join-analysis join_step computes null rate correctly"+ (let [card {:metadata {:card_type "join_step"}}]+ (testing "normal case: some unmatched rows"+ (is (= {"output_count" 100+ "matched_count" 80+ "null_count" 20+ "null_rate" 1/5}+ (card-result/compute-card-result :join-analysis card [100 80]))))+ (testing "all rows matched"+ (is (= {"output_count" 50+ "matched_count" 50+ "null_count" 0+ "null_rate" 0}+ (card-result/compute-card-result :join-analysis card [50 50]))))+ (testing "no rows matched"+ (is (= {"output_count" 100+ "matched_count" 0+ "null_count" 100+ "null_rate" 1}+ (card-result/compute-card-result :join-analysis card [100 0]))))+ (testing "zero output count avoids divide-by-zero"+ (is (= {"output_count" 0+ "matched_count" 0+ "null_count" 0+ "null_rate" nil}+ (card-result/compute-card-result :join-analysis card [0 0]))))+ (testing "nil row returns no_data with zero counts"+ (is (= {"no_data" true+ "output_count" 0+ "matched_count" 0+ "null_count" 0+ "null_rate" nil}+ (card-result/compute-card-result :join-analysis card nil)))))))diff --git a/test/metabase/transforms_inspector/context_test.clj b/test/metabase/transforms_inspector/context_test.cljnew file mode 100644index 00000000000..30e05cda4b0--- /dev/null+++ b/test/metabase/transforms_inspector/context_test.clj@@ -0,0 +1,265 @@+(ns ^:mb/driver-tests metabase.transforms-inspector.context-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.transforms-inspector.context :as context]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Private fn access --------------------------------------------------++(def ^:private normalize-column-name @#'context/normalize-column-name)+(def ^:private parse-joined-column-name @#'context/parse-joined-column-name)+(def ^:private match-columns-by-name @#'context/match-columns-by-name)++;;; -------------------------------------------------- normalize-column-name --------------------------------------------------++(deftest ^:parallel normalize-column-name-test+ (testing "lowercases"+ (is (= "userid" (normalize-column-name "UserId")))+ (is (= "total" (normalize-column-name "TOTAL"))))+ (testing "strips underscores and hyphens"+ (is (= "userid" (normalize-column-name "user_id")))+ (is (= "userid" (normalize-column-name "user-id")))+ (is (= "userid" (normalize-column-name "User_Id")))+ (is (= "userid" (normalize-column-name "USER-ID"))))+ (testing "nil returns nil"+ (is (nil? (normalize-column-name nil)))))++;;; -------------------------------------------------- parse-joined-column-name --------------------------------------------------++(deftest ^:parallel parse-joined-column-name-test+ (testing "parses Alias__field pattern"+ (is (= {:alias "Products" :field-name "price"}+ (parse-joined-column-name "Products__price")))+ (is (= {:alias "Users" :field-name "created_at"}+ (parse-joined-column-name "Users__created_at"))))+ (testing "parses multi-word alias"+ (is (= {:alias "My_Table" :field-name "col"}+ (parse-joined-column-name "My_Table__col"))))+ (testing "returns nil for non-joined names"+ (is (nil? (parse-joined-column-name "price")))+ (is (nil? (parse-joined-column-name "products_price")))+ (is (nil? (parse-joined-column-name nil)))))++;;; -------------------------------------------------- match-columns-by-name --------------------------------------------------++(deftest ^:parallel match-columns-by-name-basic-test+ (let [sources [{:table_id 1 :table_name "orders"+ :fields [{:name "id" :id 10} {:name "total" :id 11} {:name "product_id" :id 12}]}+ {:table_id 2 :table_name "products"+ :fields [{:name "id" :id 20} {:name "price" :id 21} {:name "category" :id 22}]}]+ join-structure [{:alias "Products" :source-table 2}]]+ (testing "matches joined column by Alias__field pattern"+ (let [target {:table_id 3 :table_name "result"+ :fields [{:name "Products__price" :id 30}]}+ matches (seq (match-columns-by-name sources target join-structure))]+ (is (some? matches))+ (let [m (first matches)]+ (is (= "Products__price" (:name (:output-field m))))+ (is (= 2 (:source-table-id (first (:input-columns m))))))))+ (testing "matches plain column by name"+ (let [target {:table_id 3 :table_name "result"+ :fields [{:name "total" :id 31}]}+ matches (seq (match-columns-by-name sources target join-structure))]+ (is (some? matches))+ (is (= "total" (:name (:output-field (first matches)))))+ (is (= 1 (:source-table-id (first (:input-columns (first matches))))))))+ (testing "no match for unrecognized columns"+ (let [target {:table_id 3 :table_name "result"+ :fields [{:name "xyz_unknown" :id 40}]}]+ (is (empty? (match-columns-by-name sources target join-structure)))))))++(deftest ^:parallel match-columns-by-name-case-insensitive-test+ (let [sources [{:table_id 1 :table_name "t1"+ :fields [{:name "USER_ID" :id 10}]}]+ join-structure []]+ (testing "matches case-insensitively (ignoring underscores/hyphens)"+ (let [target {:table_id 2 :table_name "t2"+ :fields [{:name "user-id" :id 20}]}+ matches (seq (match-columns-by-name sources target join-structure))]+ (is (some? matches))+ (is (= "user-id" (:name (:output-field (first matches)))))))))++(deftest ^:parallel match-columns-by-name-multiple-sources-test+ (let [sources [{:table_id 1 :table_name "orders"+ :fields [{:name "total" :id 10}]}+ {:table_id 2 :table_name "products"+ :fields [{:name "total" :id 20}]}]+ join-structure []]+ (testing "when same column name exists in multiple sources, both appear as input-columns"+ (let [target {:table_id 3 :table_name "result"+ :fields [{:name "total" :id 30}]}+ matches (seq (match-columns-by-name sources target join-structure))+ m (first matches)]+ (is (some? m))+ (is (= 2 (count (:input-columns m))))))))++;;; -------------------------------------------------- extract-sources --------------------------------------------------++(defn- simple-orders-query+ "Build a simple query against the orders table."+ []+ (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id :orders)))))++(defn- joined-orders-products-query+ "Build a LEFT JOIN query between orders and products."+ []+ (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/join (-> (lib/join-clause+ (lib.metadata/table mp (mt/id :products))+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))+ (-> (lib.metadata/field mp (mt/id :products :id))+ (lib/with-join-alias "Products")))])+ (lib/with-join-alias "Products")+ (lib/with-join-fields :all))))))++(defn- native-orders-query+ "Build a native-only pMBQL query selecting from the orders table."+ []+ {:lib/type :mbql/query+ :lib/metadata (mt/metadata-provider)+ :database (mt/id)+ :stages [{:lib/type :mbql.stage/native+ :native "SELECT * FROM ORDERS"}]})++(deftest extract-sources-mbql-test+ (testing "MBQL: extracts source table from simple query"+ (let [transform {:source {:type :query :query (simple-orders-query)}+ :name "test"}+ sources (context/extract-sources transform)]+ (is (seq sources))+ (is (= 1 (count sources)))+ (let [s (first sources)]+ (is (= (mt/id :orders) (:table-id s)))+ (is (string? (:table-name s)))+ (is (= (mt/id) (:db-id s)))))))++(deftest extract-sources-mbql-with-joins-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (testing "MBQL: extracts multiple source tables from joined query"+ (let [transform {:source {:type :query :query (joined-orders-products-query)}+ :name "test"}+ sources (context/extract-sources transform)+ table-ids (set (map :table-id sources))]+ (is (>= (count sources) 2))+ (is (contains? table-ids (mt/id :orders)))+ (is (contains? table-ids (mt/id :products)))))))++(deftest extract-sources-native-test+ (testing "Native: extracts source tables from SQL query"+ (let [transform {:source {:type :query :query (native-orders-query)}+ :name "test"}+ sources (context/extract-sources transform)]+ (is (seq sources))+ (is (= (mt/id :orders) (:table-id (first sources))))+ (is (= (mt/id) (:db-id (first sources)))))))++(deftest extract-sources-native-with-joins-test+ (testing "Native: extracts multiple source tables from SQL with JOIN"+ (let [transform {:source {:type :query+ :query {:lib/type :mbql/query+ :lib/metadata (mt/metadata-provider)+ :database (mt/id)+ :stages [{:lib/type :mbql.stage/native+ :native "SELECT * FROM ORDERS JOIN PRODUCTS ON ORDERS.PRODUCT_ID = PRODUCTS.ID"}]}}+ :name "test"}+ sources (context/extract-sources transform)+ table-ids (set (map :table-id sources))]+ (is (>= (count sources) 2))+ (is (contains? table-ids (mt/id :orders)))+ (is (contains? table-ids (mt/id :products))))))++(deftest extract-sources-python-test+ (testing "Python: extracts source tables from source-tables map"+ (let [transform {:source {:type :python+ :source-tables {"orders" (mt/id :orders)}+ :source-database (mt/id)}+ :name "test"}+ sources (context/extract-sources transform)]+ (is (seq sources))+ (is (= (mt/id :orders) (:table-id (first sources))))+ (is (= (mt/id) (:db-id (first sources)))))))++(deftest extract-sources-python-multiple-tables-test+ (testing "Python: extracts multiple source tables"+ (let [transform {:source {:type :python+ :source-tables {"orders" (mt/id :orders)+ "products" (mt/id :products)}+ :source-database (mt/id)}+ :name "test"}+ sources (context/extract-sources transform)+ table-ids (set (map :table-id sources))]+ (is (= 2 (count sources)))+ (is (contains? table-ids (mt/id :orders)))+ (is (contains? table-ids (mt/id :products))))))++;;; -------------------------------------------------- build-context integration --------------------------------------------------++(deftest build-context-mbql-sources-test+ (testing "build-context extracts source tables from MBQL query"+ (let [transform {:source {:type :query+ :query (simple-orders-query)}+ :target {:schema "nonexistent" :name "nonexistent_table" :type :table}+ :name "test"}+ ctx (context/build-context transform)]+ (is (= :mbql (:source-type ctx)))+ (is (seq (:sources ctx)))+ (is (= (mt/id :orders) (:table_id (first (:sources ctx)))))+ (testing "source fields are populated"+ (is (seq (:fields (first (:sources ctx)))))))))++(deftest build-context-not-run-test+ (testing "build-context returns nil target when target table doesn't exist"+ (let [transform {:source {:type :query+ :query (simple-orders-query)}+ :target {:schema "nonexistent" :name "nonexistent_table" :type :table}+ :name "test"}+ ctx (context/build-context transform)]+ (is (nil? (:target ctx)))+ (is (false? (:has-column-matches? ctx))))))++(deftest build-context-with-joins-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (testing "build-context extracts join structure from MBQL query with LEFT JOIN"+ (let [transform {:source {:type :query+ :query (joined-orders-products-query)}+ :target {:schema "nonexistent" :name "nonexistent_table" :type :table}+ :name "test"}+ ctx (context/build-context transform)]+ (is (true? (:has-joins? ctx)))+ (is (= 1 (count (:join-structure (:mbql-context ctx)))))+ (let [join (first (:join-structure (:mbql-context ctx)))]+ (is (= :left-join (:strategy join)))+ (is (= "Products" (:alias join)))+ (is (= (mt/id :products) (:source-table join))))+ (testing "visited-fields includes join fields"+ (is (seq (get-in ctx [:visited-fields :all]))))+ (testing "multiple source tables detected"+ (is (>= (count (:sources ctx)) 2)))))))++(deftest build-context-column-matches-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (testing "build-context populates column-matches when target exists"+ ;; Use products table as a fake target to avoid running a transform+ (let [products-table (t2/select-one :model/Table (mt/id :products))+ transform {:source {:type :query+ :query (joined-orders-products-query)}+ :target {:schema (:schema products-table)+ :name (:name products-table)+ :type :table}+ :name "test"}+ ctx (context/build-context transform)]+ (is (some? (:target ctx)))+ (is (true? (:has-column-matches? ctx)))+ (is (seq (:column-matches ctx)))+ (testing "column matches have expected structure"+ (let [m (first (:column-matches ctx))]+ (is (map? (:output-field m)))+ (is (string? (:name (:output-field m))))+ (is (seq (:input-columns m)))))))))diff --git a/test/metabase/transforms_inspector/core_test.clj b/test/metabase/transforms_inspector/core_test.cljnew file mode 100644index 00000000000..5aa1bc9967e--- /dev/null+++ b/test/metabase/transforms_inspector/core_test.clj@@ -0,0 +1,113 @@+(ns metabase.transforms-inspector.core-test+ (:require+ [clojure.test :refer :all]+ [metabase.transforms-inspector.core :as inspector.core]))++;;; -------------------------------------------------- compute-card-result --------------------------------------------------++(deftest compute-card-result-test+ (testing "join-analysis join_step computes null rate"+ (is (= {"output_count" 100 "matched_count" 80 "null_count" 20 "null_rate" 1/5}+ (inspector.core/compute-card-result+ :join-analysis+ {:metadata {:card_type "join_step"}}+ [100 80]))))+ (testing "nil row returns no_data with zero counts"+ (is (= {"no_data" true "output_count" 0 "matched_count" 0 "null_count" 0 "null_rate" nil}+ (inspector.core/compute-card-result+ :join-analysis+ {:metadata {:card_type "join_step"}}+ nil)))))++;;; -------------------------------------------------- triggered-alerts --------------------------------------------------++(deftest triggered-alerts-test+ (let [triggers [{:id "alert-1"+ :condition {:name :high-null-rate :card_id "step-1"}+ :severity :warning+ :message "High null rate"}+ {:id "alert-2"+ :condition {:name :high-null-rate :card_id "step-2"}+ :severity :warning+ :message "High null rate"}]]+ (testing "returns alerts whose conditions are met"+ (let [card-results {"step-1" {"null_rate" 0.3}+ "step-2" {"null_rate" 0.1}}+ result (inspector.core/triggered-alerts card-results triggers)]+ (is (= 1 (count result)))+ (is (= "alert-1" (:id (first result))))))+ (testing "returns all matching alerts"+ (let [card-results {"step-1" {"null_rate" 0.3}+ "step-2" {"null_rate" 0.5}}+ result (inspector.core/triggered-alerts card-results triggers)]+ (is (= 2 (count result)))))+ (testing "returns empty when no conditions met"+ (let [card-results {"step-1" {"null_rate" 0.0}+ "step-2" {"null_rate" 0.0}}]+ (is (= [] (inspector.core/triggered-alerts card-results triggers)))))))++;;; -------------------------------------------------- triggered-drill-lenses --------------------------------------------------++(deftest triggered-drill-lenses-test+ (let [triggers [{:lens_id "unmatched-rows"+ :condition {:name :has-unmatched-rows :card_id "step-1"}+ :params {:join_step 1}+ :reason "Unmatched rows"}+ {:lens_id "unmatched-rows"+ :condition {:name :has-unmatched-rows :card_id "step-2"}+ :params {:join_step 2}+ :reason "Unmatched rows"}]]+ (testing "returns drill lenses whose conditions are met"+ (let [card-results {"step-1" {"null_rate" 0.1}+ "step-2" {"null_rate" 0.01}}+ result (inspector.core/triggered-drill-lenses card-results triggers)]+ (is (= 1 (count result)))+ (is (= {:join_step 1} (:params (first result))))))))++;;; -------------------------------------------------- evaluate-triggers --------------------------------------------------++(deftest evaluate-triggers-test+ (let [lens {:alert_triggers [{:id "alert-1"+ :condition {:name :high-null-rate :card_id "step-1"}+ :severity :warning+ :message "High null rate"}]+ :drill_lens_triggers [{:lens_id "unmatched-rows"+ :condition {:name :has-unmatched-rows :card_id "step-1"}+ :params {:join_step 1}+ :reason "Unmatched rows"}]}+ card-results {"step-1" {"null_rate" 0.3}}]+ (testing "evaluates both alert and drill lens triggers"+ (let [result (inspector.core/evaluate-triggers lens card-results)]+ (is (= 1 (count (:alerts result))))+ (is (= 1 (count (:drill_lenses result))))))))++(deftest evaluate-triggers-no-triggers-test+ (let [lens {:alert_triggers [] :drill_lens_triggers []}+ card-results {"step-1" {"null_rate" 0.3}}]+ (testing "returns empty collections when no triggers defined"+ (let [result (inspector.core/evaluate-triggers lens card-results)]+ (is (= [] (:alerts result)))+ (is (= [] (:drill_lenses result)))))))++;;; -------------------------------------------------- interesting-fields --------------------------------------------------++(deftest interesting-fields-test+ (testing "filters out dominated fields and keeps interesting ones"+ (let [fields [{:name "id" :semantic_type :type/PK :base_type :type/Integer}+ {:name "created" :base_type :type/DateTime}]+ result (inspector.core/interesting-fields fields {})]+ (is (= 1 (count result)))+ (is (= "created" (:name (first result)))))))++;;; -------------------------------------------------- degenerate? --------------------------------------------------++(deftest degenerate-test+ (testing "no_data is degenerate"+ (is (= {:degenerate? true :reason :no-data}+ (inspector.core/degenerate? "c1" :bar {"c1" {"no_data" true}}))))+ (testing ":hidden display is always degenerate"+ (is (= {:degenerate? true}+ (inspector.core/degenerate? "c1" :hidden {"c1" {"row_count" 100}}))))+ (testing "normal data is not degenerate"+ (is (= {:degenerate? false}+ (inspector.core/degenerate? "c1" :bar {"c1" {"row_count" 10}})))))diff --git a/test/metabase/transforms_inspector/degeneracy_test.clj b/test/metabase/transforms_inspector/degeneracy_test.cljnew file mode 100644index 00000000000..cf787482820--- /dev/null+++ b/test/metabase/transforms_inspector/degeneracy_test.clj@@ -0,0 +1,34 @@+(ns metabase.transforms-inspector.degeneracy-test+ (:require+ [clojure.test :refer :all]+ [metabase.transforms-inspector.degeneracy :as degeneracy]))++(deftest degenerate-for-display-default-test+ (testing "non-degenerate when data present"+ (is (= {:degenerate? false}+ (degeneracy/degenerate-for-display? "card-1" :bar+ {"card-1" {"row_count" 10}}))))+ (testing "degenerate when no_data flag is true"+ (is (= {:degenerate? true :reason :no-data}+ (degeneracy/degenerate-for-display? "card-1" :bar+ {"card-1" {"no_data" true}}))))+ (testing "degenerate when row_count is zero"+ (is (= {:degenerate? true :reason :no-data}+ (degeneracy/degenerate-for-display? "card-1" :line+ {"card-1" {"row_count" 0}}))))+ (testing "not-degenerate when card not in results"+ (is (= {:degenerate? false}+ (degeneracy/degenerate-for-display? "card-1" :scalar {}))))+ (testing "non-degenerate for card with empty result map"+ (is (= {:degenerate? false}+ (degeneracy/degenerate-for-display? "card-1" :bar+ {"card-1" {}})))))++(deftest degenerate-for-display-hidden-test+ (testing ":hidden display type is always degenerate"+ (is (= {:degenerate? true}+ (degeneracy/degenerate-for-display? "card-1" :hidden+ {"card-1" {"row_count" 100}}))))+ (testing ":hidden is degenerate even with no results"+ (is (= {:degenerate? true}+ (degeneracy/degenerate-for-display? "card-1" :hidden {})))))diff --git a/test/metabase/transforms_inspector/e2e_test.clj b/test/metabase/transforms_inspector/e2e_test.cljnew file mode 100644index 00000000000..93e188fef17--- /dev/null+++ b/test/metabase/transforms_inspector/e2e_test.clj@@ -0,0 +1,253 @@+(ns ^:mb/driver-tests metabase.transforms-inspector.e2e-test+ "End-to-end tests for the Transform Inspector loop:+ discover → get-lens → execute cards → compute results → evaluate triggers → drill."+ (:require+ [clojure.test :refer :all]+ [metabase.driver :as driver]+ [metabase.driver.sql.util :as sql.u]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.transforms-inspector.core :as inspector]+ [metabase.transforms.execute :as transforms.execute]+ [metabase.transforms.test-util :as transforms.tu]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Helpers --------------------------------------------------++(defn- default-schema []+ (t2/select-one-fn :schema :model/Table :id (mt/id :orders)))++(defn- execute-card+ "Execute a card's dataset_query and return the first row."+ [card]+ (let [result (mt/process-query (:dataset_query card))]+ (first (mt/rows result))))++(defn- execute-all-cards+ "Execute all cards in a lens and compute card results.+ Returns a map of card-id -> result map."+ [lens-id lens]+ (into {}+ (map (fn [card]+ (let [row (execute-card card)+ result (inspector/compute-card-result+ (keyword lens-id) card row)]+ [(:id card) result]))+ (:cards lens))))++(defn- make-transform+ "Create a transform map for testing."+ [query target-name schema]+ {:source {:type :query+ :query query}+ :name (str "inspector_e2e_" target-name)+ :target {:schema schema+ :name target-name+ :type :table}})++;;; -------------------------------------------------- Query builders --------------------------------------------------++(defn- mbql-multi-join-query+ "ORDERS INNER JOIN PEOPLE, LEFT JOIN PRODUCTS, LEFT JOIN REVIEWS.+ The 3rd LEFT JOIN (reviews on product_id) produces unmatched rows+ since not every order's product has a review."+ []+ (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/join (-> (lib/join-clause+ (lib.metadata/table mp (mt/id :people))+ [(lib/= (lib.metadata/field mp (mt/id :orders :user_id))+ (-> (lib.metadata/field mp (mt/id :people :id))+ (lib/with-join-alias "People")))]+ :inner-join)+ (lib/with-join-alias "People")+ (lib/with-join-fields :all)))+ (lib/join (-> (lib/join-clause+ (lib.metadata/table mp (mt/id :products))+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))+ (-> (lib.metadata/field mp (mt/id :products :id))+ (lib/with-join-alias "Products")))])+ (lib/with-join-alias "Products")+ (lib/with-join-fields :all)))+ (lib/join (-> (lib/join-clause+ (lib.metadata/table mp (mt/id :reviews))+ [(lib/= (lib.metadata/field mp (mt/id :orders :id))+ (-> (lib.metadata/field mp (mt/id :reviews :id))+ (lib/with-join-alias "Reviews")))])+ (lib/with-join-alias "Reviews")+ (lib/with-join-fields :all))))))++(defn- qt+ "Quote a table name for the current driver, with schema qualification.+ Uses actual physical table name from metadata."+ [s]+ (let [{:keys [name schema]} (t2/select-one [:model/Table :name :schema] :id (mt/id (keyword s)))]+ (sql.u/quote-name driver/*driver* :table schema name)))++(defn- qf+ "Quote a field/column name for the current driver.+ Uses actual physical column name from metadata."+ [table-key col-key]+ (let [col-name (t2/select-one-fn :name :model/Field :id (mt/id table-key col-key))]+ (sql.u/quote-name driver/*driver* :field col-name)))++(defn- native-multi-join-query+ "Native SQL: ORDERS INNER JOIN PEOPLE, LEFT JOIN PRODUCTS, LEFT JOIN REVIEWS."+ []+ (lib/native-query+ (mt/metadata-provider)+ (str "SELECT o.*, "+ "p." (qf :people :name) " AS person_name, "+ "p." (qf :people :state) " AS person_state, "+ "pr." (qf :products :title) " AS product_title, "+ "pr." (qf :products :category) " AS product_category, "+ "pr." (qf :products :price) " AS product_price, "+ "r." (qf :reviews :rating) " AS review_rating, "+ "r." (qf :reviews :body) " AS review_body "+ "FROM " (qt "orders") " o "+ "JOIN " (qt "people") " p ON o." (qf :orders :user_id) " = p." (qf :people :id) " "+ "LEFT JOIN " (qt "products") " pr ON o." (qf :orders :product_id) " = pr." (qf :products :id) " "+ "LEFT JOIN " (qt "reviews") " r ON o." (qf :orders :id) " = r." (qf :reviews :id))))++;;; -------------------------------------------------- Full loop helper --------------------------------------------------++(defn- run-inspector-loop+ "Run the full inspector loop for a transform:+ 1. Discover available lenses+ 2. For each applicable lens: get → execute cards → compute results → evaluate triggers+ 3. For any fired drill triggers: get drill lens → execute its cards+ Returns a map of results for assertions."+ [transform]+ (let [discovery (inspector/discover-lenses transform)]+ (when (= :ready (:status discovery))+ (let [lens-ids (map :id (:available_lenses discovery))]+ {:discovery discovery+ :lenses+ (into {}+ (for [lens-id lens-ids]+ (let [lens (inspector/get-lens transform lens-id nil)+ card-results (execute-all-cards lens-id lens)+ triggers (inspector/evaluate-triggers lens card-results)+ drill-results+ (into []+ (for [{:keys [lens_id params]} (:drill_lenses triggers)]+ (let [drill-lens (inspector/get-lens transform lens_id params)+ drill-card-results (execute-all-cards lens_id drill-lens)]+ {:lens_id lens_id+ :params params+ :card-count (count (:cards drill-lens))+ :card-results drill-card-results})))]+ [lens-id {:lens lens+ :card-results card-results+ :triggers triggers+ :drills drill-results}])))}))))++(defn- join-step-result+ "Get the card result for join step N from a join-analysis lens result."+ [{:keys [lens card-results]} step]+ (let [card (some #(when (= step (get-in % [:metadata :join_step])) %) (:cards lens))]+ (get card-results (:id card))))++;;; -------------------------------------------------- Shared assertions --------------------------------------------------++(defn- assert-inspector-loop+ "Shared assertions for both MBQL and native e2e tests.+ source-type is :mbql or :native — drill lens assertions are MBQL-only+ since unmatched-rows lens requires preprocessed MBQL query."+ [result source-type]+ (testing "discovery succeeds with join-analysis available"+ (is (= :ready (get-in result [:discovery :status])))+ (is (>= (count (get-in result [:discovery :available_lenses])) 2))+ (is (some #(= "join-analysis" (:id %))+ (get-in result [:discovery :available_lenses]))))++ (testing "generic-summary cards return data"+ (let [{:keys [card-results]} (get-in result [:lenses "generic-summary"])]+ (is (seq card-results))+ (is (every? (fn [[_id r]] (not (get r "no_data"))) card-results))))++ (let [ja (get-in result [:lenses "join-analysis"])]+ (testing "join-analysis has 3 join steps"+ (let [step-cards (filter #(re-matches #"join-step-\d+" (:id %))+ (get-in ja [:lens :cards]))]+ (is (= 3 (count step-cards)))))++ (testing "all join-step cards return output_count and matched_count"+ (doseq [step [1 2 3]+ :let [r (join-step-result ja step)]]+ (is (some? r) (str "join step " step " should have a result"))+ (when r+ (is (pos? (get r "output_count" 0))+ (str "step " step " should have rows"))+ (is (number? (get r "matched_count"))+ (str "step " step " should have matched_count")))))++ (testing "INNER JOIN (step 1) has 0% null rate"+ (let [r (join-step-result ja 1)]+ (when r+ (is (= 0 (get r "null_count"))+ "INNER JOIN should have no unmatched rows"))))++ (testing "LEFT JOIN reviews (step 3) has non-zero null rate"+ (let [r (join-step-result ja 3)]+ (when r+ (is (pos? (get r "null_count"))+ "LEFT JOIN reviews should have unmatched rows")+ (is (pos? (get r "null_rate"))+ "null_rate should be positive"))))++ (testing "triggers fire for LEFT JOINs with unmatched rows"+ (let [{:keys [triggers]} ja]+ (is (seq (:alerts triggers))+ "should have at least one alert")+ ;; drill lens triggers are filtered by applicability;+ ;; unmatched-rows is MBQL-only so native won't have drill triggers+ (when (= source-type :mbql)+ (is (seq (:drill_lenses triggers))+ "should have at least one drill lens trigger"))))++ (when (= source-type :mbql)+ (testing "unmatched-rows drill lens fires and its cards execute"+ (let [{:keys [drills]} ja]+ (is (seq drills) "should have drill results")+ (doseq [{:keys [lens_id card-count card-results]} drills]+ (is (= "unmatched-rows" lens_id))+ (is (pos? card-count) "drill lens should have cards")+ (is (seq card-results) "drill lens cards should produce results")))))))++;;; -------------------------------------------------- MBQL E2E --------------------------------------------------++(deftest mbql-full-inspector-loop-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table :left-join)+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_insp_e2e_mbql"}]+ (let [query (mbql-multi-join-query)+ transform-data (make-transform query target-name (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ (let [transform' (t2/select-one :model/Transform tid)+ result (run-inspector-loop transform')]+ (assert-inspector-loop result :mbql)))))))++;;; -------------------------------------------------- Native E2E --------------------------------------------------++(deftest native-full-inspector-loop-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table :left-join)+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_insp_e2e_nat"}]+ (let [query (native-multi-join-query)+ transform-data (make-transform query target-name (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ (let [transform' (t2/select-one :model/Transform tid)+ result (run-inspector-loop transform')]+ (assert-inspector-loop result :native)))))))diff --git a/test/metabase/transforms_inspector/interestingness_test.clj b/test/metabase/transforms_inspector/interestingness_test.cljnew file mode 100644index 00000000000..784f0937ee2--- /dev/null+++ b/test/metabase/transforms_inspector/interestingness_test.clj@@ -0,0 +1,138 @@+(ns metabase.transforms-inspector.interestingness-test+ (:require+ [clojure.test :refer :all]+ [metabase.transforms-inspector.interestingness :as interestingness]))++;;; -------------------------------------------------- dominated-column? --------------------------------------------------++(deftest dominated-column-pk-test+ (is (true? (interestingness/dominated-column?+ {:name "id" :semantic_type :type/PK :base_type :type/Integer}))))++(deftest dominated-column-fk-test+ (is (true? (interestingness/dominated-column?+ {:name "user_id" :semantic_type :type/FK :base_type :type/Integer}))))++(deftest dominated-column-uuid-test+ (is (true? (interestingness/dominated-column?+ {:name "uuid" :semantic_type :type/UUID :base_type :type/Text}))))++(deftest dominated-column-serialized-json-test+ (is (true? (interestingness/dominated-column?+ {:name "data" :semantic_type :type/SerializedJSON :base_type :type/Text}))))++(deftest dominated-column-name-pattern-test+ (testing "name patterns that indicate dominated columns"+ (is (true? (interestingness/dominated-column?+ {:name "user_id" :base_type :type/Integer})))+ (is (true? (interestingness/dominated-column?+ {:name "ID" :base_type :type/Integer})))+ (is (true? (interestingness/dominated-column?+ {:name "order_uuid" :base_type :type/Text})))+ (is (true? (interestingness/dominated-column?+ {:name "UUID" :base_type :type/Text}))))+ (testing "name patterns that are not dominated"+ (is (not (interestingness/dominated-column?+ {:name "description" :base_type :type/Text})))+ (is (not (interestingness/dominated-column?+ {:name "video_length" :base_type :type/Integer})))+ (is (not (interestingness/dominated-column?+ {:name "identity" :base_type :type/Text})))))++;;; -------------------------------------------------- score-field --------------------------------------------------++(deftest score-field-dominated-test+ (testing "PK fields get score 0.0"+ (let [result (interestingness/score-field {:name "id" :semantic_type :type/PK :base_type :type/Integer})]+ (is (= 0.0 (:score result)))+ (is (true? (:dominated? result)))+ (is (= [:dominated-column] (:reasons result))))))++(deftest score-field-mostly-null-test+ (testing "mostly-null fields get score 0.1"+ (let [result (interestingness/score-field {:name "notes" :base_type :type/Text+ :stats {:nil_percent 0.95}})]+ (is (= 0.1 (:score result)))+ (is (not (:dominated? result)))+ (is (= [:mostly-null] (:reasons result))))))++(deftest score-field-high-cardinality-test+ (testing "high-cardinality without semantic type gets 0.2"+ (let [result (interestingness/score-field {:name "email" :base_type :type/Text+ :stats {:distinct_count 5000}})]+ (is (= 0.2 (:score result)))+ (is (= [:high-cardinality] (:reasons result)))))+ (testing "high-cardinality with high-interest semantic type does not get 0.2"+ (let [result (interestingness/score-field {:name "city" :base_type :type/Text+ :semantic_type :type/City+ :stats {:distinct_count 5000}})]+ (is (= 0.85 (:score result))))))++(deftest score-field-temporal-test+ (testing "temporal fields get high score"+ (let [result (interestingness/score-field {:name "created_at" :base_type :type/DateTime})]+ (is (= 0.9 (:score result)))+ (is (not (:dominated? result)))+ (is (some #{:temporal} (:reasons result))))))++(deftest score-field-categorical-test+ (testing "low-cardinality text fields score as categorical"+ (let [result (interestingness/score-field {:name "status" :base_type :type/Text+ :stats {:distinct_count 5}})]+ (is (= 0.8 (:score result)))+ (is (some #{:categorical} (:reasons result))))))++(deftest score-field-numeric-with-variance-test+ (testing "numeric fields with variance get decent score"+ (let [result (interestingness/score-field {:name "amount" :base_type :type/Float+ :stats {:min 0.0 :max 100.0}})]+ (is (= 0.75 (:score result)))+ (is (some #{:numeric-variance} (:reasons result))))))++(deftest score-field-high-interest-semantic-type-test+ (testing "fields with high-interest semantic types score well"+ (let [result (interestingness/score-field {:name "price" :base_type :type/Float+ :semantic_type :type/Price})]+ (is (= 0.85 (:score result)))+ (is (some #{:high-interest-semantic-type} (:reasons result))))))++(deftest score-field-default-test+ (testing "plain field with no special attributes gets default score"+ (let [result (interestingness/score-field {:name "foo" :base_type :type/Text})]+ (is (= 0.5 (:score result)))+ (is (= [:default] (:reasons result))))))++;;; -------------------------------------------------- interesting-fields --------------------------------------------------++(deftest interesting-fields-filters-by-threshold-test+ (let [fields [{:name "id" :semantic_type :type/PK :base_type :type/Integer}+ {:name "name" :base_type :type/Text}+ {:name "notes" :base_type :type/Text :stats {:nil_percent 0.95}}+ {:name "amount" :base_type :type/Float :stats {:min 0.0 :max 100.0}}+ {:name "date" :base_type :type/DateTime}]+ result (interestingness/interesting-fields fields {})]+ (testing "dominated (PK) and mostly-null fields are excluded"+ (is (not-any? #(= "id" (:name %)) result))+ (is (not-any? #(= "notes" (:name %)) result)))+ (testing "interesting fields are included and sorted by score descending"+ (let [names (mapv :name result)]+ (is (= "date" (first names)))+ (is (some #{"amount"} names))+ (is (some #{"name"} names))))))++(deftest interesting-fields-limit-test+ (let [fields [{:name "a" :base_type :type/DateTime}+ {:name "b" :base_type :type/Float :stats {:min 0.0 :max 10.0}}+ {:name "c" :base_type :type/Text :stats {:distinct_count 3}}+ {:name "d" :base_type :type/Text}]+ result (interestingness/interesting-fields fields {:limit 2})]+ (is (= 2 (count result)))))++(deftest interesting-fields-custom-threshold-test+ (let [fields [{:name "a" :base_type :type/Text}]+ result-low (interestingness/interesting-fields fields {:threshold 0.1})+ result-high (interestingness/interesting-fields fields {:threshold 0.6})]+ (testing "with low threshold, default-scored fields pass"+ (is (= 1 (count result-low))))+ (testing "with high threshold, default-scored fields are excluded"+ (is (= 0 (count result-high))))))diff --git a/test/metabase/transforms_inspector/lens/core_test.clj b/test/metabase/transforms_inspector/lens/core_test.cljnew file mode 100644index 00000000000..74645d3c9ba--- /dev/null+++ b/test/metabase/transforms_inspector/lens/core_test.clj@@ -0,0 +1,165 @@+(ns metabase.transforms-inspector.lens.core-test+ (:require+ [clojure.test :refer :all]+ [metabase.transforms-inspector.lens.core :as lens.core]))++;;; -------------------------------------------------- lens-id->type --------------------------------------------------++(deftest lens-id->type-test+ (is (= :join-analysis (lens.core/lens-id->type "join-analysis")))+ (is (= :generic-summary (lens.core/lens-id->type "generic-summary"))))++;;; -------------------------------------------------- params->id-suffix --------------------------------------------------++(deftest params->id-suffix-empty-test+ (testing "nil params returns empty string"+ (is (= "" (lens.core/params->id-suffix nil))))+ (testing "empty map returns empty string"+ (is (= "" (lens.core/params->id-suffix {})))))++(deftest params->id-suffix-single-param-test+ (is (= "@join_step=1" (lens.core/params->id-suffix {:join_step 1}))))++(deftest params->id-suffix-multiple-params-test+ (testing "params are sorted by key"+ (is (= "@a=1,b=2" (lens.core/params->id-suffix {:b 2 :a 1})))))++;;; -------------------------------------------------- make-card-id --------------------------------------------------++(deftest make-card-id-no-params-test+ (is (= "base-count" (lens.core/make-card-id "base-count" nil)))+ (is (= "base-count" (lens.core/make-card-id "base-count" {}))))++(deftest make-card-id-with-params-test+ (is (= "join-step-1@join_step=1"+ (lens.core/make-card-id "join-step-1" {:join_step 1}))))++;;; -------------------------------------------------- with-metadata --------------------------------------------------++(deftest with-metadata-test+ (testing "merges lens metadata into lens map"+ ;; Use the :default method which returns a basic metadata map+ (let [result (lens.core/with-metadata :some-unknown-type {} {:cards [] :sections []})]+ (is (contains? result :id))+ (is (contains? result :display_name))+ (is (contains? result :cards))+ (is (contains? result :sections)))))++;;; -------------------------------------------------- lens-applicable? --------------------------------------------------++(deftest lens-applicable-default-test+ (testing "default method returns false for unknown lens types"+ (is (not (lens.core/lens-applicable? :nonexistent-lens {})))))++(deftest generic-summary-always-applicable-test+ (testing "generic-summary is always applicable"+ (is (true? (lens.core/lens-applicable? :generic-summary {})))+ (is (true? (lens.core/lens-applicable? :generic-summary {:has-joins? false})))))++(deftest join-analysis-applicability-test+ (testing "join-analysis applicable when has-joins? is true"+ (is (true? (lens.core/lens-applicable? :join-analysis {:has-joins? true}))))+ (testing "join-analysis not applicable without joins"+ (is (not (lens.core/lens-applicable? :join-analysis {:has-joins? false})))+ (is (not (lens.core/lens-applicable? :join-analysis {})))))++(deftest column-comparison-applicability-test+ (testing "column-comparison applicable when has-column-matches? is true"+ (is (true? (lens.core/lens-applicable? :column-comparison {:has-column-matches? true}))))+ (testing "column-comparison not applicable without column matches"+ (is (not (lens.core/lens-applicable? :column-comparison {:has-column-matches? false})))+ (is (not (lens.core/lens-applicable? :column-comparison {})))))++(deftest unmatched-rows-applicability-test+ (testing "unmatched-rows applicable with MBQL, joins, preprocessed query, and outer joins"+ (is (true? (lens.core/lens-applicable? :unmatched-rows+ {:source-type :mbql+ :has-joins? true+ :mbql-context {:preprocessed-query {:stages [{}]}+ :join-structure [{:strategy :left-join}]}}))))+ (testing "unmatched-rows not applicable for native queries"+ (is (not (lens.core/lens-applicable? :unmatched-rows+ {:source-type :native+ :has-joins? true+ :native-context {:from-table [:raw "t"]+ :join-structure [{:strategy :left-join}]}}))))+ (testing "unmatched-rows not applicable without outer joins"+ (is (not (lens.core/lens-applicable? :unmatched-rows+ {:source-type :mbql+ :has-joins? true+ :mbql-context {:preprocessed-query {:stages [{}]}+ :join-structure [{:strategy :inner-join}]}}))))+ (testing "unmatched-rows not applicable without joins"+ (is (not (lens.core/lens-applicable? :unmatched-rows+ {:source-type :mbql+ :has-joins? false})))))++;;; -------------------------------------------------- lens-metadata --------------------------------------------------++(deftest generic-summary-metadata-test+ (let [meta (lens.core/lens-metadata :generic-summary {})]+ (is (= "generic-summary" (:id meta)))+ (is (= "Data Summary" (:display_name meta)))+ (is (= {:level :fast} (:complexity meta)))))++(deftest join-analysis-metadata-test+ (let [meta (lens.core/lens-metadata :join-analysis {})]+ (is (= "join-analysis" (:id meta)))+ (is (= "Join Analysis" (:display_name meta)))+ (is (= {:level :very-slow} (:complexity meta)))))++(deftest column-comparison-metadata-test+ (let [meta (lens.core/lens-metadata :column-comparison {})]+ (is (= "column-comparison" (:id meta)))+ (is (= "Column Distributions" (:display_name meta)))+ (is (= {:level :slow} (:complexity meta)))))++(deftest unmatched-rows-metadata-test+ (let [meta (lens.core/lens-metadata :unmatched-rows {})]+ (is (= "unmatched-rows" (:id meta)))+ (is (= "Unmatched Rows" (:display_name meta)))+ (is (= {:level :slow} (:complexity meta)))))++(deftest default-metadata-test+ (let [meta (lens.core/lens-metadata :nonexistent {})]+ (is (= "nonexistent" (:id meta)))+ (is (nil? (:description meta)))))++;;; -------------------------------------------------- available-lenses --------------------------------------------------++(deftest available-lenses-includes-generic-test+ (testing "generic-summary always appears in available lenses"+ (let [lenses (lens.core/available-lenses {})]+ (is (some #(= "generic-summary" (:id %)) lenses)))))++(deftest available-lenses-ordering-test+ (testing "generic-summary comes first (lowest priority)"+ (let [lenses (lens.core/available-lenses {:has-joins? true+ :has-column-matches? true})]+ (is (= "generic-summary" (:id (first lenses)))))))++(deftest available-lenses-excludes-drill-lenses-test+ (testing "drill lenses (unmatched-rows) are not in discovery list"+ (let [lenses (lens.core/available-lenses {:source-type :mbql+ :has-joins? true+ :mbql-context {:preprocessed-query {:stages [{}]}+ :join-structure [{:strategy :left-join}]}})]+ (is (not-any? #(= "unmatched-rows" (:id %)) lenses)))))++;;; -------------------------------------------------- get-lens --------------------------------------------------++(deftest get-lens-throws-for-inapplicable-test+ (testing "throws when lens is not applicable"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Lens data not available"+ (lens.core/get-lens {:has-joins? false} "join-analysis")))))++;;; -------------------------------------------------- register-lens! --------------------------------------------------++(deftest register-lens-idempotent-test+ (testing "re-registering same lens replaces previous entry (no duplicates)"+ (lens.core/register-lens! :test-lens-123 50)+ (lens.core/register-lens! :test-lens-123 60)+ (let [lenses (lens.core/available-lenses {})]+ ;; :test-lens-123 has default applicability (false), so it won't show+ ;; but we verify no errors occur+ (is (not-any? #(= "test-lens-123" (:id %)) lenses)))))diff --git a/test/metabase/transforms_inspector/lens_output_test.clj b/test/metabase/transforms_inspector/lens_output_test.cljnew file mode 100644index 00000000000..41453dba6d1--- /dev/null+++ b/test/metabase/transforms_inspector/lens_output_test.clj@@ -0,0 +1,333 @@+(ns ^:mb/driver-tests metabase.transforms-inspector.lens-output-test+ "Tests for lens card output validation and trigger flow."+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.test :as mt]+ [metabase.transforms-inspector.core :as inspector.core]+ [metabase.transforms-inspector.lens.core :as lens.core]+ [metabase.transforms.util :as transforms.util]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Test context helpers --------------------------------------------------++(defn- collect-field-metadata+ "Collect field metadata from the database for a table."+ [table-id]+ (mapv (fn [field]+ (cond-> (select-keys field [:id :name :display_name :base_type :semantic_type])+ (:fingerprint field)+ (assoc :stats (not-empty+ (merge (when-let [dc (get-in field [:fingerprint :global :distinct-count])]+ {:distinct_count dc})+ (when (some? (get-in field [:fingerprint :global :nil%]))+ {:nil_percent (get-in field [:fingerprint :global :nil%])}))))))+ (t2/select :model/Field :table_id table-id :active true)))++(defn- build-table-info+ "Build a table info map with fields from a real table."+ [table-id]+ (let [table (t2/select-one :model/Table table-id)+ fields (collect-field-metadata table-id)]+ {:table_id table-id+ :table_name (:name table)+ :schema (:schema table)+ :db_id (:db_id table)+ :column_count (count fields)+ :fields fields}))++(defn- preprocess-join-query+ "Preprocess a LEFT JOIN query between orders and products."+ []+ (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/join (-> (lib/join-clause+ (lib.metadata/table mp (mt/id :products))+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))+ (-> (lib.metadata/field mp (mt/id :products :id))+ (lib/with-join-alias "Products")))])+ (lib/with-join-alias "Products")+ (lib/with-join-fields :all)))+ transforms.util/massage-sql-query+ qp.preprocess/preprocess)))++(defn- extract-join-structure+ "Extract join structure from a preprocessed query."+ [preprocessed]+ (when-let [joins (get-in preprocessed [:stages 0 :joins])]+ (mapv (fn [join]+ {:strategy (or (:strategy join) :left-join)+ :alias (:alias join)+ :source-table (get-in join [:stages 0 :source-table])+ :conditions (:conditions join)})+ joins)))++(defn- build-joined-context+ "Build a test context using real test dataset tables with a LEFT JOIN.+ Uses products as a fake 'target' to avoid running a transform."+ []+ (let [orders-info (build-table-info (mt/id :orders))+ products-info (build-table-info (mt/id :products))+ preprocessed (preprocess-join-query)+ join-structure (extract-join-structure preprocessed)]+ {:transform {}+ :source-type :mbql+ :sources [orders-info products-info]+ :target products-info ;; use products as fake target+ :db-id (mt/id)+ :from-table-id (mt/id :orders)+ :has-joins? true+ :mbql-context {:preprocessed-query preprocessed+ :join-structure join-structure}+ :visited-fields {:all #{(mt/id :orders :product_id) (mt/id :products :id)}}+ :has-column-matches? true+ :column-matches [{:output-column (:name (t2/select-one :model/Field (mt/id :products :price)))+ :output-field {:name (:name (t2/select-one :model/Field (mt/id :products :price)))+ :id (mt/id :products :price)+ :base_type :type/Float}+ :input-columns [{:source-table-id (mt/id :orders)+ :source-table-name (:name (t2/select-one :model/Table (mt/id :orders)))+ :name "TOTAL"+ :id (mt/id :orders :total)}]}]}))++(defn- build-simple-context+ "Build a test context for a simple query (no joins) using orders table."+ []+ (let [orders-info (build-table-info (mt/id :orders))+ mp (mt/metadata-provider)+ preprocessed (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ transforms.util/massage-sql-query+ qp.preprocess/preprocess)]+ {:transform {}+ :source-type :mbql+ :sources [orders-info]+ :target orders-info ;; use same table as fake target+ :db-id (mt/id)+ :from-table-id nil+ :has-joins? false+ :mbql-context {:preprocessed-query preprocessed+ :join-structure nil}+ :visited-fields {:all #{}}+ :has-column-matches? false+ :column-matches nil}))++;;; -------------------------------------------------- Generic Summary --------------------------------------------------++(deftest generic-summary-cards-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (testing "generic-summary lens with simple query"+ (let [ctx (build-simple-context)+ lens (lens.core/make-lens :generic-summary ctx nil)]+ (testing "has correct metadata"+ (is (= "generic-summary" (:id lens)))+ (is (= "Data Summary" (:display_name lens))))+ (testing "has row-counts section with comparison layout"+ (is (= 1 (count (:sections lens))))+ (is (= :comparison (:layout (first (:sections lens))))))+ (testing "cards are scalar display"+ (is (every? #(= :scalar (:display %)) (:cards lens))))+ (testing "has input and output cards"+ (is (some #(= :input (get-in % [:metadata :group_role])) (:cards lens)))+ (is (some #(= :output (get-in % [:metadata :group_role])) (:cards lens))))+ (testing "cards have dedup_key"+ (is (every? #(some? (get-in % [:metadata :dedup_key])) (:cards lens))))+ (testing "cards have valid dataset_query"+ (is (every? #(map? (:dataset_query %)) (:cards lens))))))+ (testing "generic-summary with joined query has source cards for each table"+ (let [ctx (build-joined-context)+ lens (lens.core/make-lens :generic-summary ctx nil)]+ (testing "has input cards for both source tables"+ (let [input-cards (filter #(= :input (get-in % [:metadata :group_role])) (:cards lens))]+ (is (= 2 (count input-cards)))))))))++;;; -------------------------------------------------- Join Analysis --------------------------------------------------++(deftest join-analysis-cards-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (let [ctx (build-joined-context)+ lens (lens.core/make-lens :join-analysis ctx nil)]+ (testing "has correct metadata"+ (is (= "join-analysis" (:id lens)))+ (is (= "Join Analysis" (:display_name lens))))+ (testing "has join-stats section with flat layout"+ (is (= 1 (count (:sections lens))))+ (is (= :flat (:layout (first (:sections lens))))))+ (testing "has base-count card"+ (let [base (some #(when (= "base-count" (:id %)) %) (:cards lens))]+ (is (some? base))+ (is (= :scalar (:display base)))+ (is (= :base_count (get-in base [:metadata :card_type])))))+ (testing "has join-step card(s)"+ (let [step-cards (filter #(re-matches #"join-step-\d+" (:id %)) (:cards lens))]+ (is (= 1 (count step-cards)))+ (let [step (first step-cards)]+ (is (= :table (:display step)))+ (is (= :join_step (keyword (get-in step [:metadata :card_type]))))+ (is (= 1 (get-in step [:metadata :join_step])))+ (is (= "Products" (get-in step [:metadata :join_alias]))))))+ (testing "has table count card(s)"+ (let [table-cards (filter #(re-matches #"table-\d+-count" (:id %)) (:cards lens))]+ (is (= 1 (count table-cards)))+ (let [tc (first table-cards)]+ (is (= :scalar (:display tc)))+ (is (= :table_count (keyword (get-in tc [:metadata :card_type])))))))+ (testing "all cards have dataset_query"+ (is (every? #(map? (:dataset_query %)) (:cards lens))))+ (testing "summary includes join count"+ (is (some #(= "Joins" (:label %)) (get-in lens [:summary :highlights])))))))++(deftest join-analysis-triggers-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (let [ctx (build-joined-context)+ lens (lens.core/make-lens :join-analysis ctx nil)]+ (testing "has alert triggers for outer join"+ (is (seq (:alert_triggers lens)))+ (let [alert (first (:alert_triggers lens))]+ (is (= :warning (:severity alert)))+ (is (= :high-null-rate (get-in alert [:condition :name])))+ (is (string? (get-in alert [:condition :card_id])))))+ (testing "has drill lens triggers for outer join"+ (is (seq (:drill_lens_triggers lens)))+ (let [drill (first (:drill_lens_triggers lens))]+ (is (= "unmatched-rows" (:lens_id drill)))+ (is (= :has-unmatched-rows (get-in drill [:condition :name])))+ (is (some? (:params drill)))))+ (testing "trigger card_id references an actual card"+ (let [card-ids (set (map :id (:cards lens)))+ alert-card-ids (map #(get-in % [:condition :card_id]) (:alert_triggers lens))+ drill-card-ids (map #(get-in % [:condition :card_id]) (:drill_lens_triggers lens))]+ (is (every? card-ids alert-card-ids))+ (is (every? card-ids drill-card-ids)))))))++;;; -------------------------------------------------- Column Comparison --------------------------------------------------++(deftest column-comparison-cards-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (let [ctx (build-joined-context)+ lens (lens.core/make-lens :column-comparison ctx nil)]+ (testing "has correct metadata"+ (is (= "column-comparison" (:id lens)))+ (is (= "Column Distributions" (:display_name lens))))+ (testing "has comparisons section with comparison layout"+ (is (= 1 (count (:sections lens))))+ (is (= :comparison (:layout (first (:sections lens))))))+ (testing "cards have group metadata for comparison"+ (is (every? #(some? (get-in % [:metadata :group_id])) (:cards lens)))+ (is (every? #(contains? #{:input :output} (get-in % [:metadata :group_role])) (:cards lens))))+ (testing "cards have valid display type"+ (is (every? #(contains? #{:bar :row :line :area :pie} (:display %)) (:cards lens))))+ (testing "cards have dataset_query"+ (is (every? #(map? (:dataset_query %)) (:cards lens))))+ (testing "summary has matched columns count"+ (is (some #(= "Matched Columns" (:label %))+ (get-in lens [:summary :highlights])))))))++;;; -------------------------------------------------- Unmatched Rows --------------------------------------------------++(deftest unmatched-rows-cards-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (let [ctx (build-joined-context)+ lens (lens.core/make-lens :unmatched-rows ctx nil)]+ (testing "has correct metadata"+ (is (= "unmatched-rows" (:id lens)))+ (is (= "Unmatched Rows" (:display_name lens))))+ (testing "has samples section with flat layout"+ (is (= 1 (count (:sections lens))))+ (is (= :flat (:layout (first (:sections lens))))))+ (testing "has 3 cards for LEFT JOIN (truly-unmatched, null-source-key, orphan-rhs)"+ (is (= 3 (count (:cards lens)))))+ (testing "has truly-unmatched card(s)"+ (let [unmatched (filter #(re-matches #"truly-unmatched-\d+" (:id %)) (:cards lens))]+ (is (seq unmatched))+ (let [card (first unmatched)]+ (is (= :table (:display card)))+ (is (= :truly_unmatched (keyword (get-in card [:metadata :card_type]))))+ (is (some? (get-in card [:metadata :join_step])))+ (is (= :left-join (get-in card [:metadata :join_strategy]))))))+ (testing "has null-source-key card(s)"+ (let [null-key (filter #(re-matches #"null-source-key-\d+" (:id %)) (:cards lens))]+ (is (seq null-key))+ (let [card (first null-key)]+ (is (= :table (:display card)))+ (is (= :null_source_key (keyword (get-in card [:metadata :card_type])))))))+ (testing "has orphan-rhs card(s) for LEFT JOIN"+ (let [orphan-rhs (filter #(re-matches #"orphan-rhs-\d+" (:id %)) (:cards lens))]+ (is (seq orphan-rhs))+ (let [card (first orphan-rhs)]+ (is (= :table (:display card)))+ (is (= :orphan_rhs (keyword (get-in card [:metadata :card_type]))))+ (is (= :left-join (get-in card [:metadata :join_strategy]))))))+ (testing "has no orphan-lhs card for LEFT JOIN"+ (is (empty? (filter #(re-matches #"orphan-lhs-\d+" (:id %)) (:cards lens)))))+ (testing "cards have filter clauses in dataset_query"+ (doseq [card (:cards lens)]+ (is (seq (get-in card [:dataset_query :stages 0 :filters]))))))))++(deftest unmatched-rows-with-params-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (testing "params filter to specific join step"+ (let [ctx (build-joined-context)+ lens (lens.core/make-lens :unmatched-rows ctx {:join_step 1})]+ (testing "all cards are for join step 1"+ (is (every? #(= 1 (get-in % [:metadata :join_step])) (:cards lens))))+ (testing "card IDs include params suffix"+ (is (every? #(re-matches #".*@join_step=1" (:id %)) (:cards lens))))))))++;;; -------------------------------------------------- Trigger Evaluation Flow --------------------------------------------------++(deftest trigger-evaluation-flow-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (testing "full flow: join-analysis triggers → evaluate → drill to unmatched-rows"+ (let [ctx (build-joined-context)+ ja-lens (lens.core/make-lens :join-analysis ctx nil)+ ;; Get the card_id from the first drill lens trigger+ drill-trigger (first (:drill_lens_triggers ja-lens))+ step-card-id (get-in drill-trigger [:condition :card_id])]+ (testing "step 1: join-analysis produces triggers"+ (is (some? drill-trigger))+ (is (= "unmatched-rows" (:lens_id drill-trigger))))+ (testing "step 2: simulate high null rate card result"+ (let [card-results {step-card-id {"output_count" 100+ "matched_count" 70+ "null_count" 30+ "null_rate" 0.3}}+ eval-result (inspector.core/evaluate-triggers ja-lens card-results)]+ (testing "alerts fire for high null rate (>0.2)"+ (is (= 1 (count (:alerts eval-result)))))+ (testing "drill lens fires for unmatched rows (>0.05)"+ (is (= 1 (count (:drill_lenses eval-result))))+ (let [triggered-drill (first (:drill_lenses eval-result))]+ (is (= "unmatched-rows" (:lens_id triggered-drill)))+ (is (some? (:params triggered-drill)))+ (testing "step 3: drill lens is accessible with triggered params"+ (let [ur-lens (lens.core/get-lens ctx "unmatched-rows"+ (:params triggered-drill))]+ (is (= "unmatched-rows" (:id ur-lens)))+ (is (seq (:cards ur-lens)))))))))+ (testing "step 2b: low null rate does not trigger"+ (let [card-results {step-card-id {"output_count" 100+ "matched_count" 99+ "null_count" 1+ "null_rate" 0.01}}+ eval-result (inspector.core/evaluate-triggers ja-lens card-results)]+ (is (empty? (:alerts eval-result)))+ (is (empty? (:drill_lenses eval-result)))))))))++(deftest trigger-evaluation-no-data-test+ (testing "triggers don't fire when card results have no data"+ (let [lens {:alert_triggers [{:id "a1"+ :condition {:name :high-null-rate :card_id "step-1"}+ :severity :warning+ :message "test"}]+ :drill_lens_triggers [{:lens_id "unmatched-rows"+ :condition {:name :has-unmatched-rows :card_id "step-1"}+ :params {:join_step 1}+ :reason "test"}]}+ card-results {"step-1" {"no_data" true}}+ eval-result (inspector.core/evaluate-triggers lens card-results)]+ (is (empty? (:alerts eval-result)))+ (is (empty? (:drill_lenses eval-result))))))diff --git a/test/metabase/transforms_inspector/query_analysis_test.clj b/test/metabase/transforms_inspector/query_analysis_test.cljnew file mode 100644index 00000000000..d9c59429710--- /dev/null+++ b/test/metabase/transforms_inspector/query_analysis_test.clj@@ -0,0 +1,38 @@+(ns ^:mb/driver-tests metabase.transforms-inspector.query-analysis-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.transforms-inspector.query-analysis :as query-analysis]))++(set! *warn-on-reflection* true)++(deftest analyze-mbql-query-multi-stage-returns-nil-test+ (testing "analyze-mbql-query returns nil for multi-stage queries"+ (let [mp (mt/metadata-provider)+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ lib/append-stage)+ transform {:source {:type :query :query query}+ :name "multi-stage-test"}]+ (is (nil? (query-analysis/analyze-mbql-query transform))))))++(deftest analyze-mbql-query-single-stage-returns-result-test+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (testing "analyze-mbql-query returns result for single-stage queries"+ (let [mp (mt/metadata-provider)+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/join (-> (lib/join-clause+ (lib.metadata/table mp (mt/id :products))+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))+ (-> (lib.metadata/field mp (mt/id :products :id))+ (lib/with-join-alias "Products")))])+ (lib/with-join-alias "Products")+ (lib/with-join-fields :all))))+ transform {:source {:type :query :query query}+ :name "single-stage-test"}+ result (query-analysis/analyze-mbql-query transform)]+ (is (some? result))+ (is (some? (:preprocessed-query result)))+ (is (= 1 (count (:join-structure result))))+ (is (some? (:visited-fields result)))))))diff --git a/test/metabase/transforms_inspector/triggers_test.clj b/test/metabase/transforms_inspector/triggers_test.cljnew file mode 100644index 00000000000..1aec434550b--- /dev/null+++ b/test/metabase/transforms_inspector/triggers_test.clj@@ -0,0 +1,48 @@+(ns metabase.transforms-inspector.triggers-test+ (:require+ [clojure.test :refer :all]+ [metabase.transforms-inspector.triggers :as triggers]))++(deftest evaluate-condition-default-test+ (testing "unknown condition names return false"+ (is (not (triggers/evaluate-condition {:name :nonexistent} {})))))++(deftest evaluate-condition-high-null-rate-test+ (testing "high-null-rate triggers when null_rate > 0.2"+ (is (true? (triggers/evaluate-condition+ {:name :high-null-rate :card_id "step-1"}+ {"step-1" {"null_rate" 0.3}}))))+ (testing "high-null-rate does not trigger at exactly 0.2"+ (is (not (triggers/evaluate-condition+ {:name :high-null-rate :card_id "step-1"}+ {"step-1" {"null_rate" 0.2}}))))+ (testing "high-null-rate does not trigger below 0.2"+ (is (not (triggers/evaluate-condition+ {:name :high-null-rate :card_id "step-1"}+ {"step-1" {"null_rate" 0.1}}))))+ (testing "high-null-rate handles missing card result"+ (is (not (triggers/evaluate-condition+ {:name :high-null-rate :card_id "missing"}+ {}))))+ (testing "high-null-rate handles missing null_rate key"+ (is (not (triggers/evaluate-condition+ {:name :high-null-rate :card_id "step-1"}+ {"step-1" {}})))))++(deftest evaluate-condition-has-unmatched-rows-test+ (testing "has-unmatched-rows triggers when null_rate > 0.05"+ (is (true? (triggers/evaluate-condition+ {:name :has-unmatched-rows :card_id "step-1"}+ {"step-1" {"null_rate" 0.1}}))))+ (testing "has-unmatched-rows does not trigger at exactly 0.05"+ (is (not (triggers/evaluate-condition+ {:name :has-unmatched-rows :card_id "step-1"}+ {"step-1" {"null_rate" 0.05}}))))+ (testing "has-unmatched-rows does not trigger below 0.05"+ (is (not (triggers/evaluate-condition+ {:name :has-unmatched-rows :card_id "step-1"}+ {"step-1" {"null_rate" 0.01}}))))+ (testing "has-unmatched-rows handles missing card result"+ (is (not (triggers/evaluate-condition+ {:name :has-unmatched-rows :card_id "missing"}+ {})))))diff --git a/test/metabase/transforms_inspector_test.clj b/test/metabase/transforms_inspector_test.cljnew file mode 100644index 00000000000..54d453a0326--- /dev/null+++ b/test/metabase/transforms_inspector_test.clj@@ -0,0 +1,230 @@+(ns ^:mb/driver-tests metabase.transforms-inspector-test+ "Integration tests for the Transform Inspector API."+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [metabase.transforms-inspector.core :as inspector]+ [metabase.transforms.execute :as transforms.execute]+ [metabase.transforms.test-util :as transforms.tu]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- default-schema []+ (t2/select-one-fn :schema :model/Table :id (mt/id :orders)))++(defn- make-mbql-transform+ "Create a transform map for testing with an MBQL query."+ [query target-name & [schema]]+ {:source {:type :query+ :query query}+ :name (str "inspector_test_" target-name)+ :target {:schema (or schema (default-schema))+ :name target-name+ :type :table}})++;;; -------------------------------------------------- discover-lenses --------------------------------------------------++(deftest discover-lenses-not-run-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "discover-lenses returns :not-run when transform has no target table"+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_inspector_nr"}]+ (let [transform-data (make-mbql-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}+ target-name+ (default-schema))]+ (mt/with-temp [:model/Transform transform transform-data]+ (let [result (inspector/discover-lenses transform)]+ (is (= :not-run (:status result)))+ (is (string? (:name result)))+ (is (nil? (:target result)))+ (is (= [] (:available_lenses result))))))))))++(deftest discover-lenses-ready-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "discover-lenses returns :ready with available lenses after transform is run"+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_inspector_rdy"}]+ (let [transform-data (make-mbql-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}+ target-name+ (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ ;; Run the transform to create the target table+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ ;; Reload transform to get latest state+ (let [transform' (t2/select-one :model/Transform tid)+ result (inspector/discover-lenses transform')]+ (is (= :ready (:status result)))+ (is (some? (:target result)))+ (is (seq (:sources result)))+ (is (seq (:available_lenses result)))+ (testing "generic-summary is always in the available lenses"+ (is (some #(= "generic-summary" (:id %)) (:available_lenses result)))))))))))++(deftest discover-lenses-source-info-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "discover-lenses populates source table info with fields"+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_inspector_src"}]+ (let [transform-data (make-mbql-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}+ target-name+ (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ (let [transform' (t2/select-one :model/Transform tid)+ result (inspector/discover-lenses transform')]+ (testing "sources contain table metadata"+ (let [source (first (:sources result))]+ (is (pos-int? (:table_id source)))+ (is (string? (:table_name source)))+ (is (pos-int? (:column_count source)))+ (is (seq (:fields source)))))+ (testing "target contains table metadata"+ (let [target (:target result)]+ (is (pos-int? (:table_id target)))+ (is (= target-name (:table_name target)))+ (is (pos-int? (:column_count target)))+ (is (seq (:fields target))))))))))))++;;; -------------------------------------------------- get-lens --------------------------------------------------++(deftest get-lens-generic-summary-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "get-lens returns generic-summary lens with sections and cards"+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_inspector_gs"}]+ (let [transform-data (make-mbql-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}+ target-name+ (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ (let [transform' (t2/select-one :model/Transform tid)+ lens (inspector/get-lens transform' "generic-summary" nil)]+ (is (= "generic-summary" (:id lens)))+ (is (= "Data Summary" (:display_name lens)))+ (is (seq (:sections lens)))+ (is (seq (:cards lens)))+ (testing "cards have required structure"+ (let [card (first (:cards lens))]+ (is (string? (:id card)))+ (is (string? (:title card)))+ (is (keyword? (:display card)))+ (is (map? (:dataset_query card))))))))))))++(deftest get-lens-join-analysis-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table :left-join)+ (testing "get-lens returns join-analysis lens for transforms with joins"+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_inspector_ja"}]+ (let [transform-data (make-mbql-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)+ :joins [{:fields "all"+ :strategy "left-join"+ :alias "Products"+ :condition [:=+ [:field (mt/id :orders :product_id)+ {:base-type "type/Integer"}]+ [:field (mt/id :products :id)+ {:base-type "type/Integer"+ :join-alias "Products"}]]+ :source-table (mt/id :products)}]}}+ target-name+ (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ (let [transform' (t2/select-one :model/Transform tid)+ lens (inspector/get-lens transform' "join-analysis" nil)]+ (is (= "join-analysis" (:id lens)))+ (is (= "Join Analysis" (:display_name lens)))+ (testing "has join stats section"+ (is (some #(= "join-stats" (:id %)) (:sections lens))))+ (testing "has base-count and join-step cards"+ (is (some #(= "base-count" (:id %)) (:cards lens)))+ (is (some #(re-matches #"join-step-\d+" (:id %)) (:cards lens))))+ (testing "has alert and drill lens triggers for outer joins"+ (is (seq (:alert_triggers lens)))+ (is (seq (:drill_lens_triggers lens)))))))))))++(deftest get-lens-inapplicable-throws-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "get-lens throws for inapplicable lens"+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_inspector_na"}]+ (let [transform-data (make-mbql-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}+ target-name+ (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ (let [transform' (t2/select-one :model/Transform tid)]+ ;; A simple query without joins shouldn't have join-analysis applicable+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Lens data not available"+ (inspector/get-lens transform' "join-analysis" nil))))))))))++;;; -------------------------------------------------- Context building --------------------------------------------------++(deftest discover-lenses-with-joins-has-join-info-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table :left-join)+ (testing "discover-lenses includes join-related lenses for transforms with joins"+ (transforms.tu/with-transform-cleanup!+ [{target-name :name} {:type :table+ :schema (default-schema)+ :name "g_inspector_ji"}]+ (let [transform-data (make-mbql-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)+ :joins [{:fields "all"+ :strategy "left-join"+ :alias "Products"+ :condition [:=+ [:field (mt/id :orders :product_id)+ {:base-type "type/Integer"}]+ [:field (mt/id :products :id)+ {:base-type "type/Integer"+ :join-alias "Products"}]]+ :source-table (mt/id :products)}]}}+ target-name+ (default-schema))]+ (mt/with-temp [:model/Transform {tid :id :as transform} transform-data]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-name 10000)+ (let [transform' (t2/select-one :model/Transform tid)+ result (inspector/discover-lenses transform')]+ (testing "join-analysis is in available lenses"+ (is (some #(= "join-analysis" (:id %)) (:available_lenses result))))+ (testing "multiple source tables are detected"+ (is (>= (count (:sources result)) 2))))))))))diff --git a/test/metabase/transforms/api/transform_job_test.clj b/test/metabase/transforms_rest/api/transform_job_test.cljsimilarity index 99%rename from test/metabase/transforms/api/transform_job_test.cljrename to test/metabase/transforms_rest/api/transform_job_test.cljindex ecabab1bccf..1eca4c040ae 100644--- a/test/metabase/transforms/api/transform_job_test.clj+++ b/test/metabase/transforms_rest/api/transform_job_test.clj@@ -1,4 +1,4 @@-(ns metabase.transforms.api.transform-job-test+(ns metabase.transforms-rest.api.transform-job-test(:require[clojure.test :refer :all][metabase.models.transforms.transform-job]diff --git a/test/metabase/transforms/api/transform_tag_test.clj b/test/metabase/transforms_rest/api/transform_tag_test.cljsimilarity index 99%rename from test/metabase/transforms/api/transform_tag_test.cljrename to test/metabase/transforms_rest/api/transform_tag_test.cljindex 3168327eaa5..b34ea230e8d 100644--- a/test/metabase/transforms/api/transform_tag_test.clj+++ b/test/metabase/transforms_rest/api/transform_tag_test.clj@@ -1,4 +1,4 @@-(ns metabase.transforms.api.transform-tag-test+(ns metabase.transforms-rest.api.transform-tag-test"Tests for transform tag CRUD API endpoints."(:require[clojure.test :refer :all]diff --git a/test/metabase/transforms/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljsimilarity index 95%rename from test/metabase/transforms/api/transform_test.cljrename to test/metabase/transforms_rest/api/transform_test.cljindex c476e43e85a..3d35e9aa173 100644--- a/test/metabase/transforms/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -1,4 +1,4 @@-(ns ^:mb/driver-tests metabase.transforms.api.transform-test+(ns ^:mb/driver-tests metabase.transforms-rest.api.transform-test"Tests for /api/transform endpoints."(:require[clojure.test :refer :all]@@ -12,7 +12,8 @@[metabase.query-processor :as qp][metabase.search.test-util :as search.tu][metabase.test :as mt]- [metabase.transforms.api.transform]+ [metabase.transforms-rest.api.transform]+ [metabase.transforms.crud][metabase.transforms.query-test-util :as query-test-util][metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :refer [get-test-schema@@ -1720,7 +1721,7 @@(let [schema (get-test-schema)](testing "Accepts any column if they were not extractable"(with-transform-cleanup! [table-name "fallback_test_unextracted"]- (with-redefs [metabase.transforms.api.transform/extract-all-columns-from-query+ (with-redefs [metabase.transforms.crud/extract-all-columns-from-query;; simulate lack of driver support for extraction(fn [_driver _database-id _query] nil)](let [response (mt/user-http-request :crowberto :post 200 "transform"@@ -1810,3 +1811,88 @@(search.tu/with-new-search-and-legacy-search(is (= #{native-id mbql-id}(search-transform-ids search-term))))))))))++;;; -------------------------------------------------- Inspector API --------------------------------------------------++(deftest inspect-lens-not-found-test+ (mt/with-premium-features #{}+ (testing "GET /api/transform/:id/inspect/:lens-id returns 404 for a nonexistent lens"+ (mt/with-temp [:model/Transform {transform-id :id} {}]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes+ (is (= "Lens data not available"+ (:message (mt/user-http-request :lucky :get 404+ (format "transform/%d/inspect/no-such-lens" transform-id)))))))))))++;;; -------------------------------------------------- Inspector Query API --------------------------------------------------++(deftest inspect-query-execute-test+ (mt/with-premium-features #{}+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "POST /api/transform/:id/inspect/:lens-id/query executes a query with inspector context"+ (mt/with-temp [:model/Transform {transform-id :id} {}]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes+ (let [mp (mt/metadata-provider)+ query (lib/aggregate (lib/query mp (lib.metadata/table mp (mt/id :orders))) (lib/count))+ result (mt/user-http-request :lucky :post 202+ (format "transform/%d/inspect/generic-summary/query" transform-id)+ {:query query})]+ (testing "returns completed query results"+ (is (= "completed" (:status result)))+ (is (pos? (:row_count result)))+ (is (seq (get-in result [:data :rows]))))+ (testing "context is set to transform-inspector"+ (is (= "transform-inspector" (:context result))))))))))))++(deftest inspect-query-with-lens-params-test+ (mt/with-premium-features #{}+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "POST /api/transform/:id/inspect/:lens-id/query passes lens_params through to query execution"+ (mt/with-temp [:model/Transform {transform-id :id} {}]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes+ (let [mp (mt/metadata-provider)+ query (lib/aggregate (lib/query mp (lib.metadata/table mp (mt/id :orders))) (lib/count))+ result (mt/user-http-request :lucky :post 202+ (format "transform/%d/inspect/unmatched-rows/query" transform-id)+ {:query query+ :lens_params {:join-index 0 :filter-col "status"}})]+ (is (= "completed" (:status result)))+ (is (= "transform-inspector" (:context result)))))))))))++(deftest inspect-query-permissions-test+ (mt/with-premium-features #{}+ (testing "POST /api/transform/:id/inspect/:lens-id/query requires transforms permission"+ (mt/with-temp [:model/Transform {transform-id :id} {}]+ (let [mp (mt/metadata-provider)]+ (testing "user without transform permission gets 403"+ (mt/user-http-request :rasta :post 403+ (format "transform/%d/inspect/generic-summary/query" transform-id)+ {:query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}))+ (testing "data analysts with transform permission can access"+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes+ (is (= "completed"+ (:status (mt/user-http-request :lucky :post 202+ (format "transform/%d/inspect/generic-summary/query" transform-id)+ {:query (lib/aggregate (lib/query mp (lib.metadata/table mp (mt/id :orders))) (lib/count))}))))))))))))++(deftest inspect-query-not-found-test+ (mt/with-premium-features #{}+ (testing "POST /api/transform/:id/inspect/:lens-id/query returns 404 for non-existent transform"+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/user-http-request :lucky :post 404+ "transform/999999/inspect/generic-summary/query"+ {:query (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))})))))++(deftest inspect-query-invalid-params-test+ (mt/with-premium-features #{}+ (testing "POST /api/transform/:id/inspect/:lens-id/query validates parameters"+ (mt/with-temp [:model/Transform {transform-id :id} {}]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes+ (testing "missing query body returns 400"+ (is (some? (:errors (mt/user-http-request :lucky :post 400+ (format "transform/%d/inspect/generic-summary/query" transform-id)+ {})))))))))))